Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1 year ago
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295
  1. # -*- test-case-name: twisted.web.test.test_http -*-
  2. # Copyright (c) Twisted Matrix Laboratories.
  3. # See LICENSE for details.
  4. """
  5. HyperText Transfer Protocol implementation.
  6. This is the basic server-side protocol implementation used by the Twisted
  7. Web server. It can parse HTTP 1.0 requests and supports many HTTP 1.1
  8. features as well. Additionally, some functionality implemented here is
  9. also useful for HTTP clients (such as the chunked encoding parser).
  10. @var CACHED: A marker value to be returned from cache-related request methods
  11. to indicate to the caller that a cached response will be usable and no
  12. response body should be generated.
  13. @var FOUND: An HTTP response code indicating a temporary redirect.
  14. @var NOT_MODIFIED: An HTTP response code indicating that a requested
  15. pre-condition (for example, the condition represented by an
  16. I{If-Modified-Since} header is present in the request) has succeeded. This
  17. indicates a response body cached by the client can be used.
  18. @var PRECONDITION_FAILED: An HTTP response code indicating that a requested
  19. pre-condition (for example, the condition represented by an I{If-None-Match}
  20. header is present in the request) has failed. This should typically
  21. indicate that the server has not taken the requested action.
  22. @var maxChunkSizeLineLength: Maximum allowable length of the CRLF-terminated
  23. line that indicates the size of a chunk and the extensions associated with
  24. it, as in the HTTP 1.1 chunked I{Transfer-Encoding} (RFC 7230 section 4.1).
  25. This limits how much data may be buffered when decoding the line.
  26. """
  27. __all__ = [
  28. "SWITCHING",
  29. "OK",
  30. "CREATED",
  31. "ACCEPTED",
  32. "NON_AUTHORITATIVE_INFORMATION",
  33. "NO_CONTENT",
  34. "RESET_CONTENT",
  35. "PARTIAL_CONTENT",
  36. "MULTI_STATUS",
  37. "MULTIPLE_CHOICE",
  38. "MOVED_PERMANENTLY",
  39. "FOUND",
  40. "SEE_OTHER",
  41. "NOT_MODIFIED",
  42. "USE_PROXY",
  43. "TEMPORARY_REDIRECT",
  44. "PERMANENT_REDIRECT",
  45. "BAD_REQUEST",
  46. "UNAUTHORIZED",
  47. "PAYMENT_REQUIRED",
  48. "FORBIDDEN",
  49. "NOT_FOUND",
  50. "NOT_ALLOWED",
  51. "NOT_ACCEPTABLE",
  52. "PROXY_AUTH_REQUIRED",
  53. "REQUEST_TIMEOUT",
  54. "CONFLICT",
  55. "GONE",
  56. "LENGTH_REQUIRED",
  57. "PRECONDITION_FAILED",
  58. "REQUEST_ENTITY_TOO_LARGE",
  59. "REQUEST_URI_TOO_LONG",
  60. "UNSUPPORTED_MEDIA_TYPE",
  61. "REQUESTED_RANGE_NOT_SATISFIABLE",
  62. "EXPECTATION_FAILED",
  63. "INTERNAL_SERVER_ERROR",
  64. "NOT_IMPLEMENTED",
  65. "BAD_GATEWAY",
  66. "SERVICE_UNAVAILABLE",
  67. "GATEWAY_TIMEOUT",
  68. "HTTP_VERSION_NOT_SUPPORTED",
  69. "INSUFFICIENT_STORAGE_SPACE",
  70. "NOT_EXTENDED",
  71. "RESPONSES",
  72. "CACHED",
  73. "urlparse",
  74. "parse_qs",
  75. "datetimeToString",
  76. "datetimeToLogString",
  77. "timegm",
  78. "stringToDatetime",
  79. "toChunk",
  80. "fromChunk",
  81. "parseContentRange",
  82. "StringTransport",
  83. "HTTPClient",
  84. "NO_BODY_CODES",
  85. "Request",
  86. "PotentialDataLoss",
  87. "HTTPChannel",
  88. "HTTPFactory",
  89. ]
  90. import base64
  91. import binascii
  92. import calendar
  93. import cgi
  94. import math
  95. import os
  96. import re
  97. import tempfile
  98. import time
  99. import warnings
  100. from io import BytesIO
  101. from typing import AnyStr, Callable, Optional, Tuple
  102. from urllib.parse import (
  103. ParseResultBytes,
  104. unquote_to_bytes as unquote,
  105. urlparse as _urlparse,
  106. )
  107. from zope.interface import Attribute, Interface, implementer, provider
  108. from incremental import Version
  109. from twisted.internet import address, interfaces, protocol
  110. from twisted.internet._producer_helpers import _PullToPush
  111. from twisted.internet.defer import Deferred
  112. from twisted.internet.interfaces import IProtocol
  113. from twisted.logger import Logger
  114. from twisted.protocols import basic, policies
  115. from twisted.python import log
  116. from twisted.python.compat import _PY37PLUS, nativeString, networkString
  117. from twisted.python.components import proxyForInterface
  118. from twisted.python.deprecate import deprecated
  119. from twisted.python.failure import Failure
  120. # twisted imports
  121. from twisted.web._responses import (
  122. ACCEPTED,
  123. BAD_GATEWAY,
  124. BAD_REQUEST,
  125. CONFLICT,
  126. CREATED,
  127. EXPECTATION_FAILED,
  128. FORBIDDEN,
  129. FOUND,
  130. GATEWAY_TIMEOUT,
  131. GONE,
  132. HTTP_VERSION_NOT_SUPPORTED,
  133. INSUFFICIENT_STORAGE_SPACE,
  134. INTERNAL_SERVER_ERROR,
  135. LENGTH_REQUIRED,
  136. MOVED_PERMANENTLY,
  137. MULTI_STATUS,
  138. MULTIPLE_CHOICE,
  139. NO_CONTENT,
  140. NON_AUTHORITATIVE_INFORMATION,
  141. NOT_ACCEPTABLE,
  142. NOT_ALLOWED,
  143. NOT_EXTENDED,
  144. NOT_FOUND,
  145. NOT_IMPLEMENTED,
  146. NOT_MODIFIED,
  147. OK,
  148. PARTIAL_CONTENT,
  149. PAYMENT_REQUIRED,
  150. PERMANENT_REDIRECT,
  151. PRECONDITION_FAILED,
  152. PROXY_AUTH_REQUIRED,
  153. REQUEST_ENTITY_TOO_LARGE,
  154. REQUEST_TIMEOUT,
  155. REQUEST_URI_TOO_LONG,
  156. REQUESTED_RANGE_NOT_SATISFIABLE,
  157. RESET_CONTENT,
  158. RESPONSES,
  159. SEE_OTHER,
  160. SERVICE_UNAVAILABLE,
  161. SWITCHING,
  162. TEMPORARY_REDIRECT,
  163. UNAUTHORIZED,
  164. UNSUPPORTED_MEDIA_TYPE,
  165. USE_PROXY,
  166. )
  167. from twisted.web.http_headers import Headers, _sanitizeLinearWhitespace
  168. from twisted.web.iweb import IAccessLogFormatter, INonQueuedRequestFactory, IRequest
  169. try:
  170. from twisted.web._http2 import H2Connection
  171. H2_ENABLED = True
  172. except ImportError:
  173. H2_ENABLED = False
  174. # A common request timeout -- 1 minute. This is roughly what nginx uses, and
  175. # so it seems to be a good choice for us too.
  176. _REQUEST_TIMEOUT = 1 * 60
  177. protocol_version = "HTTP/1.1"
  178. CACHED = """Magic constant returned by http.Request methods to set cache
  179. validation headers when the request is conditional and the value fails
  180. the condition."""
  181. # backwards compatibility
  182. responses = RESPONSES
  183. # datetime parsing and formatting
  184. weekdayname = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
  185. monthname = [
  186. None,
  187. "Jan",
  188. "Feb",
  189. "Mar",
  190. "Apr",
  191. "May",
  192. "Jun",
  193. "Jul",
  194. "Aug",
  195. "Sep",
  196. "Oct",
  197. "Nov",
  198. "Dec",
  199. ]
  200. weekdayname_lower = [name.lower() for name in weekdayname]
  201. monthname_lower = [name and name.lower() for name in monthname]
  202. def _parseHeader(line):
  203. # cgi.parse_header requires a str
  204. key, pdict = cgi.parse_header(line.decode("charmap"))
  205. # We want the key as bytes, and cgi.parse_multipart (which consumes
  206. # pdict) expects a dict of str keys but bytes values
  207. key = key.encode("charmap")
  208. pdict = {x: y.encode("charmap") for x, y in pdict.items()}
  209. return (key, pdict)
  210. def urlparse(url):
  211. """
  212. Parse an URL into six components.
  213. This is similar to C{urlparse.urlparse}, but rejects C{str} input
  214. and always produces C{bytes} output.
  215. @type url: C{bytes}
  216. @raise TypeError: The given url was a C{str} string instead of a
  217. C{bytes}.
  218. @return: The scheme, net location, path, params, query string, and fragment
  219. of the URL - all as C{bytes}.
  220. @rtype: C{ParseResultBytes}
  221. """
  222. if isinstance(url, str):
  223. raise TypeError("url must be bytes, not unicode")
  224. scheme, netloc, path, params, query, fragment = _urlparse(url)
  225. if isinstance(scheme, str):
  226. scheme = scheme.encode("ascii")
  227. netloc = netloc.encode("ascii")
  228. path = path.encode("ascii")
  229. query = query.encode("ascii")
  230. fragment = fragment.encode("ascii")
  231. return ParseResultBytes(scheme, netloc, path, params, query, fragment)
  232. def parse_qs(qs, keep_blank_values=0, strict_parsing=0):
  233. """
  234. Like C{cgi.parse_qs}, but with support for parsing byte strings on Python 3.
  235. @type qs: C{bytes}
  236. """
  237. d = {}
  238. items = [s2 for s1 in qs.split(b"&") for s2 in s1.split(b";")]
  239. for item in items:
  240. try:
  241. k, v = item.split(b"=", 1)
  242. except ValueError:
  243. if strict_parsing:
  244. raise
  245. continue
  246. if v or keep_blank_values:
  247. k = unquote(k.replace(b"+", b" "))
  248. v = unquote(v.replace(b"+", b" "))
  249. if k in d:
  250. d[k].append(v)
  251. else:
  252. d[k] = [v]
  253. return d
  254. def datetimeToString(msSinceEpoch=None):
  255. """
  256. Convert seconds since epoch to HTTP datetime string.
  257. @rtype: C{bytes}
  258. """
  259. if msSinceEpoch == None:
  260. msSinceEpoch = time.time()
  261. year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
  262. s = networkString(
  263. "%s, %02d %3s %4d %02d:%02d:%02d GMT"
  264. % (weekdayname[wd], day, monthname[month], year, hh, mm, ss)
  265. )
  266. return s
  267. def datetimeToLogString(msSinceEpoch=None):
  268. """
  269. Convert seconds since epoch to log datetime string.
  270. @rtype: C{str}
  271. """
  272. if msSinceEpoch == None:
  273. msSinceEpoch = time.time()
  274. year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
  275. s = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % (
  276. day,
  277. monthname[month],
  278. year,
  279. hh,
  280. mm,
  281. ss,
  282. )
  283. return s
  284. def timegm(year, month, day, hour, minute, second):
  285. """
  286. Convert time tuple in GMT to seconds since epoch, GMT
  287. """
  288. EPOCH = 1970
  289. if year < EPOCH:
  290. raise ValueError("Years prior to %d not supported" % (EPOCH,))
  291. assert 1 <= month <= 12
  292. days = 365 * (year - EPOCH) + calendar.leapdays(EPOCH, year)
  293. for i in range(1, month):
  294. days = days + calendar.mdays[i]
  295. if month > 2 and calendar.isleap(year):
  296. days = days + 1
  297. days = days + day - 1
  298. hours = days * 24 + hour
  299. minutes = hours * 60 + minute
  300. seconds = minutes * 60 + second
  301. return seconds
  302. def stringToDatetime(dateString):
  303. """
  304. Convert an HTTP date string (one of three formats) to seconds since epoch.
  305. @type dateString: C{bytes}
  306. """
  307. parts = nativeString(dateString).split()
  308. if not parts[0][0:3].lower() in weekdayname_lower:
  309. # Weekday is stupid. Might have been omitted.
  310. try:
  311. return stringToDatetime(b"Sun, " + dateString)
  312. except ValueError:
  313. # Guess not.
  314. pass
  315. partlen = len(parts)
  316. if (partlen == 5 or partlen == 6) and parts[1].isdigit():
  317. # 1st date format: Sun, 06 Nov 1994 08:49:37 GMT
  318. # (Note: "GMT" is literal, not a variable timezone)
  319. # (also handles without "GMT")
  320. # This is the normal format
  321. day = parts[1]
  322. month = parts[2]
  323. year = parts[3]
  324. time = parts[4]
  325. elif (partlen == 3 or partlen == 4) and parts[1].find("-") != -1:
  326. # 2nd date format: Sunday, 06-Nov-94 08:49:37 GMT
  327. # (Note: "GMT" is literal, not a variable timezone)
  328. # (also handles without without "GMT")
  329. # Two digit year, yucko.
  330. day, month, year = parts[1].split("-")
  331. time = parts[2]
  332. year = int(year)
  333. if year < 69:
  334. year = year + 2000
  335. elif year < 100:
  336. year = year + 1900
  337. elif len(parts) == 5:
  338. # 3rd date format: Sun Nov 6 08:49:37 1994
  339. # ANSI C asctime() format.
  340. day = parts[2]
  341. month = parts[1]
  342. year = parts[4]
  343. time = parts[3]
  344. else:
  345. raise ValueError("Unknown datetime format %r" % dateString)
  346. day = int(day)
  347. month = int(monthname_lower.index(month.lower()))
  348. year = int(year)
  349. hour, min, sec = map(int, time.split(":"))
  350. return int(timegm(year, month, day, hour, min, sec))
  351. def toChunk(data):
  352. """
  353. Convert string to a chunk.
  354. @type data: C{bytes}
  355. @returns: a tuple of C{bytes} representing the chunked encoding of data
  356. """
  357. return (networkString(f"{len(data):x}"), b"\r\n", data, b"\r\n")
  358. def _ishexdigits(b: bytes) -> bool:
  359. """
  360. Is the string case-insensitively hexidecimal?
  361. It must be composed of one or more characters in the ranges a-f, A-F
  362. and 0-9.
  363. """
  364. for c in b:
  365. if c not in b"0123456789abcdefABCDEF":
  366. return False
  367. return b != b""
  368. def _hexint(b: bytes) -> int:
  369. """
  370. Decode a hexadecimal integer.
  371. Unlike L{int(b, 16)}, this raises L{ValueError} when the integer has
  372. a prefix like C{b'0x'}, C{b'+'}, or C{b'-'}, which is desirable when
  373. parsing network protocols.
  374. """
  375. if not _ishexdigits(b):
  376. raise ValueError(b)
  377. return int(b, 16)
  378. def fromChunk(data: bytes) -> Tuple[bytes, bytes]:
  379. """
  380. Convert chunk to string.
  381. Note that this function is not specification compliant: it doesn't handle
  382. chunk extensions.
  383. @type data: C{bytes}
  384. @return: tuple of (result, remaining) - both C{bytes}.
  385. @raise ValueError: If the given data is not a correctly formatted chunked
  386. byte string.
  387. """
  388. prefix, rest = data.split(b"\r\n", 1)
  389. length = _hexint(prefix)
  390. if length < 0:
  391. raise ValueError("Chunk length must be >= 0, not %d" % (length,))
  392. if rest[length : length + 2] != b"\r\n":
  393. raise ValueError("chunk must end with CRLF")
  394. return rest[:length], rest[length + 2 :]
  395. def parseContentRange(header):
  396. """
  397. Parse a content-range header into (start, end, realLength).
  398. realLength might be None if real length is not known ('*').
  399. """
  400. kind, other = header.strip().split()
  401. if kind.lower() != "bytes":
  402. raise ValueError("a range of type %r is not supported")
  403. startend, realLength = other.split("/")
  404. start, end = map(int, startend.split("-"))
  405. if realLength == "*":
  406. realLength = None
  407. else:
  408. realLength = int(realLength)
  409. return (start, end, realLength)
  410. class _IDeprecatedHTTPChannelToRequestInterface(Interface):
  411. """
  412. The interface L{HTTPChannel} expects of L{Request}.
  413. """
  414. requestHeaders = Attribute(
  415. "A L{http_headers.Headers} instance giving all received HTTP request "
  416. "headers."
  417. )
  418. responseHeaders = Attribute(
  419. "A L{http_headers.Headers} instance holding all HTTP response "
  420. "headers to be sent."
  421. )
  422. def connectionLost(reason):
  423. """
  424. The underlying connection has been lost.
  425. @param reason: A failure instance indicating the reason why
  426. the connection was lost.
  427. @type reason: L{twisted.python.failure.Failure}
  428. """
  429. def gotLength(length):
  430. """
  431. Called when L{HTTPChannel} has determined the length, if any,
  432. of the incoming request's body.
  433. @param length: The length of the request's body.
  434. @type length: L{int} if the request declares its body's length
  435. and L{None} if it does not.
  436. """
  437. def handleContentChunk(data):
  438. """
  439. Deliver a received chunk of body data to the request. Note
  440. this does not imply chunked transfer encoding.
  441. @param data: The received chunk.
  442. @type data: L{bytes}
  443. """
  444. def parseCookies():
  445. """
  446. Parse the request's cookies out of received headers.
  447. """
  448. def requestReceived(command, path, version):
  449. """
  450. Called when the entire request, including its body, has been
  451. received.
  452. @param command: The request's HTTP command.
  453. @type command: L{bytes}
  454. @param path: The request's path. Note: this is actually what
  455. RFC7320 calls the URI.
  456. @type path: L{bytes}
  457. @param version: The request's HTTP version.
  458. @type version: L{bytes}
  459. """
  460. def __eq__(other: object) -> bool:
  461. """
  462. Determines if two requests are the same object.
  463. @param other: Another object whose identity will be compared
  464. to this instance's.
  465. @return: L{True} when the two are the same object and L{False}
  466. when not.
  467. """
  468. def __ne__(other: object) -> bool:
  469. """
  470. Determines if two requests are not the same object.
  471. @param other: Another object whose identity will be compared
  472. to this instance's.
  473. @return: L{True} when the two are not the same object and
  474. L{False} when they are.
  475. """
  476. def __hash__():
  477. """
  478. Generate a hash value for the request.
  479. @return: The request's hash value.
  480. @rtype: L{int}
  481. """
  482. class StringTransport:
  483. """
  484. I am a BytesIO wrapper that conforms for the transport API. I support
  485. the `writeSequence' method.
  486. """
  487. def __init__(self):
  488. self.s = BytesIO()
  489. def writeSequence(self, seq):
  490. self.s.write(b"".join(seq))
  491. def __getattr__(self, attr):
  492. return getattr(self.__dict__["s"], attr)
  493. class HTTPClient(basic.LineReceiver):
  494. """
  495. A client for HTTP 1.0.
  496. Notes:
  497. You probably want to send a 'Host' header with the name of the site you're
  498. connecting to, in order to not break name based virtual hosting.
  499. @ivar length: The length of the request body in bytes.
  500. @type length: C{int}
  501. @ivar firstLine: Are we waiting for the first header line?
  502. @type firstLine: C{bool}
  503. @ivar __buffer: The buffer that stores the response to the HTTP request.
  504. @type __buffer: A C{BytesIO} object.
  505. @ivar _header: Part or all of an HTTP request header.
  506. @type _header: C{bytes}
  507. """
  508. length = None
  509. firstLine = True
  510. __buffer = None
  511. _header = b""
  512. def sendCommand(self, command, path):
  513. self.transport.writeSequence([command, b" ", path, b" HTTP/1.0\r\n"])
  514. def sendHeader(self, name, value):
  515. if not isinstance(value, bytes):
  516. # XXX Deprecate this case
  517. value = networkString(str(value))
  518. santizedName = _sanitizeLinearWhitespace(name)
  519. santizedValue = _sanitizeLinearWhitespace(value)
  520. self.transport.writeSequence([santizedName, b": ", santizedValue, b"\r\n"])
  521. def endHeaders(self):
  522. self.transport.write(b"\r\n")
  523. def extractHeader(self, header):
  524. """
  525. Given a complete HTTP header, extract the field name and value and
  526. process the header.
  527. @param header: a complete HTTP request header of the form
  528. 'field-name: value'.
  529. @type header: C{bytes}
  530. """
  531. key, val = header.split(b":", 1)
  532. val = val.lstrip()
  533. self.handleHeader(key, val)
  534. if key.lower() == b"content-length":
  535. self.length = int(val)
  536. def lineReceived(self, line):
  537. """
  538. Parse the status line and headers for an HTTP request.
  539. @param line: Part of an HTTP request header. Request bodies are parsed
  540. in L{HTTPClient.rawDataReceived}.
  541. @type line: C{bytes}
  542. """
  543. if self.firstLine:
  544. self.firstLine = False
  545. l = line.split(None, 2)
  546. version = l[0]
  547. status = l[1]
  548. try:
  549. message = l[2]
  550. except IndexError:
  551. # sometimes there is no message
  552. message = b""
  553. self.handleStatus(version, status, message)
  554. return
  555. if not line:
  556. if self._header != b"":
  557. # Only extract headers if there are any
  558. self.extractHeader(self._header)
  559. self.__buffer = BytesIO()
  560. self.handleEndHeaders()
  561. self.setRawMode()
  562. return
  563. if line.startswith(b"\t") or line.startswith(b" "):
  564. # This line is part of a multiline header. According to RFC 822, in
  565. # "unfolding" multiline headers you do not strip the leading
  566. # whitespace on the continuing line.
  567. self._header = self._header + line
  568. elif self._header:
  569. # This line starts a new header, so process the previous one.
  570. self.extractHeader(self._header)
  571. self._header = line
  572. else: # First header
  573. self._header = line
  574. def connectionLost(self, reason):
  575. self.handleResponseEnd()
  576. def handleResponseEnd(self):
  577. """
  578. The response has been completely received.
  579. This callback may be invoked more than once per request.
  580. """
  581. if self.__buffer is not None:
  582. b = self.__buffer.getvalue()
  583. self.__buffer = None
  584. self.handleResponse(b)
  585. def handleResponsePart(self, data):
  586. self.__buffer.write(data)
  587. def connectionMade(self):
  588. pass
  589. def handleStatus(self, version, status, message):
  590. """
  591. Called when the status-line is received.
  592. @param version: e.g. 'HTTP/1.0'
  593. @param status: e.g. '200'
  594. @type status: C{bytes}
  595. @param message: e.g. 'OK'
  596. """
  597. def handleHeader(self, key, val):
  598. """
  599. Called every time a header is received.
  600. """
  601. def handleEndHeaders(self):
  602. """
  603. Called when all headers have been received.
  604. """
  605. def rawDataReceived(self, data):
  606. if self.length is not None:
  607. data, rest = data[: self.length], data[self.length :]
  608. self.length -= len(data)
  609. else:
  610. rest = b""
  611. self.handleResponsePart(data)
  612. if self.length == 0:
  613. self.handleResponseEnd()
  614. self.setLineMode(rest)
  615. # response codes that must have empty bodies
  616. NO_BODY_CODES = (204, 304)
  617. # Sentinel object that detects people explicitly passing `queued` to Request.
  618. _QUEUED_SENTINEL = object()
  619. def _getContentFile(length):
  620. """
  621. Get a writeable file-like object to which request content can be written.
  622. """
  623. if length is not None and length < 100000:
  624. return BytesIO()
  625. return tempfile.TemporaryFile()
  626. _hostHeaderExpression = re.compile(br"^\[?(?P<host>.*?)\]?(:\d+)?$")
  627. @implementer(interfaces.IConsumer, _IDeprecatedHTTPChannelToRequestInterface)
  628. class Request:
  629. """
  630. A HTTP request.
  631. Subclasses should override the process() method to determine how
  632. the request will be processed.
  633. @ivar method: The HTTP method that was used, e.g. C{b'GET'}.
  634. @type method: L{bytes}
  635. @ivar uri: The full encoded URI which was requested (including query
  636. arguments), e.g. C{b'/a/b%20/c?q=v'}.
  637. @type uri: L{bytes}
  638. @ivar path: The encoded path of the request URI (not including query
  639. arguments), e.g. C{b'/a/b%20/c'}.
  640. @type path: L{bytes}
  641. @ivar args: A mapping of decoded query argument names as L{bytes} to
  642. corresponding query argument values as L{list}s of L{bytes}.
  643. For example, for a URI with C{foo=bar&foo=baz&quux=spam}
  644. as its query part C{args} will be C{{b'foo': [b'bar', b'baz'],
  645. b'quux': [b'spam']}}.
  646. @type args: L{dict} of L{bytes} to L{list} of L{bytes}
  647. @ivar content: A file-like object giving the request body. This may be
  648. a file on disk, an L{io.BytesIO}, or some other type. The
  649. implementation is free to decide on a per-request basis.
  650. @type content: L{typing.BinaryIO}
  651. @ivar cookies: The cookies that will be sent in the response.
  652. @type cookies: L{list} of L{bytes}
  653. @type requestHeaders: L{http_headers.Headers}
  654. @ivar requestHeaders: All received HTTP request headers.
  655. @type responseHeaders: L{http_headers.Headers}
  656. @ivar responseHeaders: All HTTP response headers to be sent.
  657. @ivar notifications: A L{list} of L{Deferred}s which are waiting for
  658. notification that the response to this request has been finished
  659. (successfully or with an error). Don't use this attribute directly,
  660. instead use the L{Request.notifyFinish} method.
  661. @ivar _disconnected: A flag which is C{False} until the connection over
  662. which this request was received is closed and which is C{True} after
  663. that.
  664. @type _disconnected: L{bool}
  665. @ivar _log: A logger instance for request related messages.
  666. @type _log: L{twisted.logger.Logger}
  667. """
  668. producer = None
  669. finished = 0
  670. code = OK
  671. code_message = RESPONSES[OK]
  672. method = b"(no method yet)"
  673. clientproto = b"(no clientproto yet)"
  674. uri = b"(no uri yet)"
  675. startedWriting = 0
  676. chunked = 0
  677. sentLength = 0 # content-length of response, or total bytes sent via chunking
  678. etag = None
  679. lastModified = None
  680. args = None
  681. path = None
  682. content = None
  683. _forceSSL = 0
  684. _disconnected = False
  685. _log = Logger()
  686. def __init__(self, channel, queued=_QUEUED_SENTINEL):
  687. """
  688. @param channel: the channel we're connected to.
  689. @param queued: (deprecated) are we in the request queue, or can we
  690. start writing to the transport?
  691. """
  692. self.notifications = []
  693. self.channel = channel
  694. # Cache the client and server information, we'll need this
  695. # later to be serialized and sent with the request so CGIs
  696. # will work remotely
  697. self.client = self.channel.getPeer()
  698. self.host = self.channel.getHost()
  699. self.requestHeaders: Headers = Headers()
  700. self.received_cookies = {}
  701. self.responseHeaders = Headers()
  702. self.cookies = [] # outgoing cookies
  703. self.transport = self.channel.transport
  704. if queued is _QUEUED_SENTINEL:
  705. queued = False
  706. self.queued = queued
  707. def _cleanup(self):
  708. """
  709. Called when have finished responding and are no longer queued.
  710. """
  711. if self.producer:
  712. self._log.failure(
  713. "",
  714. Failure(RuntimeError(f"Producer was not unregistered for {self.uri}")),
  715. )
  716. self.unregisterProducer()
  717. self.channel.requestDone(self)
  718. del self.channel
  719. if self.content is not None:
  720. try:
  721. self.content.close()
  722. except OSError:
  723. # win32 suckiness, no idea why it does this
  724. pass
  725. del self.content
  726. for d in self.notifications:
  727. d.callback(None)
  728. self.notifications = []
  729. # methods for channel - end users should not use these
  730. @deprecated(Version("Twisted", 16, 3, 0))
  731. def noLongerQueued(self):
  732. """
  733. Notify the object that it is no longer queued.
  734. We start writing whatever data we have to the transport, etc.
  735. This method is not intended for users.
  736. In 16.3 this method was changed to become a no-op, as L{Request}
  737. objects are now never queued.
  738. """
  739. pass
  740. def gotLength(self, length):
  741. """
  742. Called when HTTP channel got length of content in this request.
  743. This method is not intended for users.
  744. @param length: The length of the request body, as indicated by the
  745. request headers. L{None} if the request headers do not indicate a
  746. length.
  747. """
  748. self.content = _getContentFile(length)
  749. def parseCookies(self):
  750. """
  751. Parse cookie headers.
  752. This method is not intended for users.
  753. """
  754. cookieheaders = self.requestHeaders.getRawHeaders(b"cookie")
  755. if cookieheaders is None:
  756. return
  757. for cookietxt in cookieheaders:
  758. if cookietxt:
  759. for cook in cookietxt.split(b";"):
  760. cook = cook.lstrip()
  761. try:
  762. k, v = cook.split(b"=", 1)
  763. self.received_cookies[k] = v
  764. except ValueError:
  765. pass
  766. def handleContentChunk(self, data):
  767. """
  768. Write a chunk of data.
  769. This method is not intended for users.
  770. """
  771. self.content.write(data)
  772. def requestReceived(self, command, path, version):
  773. """
  774. Called by channel when all data has been received.
  775. This method is not intended for users.
  776. @type command: C{bytes}
  777. @param command: The HTTP verb of this request. This has the case
  778. supplied by the client (eg, it maybe "get" rather than "GET").
  779. @type path: C{bytes}
  780. @param path: The URI of this request.
  781. @type version: C{bytes}
  782. @param version: The HTTP version of this request.
  783. """
  784. clength = self.content.tell()
  785. self.content.seek(0, 0)
  786. self.args = {}
  787. self.method, self.uri = command, path
  788. self.clientproto = version
  789. x = self.uri.split(b"?", 1)
  790. if len(x) == 1:
  791. self.path = self.uri
  792. else:
  793. self.path, argstring = x
  794. self.args = parse_qs(argstring, 1)
  795. # Argument processing
  796. args = self.args
  797. ctype = self.requestHeaders.getRawHeaders(b"content-type")
  798. if ctype is not None:
  799. ctype = ctype[0]
  800. if self.method == b"POST" and ctype and clength:
  801. mfd = b"multipart/form-data"
  802. key, pdict = _parseHeader(ctype)
  803. # This weird CONTENT-LENGTH param is required by
  804. # cgi.parse_multipart() in some versions of Python 3.7+, see
  805. # bpo-29979. It looks like this will be relaxed and backported, see
  806. # https://github.com/python/cpython/pull/8530.
  807. pdict["CONTENT-LENGTH"] = clength
  808. if key == b"application/x-www-form-urlencoded":
  809. args.update(parse_qs(self.content.read(), 1))
  810. elif key == mfd:
  811. try:
  812. if _PY37PLUS:
  813. cgiArgs = cgi.parse_multipart(
  814. self.content,
  815. pdict,
  816. encoding="utf8",
  817. errors="surrogateescape",
  818. )
  819. else:
  820. cgiArgs = cgi.parse_multipart(self.content, pdict)
  821. if _PY37PLUS:
  822. # The parse_multipart function on Python 3.7+
  823. # decodes the header bytes as iso-8859-1 and
  824. # decodes the body bytes as utf8 with
  825. # surrogateescape -- we want bytes
  826. self.args.update(
  827. {
  828. x.encode("iso-8859-1"): [
  829. z.encode("utf8", "surrogateescape")
  830. if isinstance(z, str)
  831. else z
  832. for z in y
  833. ]
  834. for x, y in cgiArgs.items()
  835. if isinstance(x, str)
  836. }
  837. )
  838. else:
  839. # The parse_multipart function on Python 3
  840. # decodes the header bytes as iso-8859-1 and
  841. # returns a str key -- we want bytes so encode
  842. # it back
  843. self.args.update(
  844. {x.encode("iso-8859-1"): y for x, y in cgiArgs.items()}
  845. )
  846. except Exception as e:
  847. # It was a bad request, or we got a signal.
  848. self.channel._respondToBadRequestAndDisconnect()
  849. if isinstance(e, (TypeError, ValueError, KeyError)):
  850. return
  851. else:
  852. # If it's not a userspace error from CGI, reraise
  853. raise
  854. self.content.seek(0, 0)
  855. self.process()
  856. def __repr__(self) -> str:
  857. """
  858. Return a string description of the request including such information
  859. as the request method and request URI.
  860. @return: A string loosely describing this L{Request} object.
  861. @rtype: L{str}
  862. """
  863. return "<{} at 0x{:x} method={} uri={} clientproto={}>".format(
  864. self.__class__.__name__,
  865. id(self),
  866. nativeString(self.method),
  867. nativeString(self.uri),
  868. nativeString(self.clientproto),
  869. )
  870. def process(self):
  871. """
  872. Override in subclasses.
  873. This method is not intended for users.
  874. """
  875. pass
  876. # consumer interface
  877. def registerProducer(self, producer, streaming):
  878. """
  879. Register a producer.
  880. """
  881. if self.producer:
  882. raise ValueError(
  883. "registering producer %s before previous one (%s) was "
  884. "unregistered" % (producer, self.producer)
  885. )
  886. self.streamingProducer = streaming
  887. self.producer = producer
  888. self.channel.registerProducer(producer, streaming)
  889. def unregisterProducer(self):
  890. """
  891. Unregister the producer.
  892. """
  893. self.channel.unregisterProducer()
  894. self.producer = None
  895. # The following is the public interface that people should be
  896. # writing to.
  897. def getHeader(self, key: AnyStr) -> Optional[AnyStr]:
  898. """
  899. Get an HTTP request header.
  900. @type key: C{bytes} or C{str}
  901. @param key: The name of the header to get the value of.
  902. @rtype: C{bytes} or C{str} or L{None}
  903. @return: The value of the specified header, or L{None} if that header
  904. was not present in the request. The string type of the result
  905. matches the type of C{key}.
  906. """
  907. value = self.requestHeaders.getRawHeaders(key)
  908. if value is not None:
  909. return value[-1]
  910. return None
  911. def getCookie(self, key):
  912. """
  913. Get a cookie that was sent from the network.
  914. @type key: C{bytes}
  915. @param key: The name of the cookie to get.
  916. @rtype: C{bytes} or C{None}
  917. @returns: The value of the specified cookie, or L{None} if that cookie
  918. was not present in the request.
  919. """
  920. return self.received_cookies.get(key)
  921. def notifyFinish(self):
  922. """
  923. Notify when the response to this request has finished.
  924. @note: There are some caveats around the reliability of the delivery of
  925. this notification.
  926. 1. If this L{Request}'s channel is paused, the notification
  927. will not be delivered. This can happen in one of two ways;
  928. either you can call C{request.transport.pauseProducing}
  929. yourself, or,
  930. 2. In order to deliver this notification promptly when a client
  931. disconnects, the reactor must continue reading from the
  932. transport, so that it can tell when the underlying network
  933. connection has gone away. Twisted Web will only keep
  934. reading up until a finite (small) maximum buffer size before
  935. it gives up and pauses the transport itself. If this
  936. occurs, you will not discover that the connection has gone
  937. away until a timeout fires or until the application attempts
  938. to send some data via L{Request.write}.
  939. 3. It is theoretically impossible to distinguish between
  940. successfully I{sending} a response and the peer successfully
  941. I{receiving} it. There are several networking edge cases
  942. where the L{Deferred}s returned by C{notifyFinish} will
  943. indicate success, but the data will never be received.
  944. There are also edge cases where the connection will appear
  945. to fail, but in reality the response was delivered. As a
  946. result, the information provided by the result of the
  947. L{Deferred}s returned by this method should be treated as a
  948. guess; do not make critical decisions in your applications
  949. based upon it.
  950. @rtype: L{Deferred}
  951. @return: A L{Deferred} which will be triggered when the request is
  952. finished -- with a L{None} value if the request finishes
  953. successfully or with an error if the request is interrupted by an
  954. error (for example, the client closing the connection prematurely).
  955. """
  956. self.notifications.append(Deferred())
  957. return self.notifications[-1]
  958. def finish(self):
  959. """
  960. Indicate that all response data has been written to this L{Request}.
  961. """
  962. if self._disconnected:
  963. raise RuntimeError(
  964. "Request.finish called on a request after its connection was lost; "
  965. "use Request.notifyFinish to keep track of this."
  966. )
  967. if self.finished:
  968. warnings.warn("Warning! request.finish called twice.", stacklevel=2)
  969. return
  970. if not self.startedWriting:
  971. # write headers
  972. self.write(b"")
  973. if self.chunked:
  974. # write last chunk and closing CRLF
  975. self.channel.write(b"0\r\n\r\n")
  976. # log request
  977. if hasattr(self.channel, "factory") and self.channel.factory is not None:
  978. self.channel.factory.log(self)
  979. self.finished = 1
  980. if not self.queued:
  981. self._cleanup()
  982. def write(self, data):
  983. """
  984. Write some data as a result of an HTTP request. The first
  985. time this is called, it writes out response data.
  986. @type data: C{bytes}
  987. @param data: Some bytes to be sent as part of the response body.
  988. """
  989. if self.finished:
  990. raise RuntimeError(
  991. "Request.write called on a request after " "Request.finish was called."
  992. )
  993. if self._disconnected:
  994. # Don't attempt to write any data to a disconnected client.
  995. # The RuntimeError exception will be thrown as usual when
  996. # request.finish is called
  997. return
  998. if not self.startedWriting:
  999. self.startedWriting = 1
  1000. version = self.clientproto
  1001. code = b"%d" % (self.code,)
  1002. reason = self.code_message
  1003. headers = []
  1004. # if we don't have a content length, we send data in
  1005. # chunked mode, so that we can support pipelining in
  1006. # persistent connections.
  1007. if (
  1008. (version == b"HTTP/1.1")
  1009. and (self.responseHeaders.getRawHeaders(b"content-length") is None)
  1010. and self.method != b"HEAD"
  1011. and self.code not in NO_BODY_CODES
  1012. ):
  1013. headers.append((b"Transfer-Encoding", b"chunked"))
  1014. self.chunked = 1
  1015. if self.lastModified is not None:
  1016. if self.responseHeaders.hasHeader(b"last-modified"):
  1017. self._log.info(
  1018. "Warning: last-modified specified both in"
  1019. " header list and lastModified attribute."
  1020. )
  1021. else:
  1022. self.responseHeaders.setRawHeaders(
  1023. b"last-modified", [datetimeToString(self.lastModified)]
  1024. )
  1025. if self.etag is not None:
  1026. self.responseHeaders.setRawHeaders(b"ETag", [self.etag])
  1027. for name, values in self.responseHeaders.getAllRawHeaders():
  1028. for value in values:
  1029. headers.append((name, value))
  1030. for cookie in self.cookies:
  1031. headers.append((b"Set-Cookie", cookie))
  1032. self.channel.writeHeaders(version, code, reason, headers)
  1033. # if this is a "HEAD" request, we shouldn't return any data
  1034. if self.method == b"HEAD":
  1035. self.write = lambda data: None
  1036. return
  1037. # for certain result codes, we should never return any data
  1038. if self.code in NO_BODY_CODES:
  1039. self.write = lambda data: None
  1040. return
  1041. self.sentLength = self.sentLength + len(data)
  1042. if data:
  1043. if self.chunked:
  1044. self.channel.writeSequence(toChunk(data))
  1045. else:
  1046. self.channel.write(data)
  1047. def addCookie(
  1048. self,
  1049. k,
  1050. v,
  1051. expires=None,
  1052. domain=None,
  1053. path=None,
  1054. max_age=None,
  1055. comment=None,
  1056. secure=None,
  1057. httpOnly=False,
  1058. sameSite=None,
  1059. ):
  1060. """
  1061. Set an outgoing HTTP cookie.
  1062. In general, you should consider using sessions instead of cookies, see
  1063. L{twisted.web.server.Request.getSession} and the
  1064. L{twisted.web.server.Session} class for details.
  1065. @param k: cookie name
  1066. @type k: L{bytes} or L{str}
  1067. @param v: cookie value
  1068. @type v: L{bytes} or L{str}
  1069. @param expires: cookie expire attribute value in
  1070. "Wdy, DD Mon YYYY HH:MM:SS GMT" format
  1071. @type expires: L{bytes} or L{str}
  1072. @param domain: cookie domain
  1073. @type domain: L{bytes} or L{str}
  1074. @param path: cookie path
  1075. @type path: L{bytes} or L{str}
  1076. @param max_age: cookie expiration in seconds from reception
  1077. @type max_age: L{bytes} or L{str}
  1078. @param comment: cookie comment
  1079. @type comment: L{bytes} or L{str}
  1080. @param secure: direct browser to send the cookie on encrypted
  1081. connections only
  1082. @type secure: L{bool}
  1083. @param httpOnly: direct browser not to expose cookies through channels
  1084. other than HTTP (and HTTPS) requests
  1085. @type httpOnly: L{bool}
  1086. @param sameSite: One of L{None} (default), C{'lax'} or C{'strict'}.
  1087. Direct browsers not to send this cookie on cross-origin requests.
  1088. Please see:
  1089. U{https://tools.ietf.org/html/draft-west-first-party-cookies-07}
  1090. @type sameSite: L{None}, L{bytes} or L{str}
  1091. @raise ValueError: If the value for C{sameSite} is not supported.
  1092. """
  1093. def _ensureBytes(val):
  1094. """
  1095. Ensure that C{val} is bytes, encoding using UTF-8 if
  1096. needed.
  1097. @param val: L{bytes} or L{str}
  1098. @return: L{bytes}
  1099. """
  1100. if val is None:
  1101. # It's None, so we don't want to touch it
  1102. return val
  1103. if isinstance(val, bytes):
  1104. return val
  1105. else:
  1106. return val.encode("utf8")
  1107. def _sanitize(val):
  1108. r"""
  1109. Replace linear whitespace (C{\r}, C{\n}, C{\r\n}) and
  1110. semicolons C{;} in C{val} with a single space.
  1111. @param val: L{bytes}
  1112. @return: L{bytes}
  1113. """
  1114. return _sanitizeLinearWhitespace(val).replace(b";", b" ")
  1115. cookie = _sanitize(_ensureBytes(k)) + b"=" + _sanitize(_ensureBytes(v))
  1116. if expires is not None:
  1117. cookie = cookie + b"; Expires=" + _sanitize(_ensureBytes(expires))
  1118. if domain is not None:
  1119. cookie = cookie + b"; Domain=" + _sanitize(_ensureBytes(domain))
  1120. if path is not None:
  1121. cookie = cookie + b"; Path=" + _sanitize(_ensureBytes(path))
  1122. if max_age is not None:
  1123. cookie = cookie + b"; Max-Age=" + _sanitize(_ensureBytes(max_age))
  1124. if comment is not None:
  1125. cookie = cookie + b"; Comment=" + _sanitize(_ensureBytes(comment))
  1126. if secure:
  1127. cookie = cookie + b"; Secure"
  1128. if httpOnly:
  1129. cookie = cookie + b"; HttpOnly"
  1130. if sameSite:
  1131. sameSite = _ensureBytes(sameSite).lower()
  1132. if sameSite not in [b"lax", b"strict"]:
  1133. raise ValueError("Invalid value for sameSite: " + repr(sameSite))
  1134. cookie += b"; SameSite=" + sameSite
  1135. self.cookies.append(cookie)
  1136. def setResponseCode(self, code, message=None):
  1137. """
  1138. Set the HTTP response code.
  1139. @type code: L{int}
  1140. @type message: L{bytes}
  1141. """
  1142. if not isinstance(code, int):
  1143. raise TypeError("HTTP response code must be int or long")
  1144. self.code = code
  1145. if message:
  1146. if not isinstance(message, bytes):
  1147. raise TypeError("HTTP response status message must be bytes")
  1148. self.code_message = message
  1149. else:
  1150. self.code_message = RESPONSES.get(code, b"Unknown Status")
  1151. def setHeader(self, name, value):
  1152. """
  1153. Set an HTTP response header. Overrides any previously set values for
  1154. this header.
  1155. @type name: L{bytes} or L{str}
  1156. @param name: The name of the header for which to set the value.
  1157. @type value: L{bytes} or L{str}
  1158. @param value: The value to set for the named header. A L{str} will be
  1159. UTF-8 encoded, which may not interoperable with other
  1160. implementations. Avoid passing non-ASCII characters if possible.
  1161. """
  1162. self.responseHeaders.setRawHeaders(name, [value])
  1163. def redirect(self, url):
  1164. """
  1165. Utility function that does a redirect.
  1166. Set the response code to L{FOUND} and the I{Location} header to the
  1167. given URL.
  1168. The request should have C{finish()} called after this.
  1169. @param url: I{Location} header value.
  1170. @type url: L{bytes} or L{str}
  1171. """
  1172. self.setResponseCode(FOUND)
  1173. self.setHeader(b"location", url)
  1174. def setLastModified(self, when):
  1175. """
  1176. Set the C{Last-Modified} time for the response to this request.
  1177. If I am called more than once, I ignore attempts to set
  1178. Last-Modified earlier, only replacing the Last-Modified time
  1179. if it is to a later value.
  1180. If I am a conditional request, I may modify my response code
  1181. to L{NOT_MODIFIED} if appropriate for the time given.
  1182. @param when: The last time the resource being returned was
  1183. modified, in seconds since the epoch.
  1184. @type when: number
  1185. @return: If I am a I{If-Modified-Since} conditional request and
  1186. the time given is not newer than the condition, I return
  1187. L{http.CACHED<CACHED>} to indicate that you should write no
  1188. body. Otherwise, I return a false value.
  1189. """
  1190. # time.time() may be a float, but the HTTP-date strings are
  1191. # only good for whole seconds.
  1192. when = int(math.ceil(when))
  1193. if (not self.lastModified) or (self.lastModified < when):
  1194. self.lastModified = when
  1195. modifiedSince = self.getHeader(b"if-modified-since")
  1196. if modifiedSince:
  1197. firstPart = modifiedSince.split(b";", 1)[0]
  1198. try:
  1199. modifiedSince = stringToDatetime(firstPart)
  1200. except ValueError:
  1201. return None
  1202. if modifiedSince >= self.lastModified:
  1203. self.setResponseCode(NOT_MODIFIED)
  1204. return CACHED
  1205. return None
  1206. def setETag(self, etag):
  1207. """
  1208. Set an C{entity tag} for the outgoing response.
  1209. That's \"entity tag\" as in the HTTP/1.1 C{ETag} header, \"used
  1210. for comparing two or more entities from the same requested
  1211. resource.\"
  1212. If I am a conditional request, I may modify my response code
  1213. to L{NOT_MODIFIED} or L{PRECONDITION_FAILED}, if appropriate
  1214. for the tag given.
  1215. @param etag: The entity tag for the resource being returned.
  1216. @type etag: string
  1217. @return: If I am a C{If-None-Match} conditional request and
  1218. the tag matches one in the request, I return
  1219. L{http.CACHED<CACHED>} to indicate that you should write
  1220. no body. Otherwise, I return a false value.
  1221. """
  1222. if etag:
  1223. self.etag = etag
  1224. tags = self.getHeader(b"if-none-match")
  1225. if tags:
  1226. tags = tags.split()
  1227. if (etag in tags) or (b"*" in tags):
  1228. self.setResponseCode(
  1229. ((self.method in (b"HEAD", b"GET")) and NOT_MODIFIED)
  1230. or PRECONDITION_FAILED
  1231. )
  1232. return CACHED
  1233. return None
  1234. def getAllHeaders(self):
  1235. """
  1236. Return dictionary mapping the names of all received headers to the last
  1237. value received for each.
  1238. Since this method does not return all header information,
  1239. C{self.requestHeaders.getAllRawHeaders()} may be preferred.
  1240. """
  1241. headers = {}
  1242. for k, v in self.requestHeaders.getAllRawHeaders():
  1243. headers[k.lower()] = v[-1]
  1244. return headers
  1245. def getRequestHostname(self):
  1246. """
  1247. Get the hostname that the HTTP client passed in to the request.
  1248. @see: L{IRequest.getRequestHostname}
  1249. @returns: the requested hostname
  1250. @rtype: C{bytes}
  1251. """
  1252. host = self.getHeader(b"host")
  1253. if host is not None:
  1254. match = _hostHeaderExpression.match(host)
  1255. if match is not None:
  1256. return match.group("host")
  1257. return networkString(self.getHost().host)
  1258. def getHost(self):
  1259. """
  1260. Get my originally requesting transport's host.
  1261. Don't rely on the 'transport' attribute, since Request objects may be
  1262. copied remotely. For information on this method's return value, see
  1263. L{twisted.internet.tcp.Port}.
  1264. """
  1265. return self.host
  1266. def setHost(self, host, port, ssl=0):
  1267. """
  1268. Change the host and port the request thinks it's using.
  1269. This method is useful for working with reverse HTTP proxies (e.g.
  1270. both Squid and Apache's mod_proxy can do this), when the address
  1271. the HTTP client is using is different than the one we're listening on.
  1272. For example, Apache may be listening on https://www.example.com/, and
  1273. then forwarding requests to http://localhost:8080/, but we don't want
  1274. HTML produced by Twisted to say b'http://localhost:8080/', they should
  1275. say b'https://www.example.com/', so we do::
  1276. request.setHost(b'www.example.com', 443, ssl=1)
  1277. @type host: C{bytes}
  1278. @param host: The value to which to change the host header.
  1279. @type ssl: C{bool}
  1280. @param ssl: A flag which, if C{True}, indicates that the request is
  1281. considered secure (if C{True}, L{isSecure} will return C{True}).
  1282. """
  1283. self._forceSSL = ssl # set first so isSecure will work
  1284. if self.isSecure():
  1285. default = 443
  1286. else:
  1287. default = 80
  1288. if port == default:
  1289. hostHeader = host
  1290. else:
  1291. hostHeader = b"%b:%d" % (host, port)
  1292. self.requestHeaders.setRawHeaders(b"host", [hostHeader])
  1293. self.host = address.IPv4Address("TCP", host, port)
  1294. @deprecated(Version("Twisted", 18, 4, 0), replacement="getClientAddress")
  1295. def getClientIP(self):
  1296. """
  1297. Return the IP address of the client who submitted this request.
  1298. This method is B{deprecated}. Use L{getClientAddress} instead.
  1299. @returns: the client IP address
  1300. @rtype: C{str}
  1301. """
  1302. if isinstance(self.client, (address.IPv4Address, address.IPv6Address)):
  1303. return self.client.host
  1304. else:
  1305. return None
  1306. def getClientAddress(self):
  1307. """
  1308. Return the address of the client who submitted this request.
  1309. This may not be a network address (e.g., a server listening on
  1310. a UNIX domain socket will cause this to return
  1311. L{UNIXAddress}). Callers must check the type of the returned
  1312. address.
  1313. @since: 18.4
  1314. @return: the client's address.
  1315. @rtype: L{IAddress}
  1316. """
  1317. return self.client
  1318. def isSecure(self):
  1319. """
  1320. Return L{True} if this request is using a secure transport.
  1321. Normally this method returns L{True} if this request's L{HTTPChannel}
  1322. instance is using a transport that implements
  1323. L{interfaces.ISSLTransport}.
  1324. This will also return L{True} if L{Request.setHost} has been called
  1325. with C{ssl=True}.
  1326. @returns: L{True} if this request is secure
  1327. @rtype: C{bool}
  1328. """
  1329. if self._forceSSL:
  1330. return True
  1331. channel = getattr(self, "channel", None)
  1332. if channel is None:
  1333. return False
  1334. return channel.isSecure()
  1335. def _authorize(self):
  1336. # Authorization, (mostly) per the RFC
  1337. try:
  1338. authh = self.getHeader(b"Authorization")
  1339. if not authh:
  1340. self.user = self.password = b""
  1341. return
  1342. bas, upw = authh.split()
  1343. if bas.lower() != b"basic":
  1344. raise ValueError()
  1345. upw = base64.b64decode(upw)
  1346. self.user, self.password = upw.split(b":", 1)
  1347. except (binascii.Error, ValueError):
  1348. self.user = self.password = b""
  1349. except BaseException:
  1350. self._log.failure("")
  1351. self.user = self.password = b""
  1352. def getUser(self):
  1353. """
  1354. Return the HTTP user sent with this request, if any.
  1355. If no user was supplied, return the empty string.
  1356. @returns: the HTTP user, if any
  1357. @rtype: C{bytes}
  1358. """
  1359. try:
  1360. return self.user
  1361. except BaseException:
  1362. pass
  1363. self._authorize()
  1364. return self.user
  1365. def getPassword(self):
  1366. """
  1367. Return the HTTP password sent with this request, if any.
  1368. If no password was supplied, return the empty string.
  1369. @returns: the HTTP password, if any
  1370. @rtype: C{bytes}
  1371. """
  1372. try:
  1373. return self.password
  1374. except BaseException:
  1375. pass
  1376. self._authorize()
  1377. return self.password
  1378. def connectionLost(self, reason):
  1379. """
  1380. There is no longer a connection for this request to respond over.
  1381. Clean up anything which can't be useful anymore.
  1382. """
  1383. self._disconnected = True
  1384. self.channel = None
  1385. if self.content is not None:
  1386. self.content.close()
  1387. for d in self.notifications:
  1388. d.errback(reason)
  1389. self.notifications = []
  1390. def loseConnection(self):
  1391. """
  1392. Pass the loseConnection through to the underlying channel.
  1393. """
  1394. if self.channel is not None:
  1395. self.channel.loseConnection()
  1396. def __eq__(self, other: object) -> bool:
  1397. """
  1398. Determines if two requests are the same object.
  1399. @param other: Another object whose identity will be compared
  1400. to this instance's.
  1401. @return: L{True} when the two are the same object and L{False}
  1402. when not.
  1403. @rtype: L{bool}
  1404. """
  1405. # When other is not an instance of request, return
  1406. # NotImplemented so that Python uses other.__eq__ to perform
  1407. # the comparison. This ensures that a Request proxy generated
  1408. # by proxyForInterface compares equal to an actual Request
  1409. # instanceby turning request != proxy into proxy != request.
  1410. if isinstance(other, Request):
  1411. return self is other
  1412. return NotImplemented
  1413. def __hash__(self):
  1414. """
  1415. A C{Request} is hashable so that it can be used as a mapping key.
  1416. @return: A C{int} based on the instance's identity.
  1417. """
  1418. return id(self)
  1419. class _DataLoss(Exception):
  1420. """
  1421. L{_DataLoss} indicates that not all of a message body was received. This
  1422. is only one of several possible exceptions which may indicate that data
  1423. was lost. Because of this, it should not be checked for by
  1424. specifically; any unexpected exception should be treated as having
  1425. caused data loss.
  1426. """
  1427. class PotentialDataLoss(Exception):
  1428. """
  1429. L{PotentialDataLoss} may be raised by a transfer encoding decoder's
  1430. C{noMoreData} method to indicate that it cannot be determined if the
  1431. entire response body has been delivered. This only occurs when making
  1432. requests to HTTP servers which do not set I{Content-Length} or a
  1433. I{Transfer-Encoding} in the response because in this case the end of the
  1434. response is indicated by the connection being closed, an event which may
  1435. also be due to a transient network problem or other error.
  1436. """
  1437. class _MalformedChunkedDataError(Exception):
  1438. """
  1439. C{_ChunkedTransferDecoder} raises L{_MalformedChunkedDataError} from its
  1440. C{dataReceived} method when it encounters malformed data. This exception
  1441. indicates a client-side error. If this exception is raised, the connection
  1442. should be dropped with a 400 error.
  1443. """
  1444. class _IdentityTransferDecoder:
  1445. """
  1446. Protocol for accumulating bytes up to a specified length. This handles the
  1447. case where no I{Transfer-Encoding} is specified.
  1448. @ivar contentLength: Counter keeping track of how many more bytes there are
  1449. to receive.
  1450. @ivar dataCallback: A one-argument callable which will be invoked each
  1451. time application data is received.
  1452. @ivar finishCallback: A one-argument callable which will be invoked when
  1453. the terminal chunk is received. It will be invoked with all bytes
  1454. which were delivered to this protocol which came after the terminal
  1455. chunk.
  1456. """
  1457. def __init__(self, contentLength, dataCallback, finishCallback):
  1458. self.contentLength = contentLength
  1459. self.dataCallback = dataCallback
  1460. self.finishCallback = finishCallback
  1461. def dataReceived(self, data):
  1462. """
  1463. Interpret the next chunk of bytes received. Either deliver them to the
  1464. data callback or invoke the finish callback if enough bytes have been
  1465. received.
  1466. @raise RuntimeError: If the finish callback has already been invoked
  1467. during a previous call to this methood.
  1468. """
  1469. if self.dataCallback is None:
  1470. raise RuntimeError(
  1471. "_IdentityTransferDecoder cannot decode data after finishing"
  1472. )
  1473. if self.contentLength is None:
  1474. self.dataCallback(data)
  1475. elif len(data) < self.contentLength:
  1476. self.contentLength -= len(data)
  1477. self.dataCallback(data)
  1478. else:
  1479. # Make the state consistent before invoking any code belonging to
  1480. # anyone else in case noMoreData ends up being called beneath this
  1481. # stack frame.
  1482. contentLength = self.contentLength
  1483. dataCallback = self.dataCallback
  1484. finishCallback = self.finishCallback
  1485. self.dataCallback = self.finishCallback = None
  1486. self.contentLength = 0
  1487. dataCallback(data[:contentLength])
  1488. finishCallback(data[contentLength:])
  1489. def noMoreData(self):
  1490. """
  1491. All data which will be delivered to this decoder has been. Check to
  1492. make sure as much data as was expected has been received.
  1493. @raise PotentialDataLoss: If the content length is unknown.
  1494. @raise _DataLoss: If the content length is known and fewer than that
  1495. many bytes have been delivered.
  1496. @return: L{None}
  1497. """
  1498. finishCallback = self.finishCallback
  1499. self.dataCallback = self.finishCallback = None
  1500. if self.contentLength is None:
  1501. finishCallback(b"")
  1502. raise PotentialDataLoss()
  1503. elif self.contentLength != 0:
  1504. raise _DataLoss()
  1505. maxChunkSizeLineLength = 1024
  1506. _chunkExtChars = (
  1507. b"\t !\"#$%&'()*+,-./0123456789:;<=>?@"
  1508. b"ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_`"
  1509. b"abcdefghijklmnopqrstuvwxyz{|}~"
  1510. b"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
  1511. b"\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
  1512. b"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
  1513. b"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
  1514. b"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
  1515. b"\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
  1516. b"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef"
  1517. b"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
  1518. )
  1519. """
  1520. Characters that are valid in a chunk extension.
  1521. See RFC 7230 section 4.1.1::
  1522. chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
  1523. chunk-ext-name = token
  1524. chunk-ext-val = token / quoted-string
  1525. And section 3.2.6::
  1526. token = 1*tchar
  1527. tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*"
  1528. / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
  1529. / DIGIT / ALPHA
  1530. ; any VCHAR, except delimiters
  1531. quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE
  1532. qdtext = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
  1533. obs-text = %x80-FF
  1534. We don't check if chunk extensions are well-formed beyond validating that they
  1535. don't contain characters outside this range.
  1536. """
  1537. class _ChunkedTransferDecoder:
  1538. """
  1539. Protocol for decoding I{chunked} Transfer-Encoding, as defined by RFC 7230,
  1540. section 4.1. This protocol can interpret the contents of a request or
  1541. response body which uses the I{chunked} Transfer-Encoding. It cannot
  1542. interpret any of the rest of the HTTP protocol.
  1543. It may make sense for _ChunkedTransferDecoder to be an actual IProtocol
  1544. implementation. Currently, the only user of this class will only ever
  1545. call dataReceived on it. However, it might be an improvement if the
  1546. user could connect this to a transport and deliver connection lost
  1547. notification. This way, `dataCallback` becomes `self.transport.write`
  1548. and perhaps `finishCallback` becomes `self.transport.loseConnection()`
  1549. (although I'm not sure where the extra data goes in that case). This
  1550. could also allow this object to indicate to the receiver of data that
  1551. the stream was not completely received, an error case which should be
  1552. noticed. -exarkun
  1553. @ivar dataCallback: A one-argument callable which will be invoked each
  1554. time application data is received. This callback is not reentrant.
  1555. @ivar finishCallback: A one-argument callable which will be invoked when
  1556. the terminal chunk is received. It will be invoked with all bytes
  1557. which were delivered to this protocol which came after the terminal
  1558. chunk.
  1559. @ivar length: Counter keeping track of how many more bytes in a chunk there
  1560. are to receive.
  1561. @ivar state: One of C{'CHUNK_LENGTH'}, C{'CRLF'}, C{'TRAILER'},
  1562. C{'BODY'}, or C{'FINISHED'}. For C{'CHUNK_LENGTH'}, data for the
  1563. chunk length line is currently being read. For C{'CRLF'}, the CR LF
  1564. pair which follows each chunk is being read. For C{'TRAILER'}, the CR
  1565. LF pair which follows the terminal 0-length chunk is currently being
  1566. read. For C{'BODY'}, the contents of a chunk are being read. For
  1567. C{'FINISHED'}, the last chunk has been completely read and no more
  1568. input is valid.
  1569. @ivar _buffer: Accumulated received data for the current state. At each
  1570. state transition this is truncated at the front so that index 0 is
  1571. where the next state shall begin.
  1572. @ivar _start: While in the C{'CHUNK_LENGTH'} state, tracks the index into
  1573. the buffer at which search for CRLF should resume. Resuming the search
  1574. at this position avoids doing quadratic work if the chunk length line
  1575. arrives over many calls to C{dataReceived}.
  1576. Not used in any other state.
  1577. """
  1578. state = "CHUNK_LENGTH"
  1579. def __init__(
  1580. self,
  1581. dataCallback: Callable[[bytes], None],
  1582. finishCallback: Callable[[bytes], None],
  1583. ) -> None:
  1584. self.dataCallback = dataCallback
  1585. self.finishCallback = finishCallback
  1586. self._buffer = bytearray()
  1587. self._start = 0
  1588. def _dataReceived_CHUNK_LENGTH(self) -> bool:
  1589. """
  1590. Read the chunk size line, ignoring any extensions.
  1591. @returns: C{True} once the line has been read and removed from
  1592. C{self._buffer}. C{False} when more data is required.
  1593. @raises _MalformedChunkedDataError: when the chunk size cannot be
  1594. decoded or the length of the line exceeds L{maxChunkSizeLineLength}.
  1595. """
  1596. eolIndex = self._buffer.find(b"\r\n", self._start)
  1597. if eolIndex >= maxChunkSizeLineLength or (
  1598. eolIndex == -1 and len(self._buffer) > maxChunkSizeLineLength
  1599. ):
  1600. raise _MalformedChunkedDataError(
  1601. "Chunk size line exceeds maximum of {} bytes.".format(
  1602. maxChunkSizeLineLength
  1603. )
  1604. )
  1605. if eolIndex == -1:
  1606. # Restart the search upon receipt of more data at the start of the
  1607. # new data, minus one in case the last character of the buffer is
  1608. # CR.
  1609. self._start = len(self._buffer) - 1
  1610. return False
  1611. endOfLengthIndex = self._buffer.find(b";", 0, eolIndex)
  1612. if endOfLengthIndex == -1:
  1613. endOfLengthIndex = eolIndex
  1614. rawLength = self._buffer[0:endOfLengthIndex]
  1615. try:
  1616. length = _hexint(rawLength)
  1617. except ValueError:
  1618. raise _MalformedChunkedDataError("Chunk-size must be an integer.")
  1619. ext = self._buffer[endOfLengthIndex + 1 : eolIndex]
  1620. if ext and ext.translate(None, _chunkExtChars) != b"":
  1621. raise _MalformedChunkedDataError(
  1622. f"Invalid characters in chunk extensions: {ext!r}."
  1623. )
  1624. if length == 0:
  1625. self.state = "TRAILER"
  1626. else:
  1627. self.state = "BODY"
  1628. self.length = length
  1629. del self._buffer[0 : eolIndex + 2]
  1630. self._start = 0
  1631. return True
  1632. def _dataReceived_CRLF(self) -> bool:
  1633. """
  1634. Await the carriage return and line feed characters that are the end of
  1635. chunk marker that follow the chunk data.
  1636. @returns: C{True} when the CRLF have been read, otherwise C{False}.
  1637. @raises _MalformedChunkedDataError: when anything other than CRLF are
  1638. received.
  1639. """
  1640. if len(self._buffer) < 2:
  1641. return False
  1642. if not self._buffer.startswith(b"\r\n"):
  1643. raise _MalformedChunkedDataError("Chunk did not end with CRLF")
  1644. self.state = "CHUNK_LENGTH"
  1645. del self._buffer[0:2]
  1646. return True
  1647. def _dataReceived_TRAILER(self) -> bool:
  1648. """
  1649. Await the carriage return and line feed characters that follow the
  1650. terminal zero-length chunk. Then invoke C{finishCallback} and switch to
  1651. state C{'FINISHED'}.
  1652. @returns: C{False}, as there is either insufficient data to continue,
  1653. or no data remains.
  1654. @raises _MalformedChunkedDataError: when anything other than CRLF is
  1655. received.
  1656. """
  1657. if len(self._buffer) < 2:
  1658. return False
  1659. if not self._buffer.startswith(b"\r\n"):
  1660. raise _MalformedChunkedDataError("Chunk did not end with CRLF")
  1661. data = memoryview(self._buffer)[2:].tobytes()
  1662. del self._buffer[:]
  1663. self.state = "FINISHED"
  1664. self.finishCallback(data)
  1665. return False
  1666. def _dataReceived_BODY(self) -> bool:
  1667. """
  1668. Deliver any available chunk data to the C{dataCallback}. When all the
  1669. remaining data for the chunk arrives, switch to state C{'CRLF'}.
  1670. @returns: C{True} to continue processing of any buffered data.
  1671. """
  1672. if len(self._buffer) >= self.length:
  1673. chunk = memoryview(self._buffer)[: self.length].tobytes()
  1674. del self._buffer[: self.length]
  1675. self.state = "CRLF"
  1676. self.dataCallback(chunk)
  1677. else:
  1678. chunk = bytes(self._buffer)
  1679. self.length -= len(chunk)
  1680. del self._buffer[:]
  1681. self.dataCallback(chunk)
  1682. return True
  1683. def _dataReceived_FINISHED(self) -> bool:
  1684. """
  1685. Once C{finishCallback} has been invoked receipt of additional data
  1686. raises L{RuntimeError} because it represents a programming error in
  1687. the caller.
  1688. """
  1689. raise RuntimeError(
  1690. "_ChunkedTransferDecoder.dataReceived called after last "
  1691. "chunk was processed"
  1692. )
  1693. def dataReceived(self, data: bytes) -> None:
  1694. """
  1695. Interpret data from a request or response body which uses the
  1696. I{chunked} Transfer-Encoding.
  1697. """
  1698. self._buffer += data
  1699. goOn = True
  1700. while goOn and self._buffer:
  1701. goOn = getattr(self, "_dataReceived_" + self.state)()
  1702. def noMoreData(self) -> None:
  1703. """
  1704. Verify that all data has been received. If it has not been, raise
  1705. L{_DataLoss}.
  1706. """
  1707. if self.state != "FINISHED":
  1708. raise _DataLoss(
  1709. "Chunked decoder in %r state, still expecting more data to "
  1710. "get to 'FINISHED' state." % (self.state,)
  1711. )
  1712. @implementer(interfaces.IPushProducer)
  1713. class _NoPushProducer:
  1714. """
  1715. A no-op version of L{interfaces.IPushProducer}, used to abstract over the
  1716. possibility that a L{HTTPChannel} transport does not provide
  1717. L{IPushProducer}.
  1718. """
  1719. def pauseProducing(self):
  1720. """
  1721. Pause producing data.
  1722. Tells a producer that it has produced too much data to process for
  1723. the time being, and to stop until resumeProducing() is called.
  1724. """
  1725. def resumeProducing(self):
  1726. """
  1727. Resume producing data.
  1728. This tells a producer to re-add itself to the main loop and produce
  1729. more data for its consumer.
  1730. """
  1731. def registerProducer(self, producer, streaming):
  1732. """
  1733. Register to receive data from a producer.
  1734. @param producer: The producer to register.
  1735. @param streaming: Whether this is a streaming producer or not.
  1736. """
  1737. def unregisterProducer(self):
  1738. """
  1739. Stop consuming data from a producer, without disconnecting.
  1740. """
  1741. def stopProducing(self):
  1742. """
  1743. IProducer.stopProducing
  1744. """
  1745. @implementer(interfaces.ITransport, interfaces.IPushProducer, interfaces.IConsumer)
  1746. class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
  1747. """
  1748. A receiver for HTTP requests.
  1749. The L{HTTPChannel} provides L{interfaces.ITransport} and
  1750. L{interfaces.IConsumer} to the L{Request} objects it creates. It also
  1751. implements L{interfaces.IPushProducer} to C{self.transport}, allowing the
  1752. transport to pause it.
  1753. @ivar MAX_LENGTH: Maximum length for initial request line and each line
  1754. from the header.
  1755. @ivar _transferDecoder: L{None} or a decoder instance if the request body
  1756. uses the I{chunked} Transfer-Encoding.
  1757. @type _transferDecoder: L{_ChunkedTransferDecoder}
  1758. @ivar maxHeaders: Maximum number of headers allowed per request.
  1759. @type maxHeaders: C{int}
  1760. @ivar totalHeadersSize: Maximum bytes for request line plus all headers
  1761. from the request.
  1762. @type totalHeadersSize: C{int}
  1763. @ivar _receivedHeaderSize: Bytes received so far for the header.
  1764. @type _receivedHeaderSize: C{int}
  1765. @ivar _handlingRequest: Whether a request is currently being processed.
  1766. @type _handlingRequest: L{bool}
  1767. @ivar _dataBuffer: Any data that has been received from the connection
  1768. while processing an outstanding request.
  1769. @type _dataBuffer: L{list} of L{bytes}
  1770. @ivar _networkProducer: Either the transport, if it provides
  1771. L{interfaces.IPushProducer}, or a null implementation of
  1772. L{interfaces.IPushProducer}. Used to attempt to prevent the transport
  1773. from producing excess data when we're responding to a request.
  1774. @type _networkProducer: L{interfaces.IPushProducer}
  1775. @ivar _requestProducer: If the L{Request} object or anything it calls
  1776. registers itself as an L{interfaces.IProducer}, it will be stored here.
  1777. This is used to create a producing pipeline: pause/resume producing
  1778. methods will be propagated from the C{transport}, through the
  1779. L{HTTPChannel} instance, to the c{_requestProducer}.
  1780. The reason we proxy through the producing methods rather than the old
  1781. behaviour (where we literally just set the L{Request} object as the
  1782. producer on the transport) is because we want to be able to exert
  1783. backpressure on the client to prevent it from sending in arbitrarily
  1784. many requests without ever reading responses. Essentially, if the
  1785. client never reads our responses we will eventually stop reading its
  1786. requests.
  1787. @type _requestProducer: L{interfaces.IPushProducer}
  1788. @ivar _requestProducerStreaming: A boolean that tracks whether the producer
  1789. on the L{Request} side of this channel has registered itself as a
  1790. L{interfaces.IPushProducer} or an L{interfaces.IPullProducer}.
  1791. @type _requestProducerStreaming: L{bool} or L{None}
  1792. @ivar _waitingForTransport: A boolean that tracks whether the transport has
  1793. asked us to stop producing. This is used to keep track of what we're
  1794. waiting for: if the transport has asked us to stop producing then we
  1795. don't want to unpause the transport until it asks us to produce again.
  1796. @type _waitingForTransport: L{bool}
  1797. @ivar abortTimeout: The number of seconds to wait after we attempt to shut
  1798. the transport down cleanly to give up and forcibly terminate it. This
  1799. is only used when we time a connection out, to prevent errors causing
  1800. the FD to get leaked. If this is L{None}, we will wait forever.
  1801. @type abortTimeout: L{int}
  1802. @ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be
  1803. used to forcibly close the transport if it doesn't close cleanly.
  1804. @type _abortingCall: L{twisted.internet.base.DelayedCall}
  1805. @ivar _optimisticEagerReadSize: When a resource takes a long time to answer
  1806. a request (via L{twisted.web.server.NOT_DONE_YET}, hopefully one day by
  1807. a L{Deferred}), we would like to be able to let that resource know
  1808. about the underlying transport disappearing as promptly as possible,
  1809. via L{Request.notifyFinish}, and therefore via
  1810. C{self.requests[...].connectionLost()} on this L{HTTPChannel}.
  1811. However, in order to simplify application logic, we implement
  1812. head-of-line blocking, and do not relay pipelined requests to the
  1813. application until the previous request has been answered. This means
  1814. that said application cannot dispose of any entity-body that comes in
  1815. from those subsequent requests, which may be arbitrarily large, and it
  1816. may need to be buffered in memory.
  1817. To implement this tradeoff between prompt notification when possible
  1818. (in the most frequent case of non-pipelined requests) and correct
  1819. behavior when not (say, if a client sends a very long-running GET
  1820. request followed by a PUT request with a very large body) we will
  1821. continue reading pipelined requests into C{self._dataBuffer} up to a
  1822. given limit.
  1823. C{_optimisticEagerReadSize} is the number of bytes we will accept from
  1824. the client and buffer before pausing the transport.
  1825. This behavior has been in place since Twisted 17.9.0 .
  1826. @type _optimisticEagerReadSize: L{int}
  1827. """
  1828. maxHeaders = 500
  1829. totalHeadersSize = 16384
  1830. abortTimeout = 15
  1831. length = 0
  1832. persistent = 1
  1833. __header = b""
  1834. __first_line = 1
  1835. __content = None
  1836. # set in instances or subclasses
  1837. requestFactory = Request
  1838. _savedTimeOut = None
  1839. _receivedHeaderCount = 0
  1840. _receivedHeaderSize = 0
  1841. _requestProducer = None
  1842. _requestProducerStreaming = None
  1843. _waitingForTransport = False
  1844. _abortingCall = None
  1845. _optimisticEagerReadSize = 0x4000
  1846. _log = Logger()
  1847. def __init__(self):
  1848. # the request queue
  1849. self.requests = []
  1850. self._handlingRequest = False
  1851. self._dataBuffer = []
  1852. self._transferDecoder = None
  1853. def connectionMade(self):
  1854. self.setTimeout(self.timeOut)
  1855. self._networkProducer = interfaces.IPushProducer(
  1856. self.transport, _NoPushProducer()
  1857. )
  1858. self._networkProducer.registerProducer(self, True)
  1859. def lineReceived(self, line):
  1860. """
  1861. Called for each line from request until the end of headers when
  1862. it enters binary mode.
  1863. """
  1864. self.resetTimeout()
  1865. self._receivedHeaderSize += len(line)
  1866. if self._receivedHeaderSize > self.totalHeadersSize:
  1867. self._respondToBadRequestAndDisconnect()
  1868. return
  1869. if self.__first_line:
  1870. # if this connection is not persistent, drop any data which
  1871. # the client (illegally) sent after the last request.
  1872. if not self.persistent:
  1873. self.dataReceived = self.lineReceived = lambda *args: None
  1874. return
  1875. # IE sends an extraneous empty line (\r\n) after a POST request;
  1876. # eat up such a line, but only ONCE
  1877. if not line and self.__first_line == 1:
  1878. self.__first_line = 2
  1879. return
  1880. # create a new Request object
  1881. if INonQueuedRequestFactory.providedBy(self.requestFactory):
  1882. request = self.requestFactory(self)
  1883. else:
  1884. request = self.requestFactory(self, len(self.requests))
  1885. self.requests.append(request)
  1886. self.__first_line = 0
  1887. parts = line.split()
  1888. if len(parts) != 3:
  1889. self._respondToBadRequestAndDisconnect()
  1890. return
  1891. command, request, version = parts
  1892. try:
  1893. command.decode("ascii")
  1894. except UnicodeDecodeError:
  1895. self._respondToBadRequestAndDisconnect()
  1896. return
  1897. self._command = command
  1898. self._path = request
  1899. self._version = version
  1900. elif line == b"":
  1901. # End of headers.
  1902. if self.__header:
  1903. ok = self.headerReceived(self.__header)
  1904. # If the last header we got is invalid, we MUST NOT proceed
  1905. # with processing. We'll have sent a 400 anyway, so just stop.
  1906. if not ok:
  1907. return
  1908. self.__header = b""
  1909. self.allHeadersReceived()
  1910. if self.length == 0:
  1911. self.allContentReceived()
  1912. else:
  1913. self.setRawMode()
  1914. elif line[0] in b" \t":
  1915. # Continuation of a multi line header.
  1916. self.__header += b" " + line.lstrip(b" \t")
  1917. # Regular header line.
  1918. # Processing of header line is delayed to allow accumulating multi
  1919. # line headers.
  1920. else:
  1921. if self.__header:
  1922. self.headerReceived(self.__header)
  1923. self.__header = line
  1924. def _finishRequestBody(self, data):
  1925. self.allContentReceived()
  1926. self._dataBuffer.append(data)
  1927. def _maybeChooseTransferDecoder(self, header, data):
  1928. """
  1929. If the provided header is C{content-length} or
  1930. C{transfer-encoding}, choose the appropriate decoder if any.
  1931. Returns L{True} if the request can proceed and L{False} if not.
  1932. """
  1933. def fail():
  1934. self._respondToBadRequestAndDisconnect()
  1935. self.length = None
  1936. return False
  1937. # Can this header determine the length?
  1938. if header == b"content-length":
  1939. if not data.isdigit():
  1940. return fail()
  1941. try:
  1942. length = int(data)
  1943. except ValueError:
  1944. return fail()
  1945. newTransferDecoder = _IdentityTransferDecoder(
  1946. length, self.requests[-1].handleContentChunk, self._finishRequestBody
  1947. )
  1948. elif header == b"transfer-encoding":
  1949. # XXX Rather poorly tested code block, apparently only exercised by
  1950. # test_chunkedEncoding
  1951. if data.lower() == b"chunked":
  1952. length = None
  1953. newTransferDecoder = _ChunkedTransferDecoder(
  1954. self.requests[-1].handleContentChunk, self._finishRequestBody
  1955. )
  1956. elif data.lower() == b"identity":
  1957. return True
  1958. else:
  1959. return fail()
  1960. else:
  1961. # It's not a length related header, so exit
  1962. return True
  1963. if self._transferDecoder is not None:
  1964. return fail()
  1965. else:
  1966. self.length = length
  1967. self._transferDecoder = newTransferDecoder
  1968. return True
  1969. def headerReceived(self, line):
  1970. """
  1971. Do pre-processing (for content-length) and store this header away.
  1972. Enforce the per-request header limit.
  1973. @type line: C{bytes}
  1974. @param line: A line from the header section of a request, excluding the
  1975. line delimiter.
  1976. @return: A flag indicating whether the header was valid.
  1977. @rtype: L{bool}
  1978. """
  1979. try:
  1980. header, data = line.split(b":", 1)
  1981. except ValueError:
  1982. self._respondToBadRequestAndDisconnect()
  1983. return False
  1984. if not header or header[-1:].isspace():
  1985. self._respondToBadRequestAndDisconnect()
  1986. return False
  1987. header = header.lower()
  1988. data = data.strip(b" \t")
  1989. if not self._maybeChooseTransferDecoder(header, data):
  1990. return False
  1991. reqHeaders = self.requests[-1].requestHeaders
  1992. values = reqHeaders.getRawHeaders(header)
  1993. if values is not None:
  1994. values.append(data)
  1995. else:
  1996. reqHeaders.setRawHeaders(header, [data])
  1997. self._receivedHeaderCount += 1
  1998. if self._receivedHeaderCount > self.maxHeaders:
  1999. self._respondToBadRequestAndDisconnect()
  2000. return False
  2001. return True
  2002. def allContentReceived(self):
  2003. command = self._command
  2004. path = self._path
  2005. version = self._version
  2006. # reset ALL state variables, so we don't interfere with next request
  2007. self.length = 0
  2008. self._receivedHeaderCount = 0
  2009. self._receivedHeaderSize = 0
  2010. self.__first_line = 1
  2011. self._transferDecoder = None
  2012. del self._command, self._path, self._version
  2013. # Disable the idle timeout, in case this request takes a long
  2014. # time to finish generating output.
  2015. if self.timeOut:
  2016. self._savedTimeOut = self.setTimeout(None)
  2017. self._handlingRequest = True
  2018. req = self.requests[-1]
  2019. req.requestReceived(command, path, version)
  2020. def dataReceived(self, data):
  2021. """
  2022. Data was received from the network. Process it.
  2023. """
  2024. # If we're currently handling a request, buffer this data.
  2025. if self._handlingRequest:
  2026. self._dataBuffer.append(data)
  2027. if (
  2028. sum(map(len, self._dataBuffer)) > self._optimisticEagerReadSize
  2029. ) and not self._waitingForTransport:
  2030. # If we received more data than a small limit while processing
  2031. # the head-of-line request, apply TCP backpressure to our peer
  2032. # to get them to stop sending more request data until we're
  2033. # ready. See docstring for _optimisticEagerReadSize above.
  2034. self._networkProducer.pauseProducing()
  2035. return
  2036. return basic.LineReceiver.dataReceived(self, data)
  2037. def rawDataReceived(self, data):
  2038. self.resetTimeout()
  2039. try:
  2040. self._transferDecoder.dataReceived(data)
  2041. except _MalformedChunkedDataError:
  2042. self._respondToBadRequestAndDisconnect()
  2043. def allHeadersReceived(self):
  2044. req = self.requests[-1]
  2045. req.parseCookies()
  2046. self.persistent = self.checkPersistence(req, self._version)
  2047. req.gotLength(self.length)
  2048. # Handle 'Expect: 100-continue' with automated 100 response code,
  2049. # a simplistic implementation of RFC 2686 8.2.3:
  2050. expectContinue = req.requestHeaders.getRawHeaders(b"expect")
  2051. if (
  2052. expectContinue
  2053. and expectContinue[0].lower() == b"100-continue"
  2054. and self._version == b"HTTP/1.1"
  2055. ):
  2056. self._send100Continue()
  2057. def checkPersistence(self, request, version):
  2058. """
  2059. Check if the channel should close or not.
  2060. @param request: The request most recently received over this channel
  2061. against which checks will be made to determine if this connection
  2062. can remain open after a matching response is returned.
  2063. @type version: C{bytes}
  2064. @param version: The version of the request.
  2065. @rtype: C{bool}
  2066. @return: A flag which, if C{True}, indicates that this connection may
  2067. remain open to receive another request; if C{False}, the connection
  2068. must be closed in order to indicate the completion of the response
  2069. to C{request}.
  2070. """
  2071. connection = request.requestHeaders.getRawHeaders(b"connection")
  2072. if connection:
  2073. tokens = [t.lower() for t in connection[0].split(b" ")]
  2074. else:
  2075. tokens = []
  2076. # Once any HTTP 0.9 or HTTP 1.0 request is received, the connection is
  2077. # no longer allowed to be persistent. At this point in processing the
  2078. # request, we don't yet know if it will be possible to set a
  2079. # Content-Length in the response. If it is not, then the connection
  2080. # will have to be closed to end an HTTP 0.9 or HTTP 1.0 response.
  2081. # If the checkPersistence call happened later, after the Content-Length
  2082. # has been determined (or determined not to be set), it would probably
  2083. # be possible to have persistent connections with HTTP 0.9 and HTTP 1.0.
  2084. # This may not be worth the effort, though. Just use HTTP 1.1, okay?
  2085. if version == b"HTTP/1.1":
  2086. if b"close" in tokens:
  2087. request.responseHeaders.setRawHeaders(b"connection", [b"close"])
  2088. return False
  2089. else:
  2090. return True
  2091. else:
  2092. return False
  2093. def requestDone(self, request):
  2094. """
  2095. Called by first request in queue when it is done.
  2096. """
  2097. if request != self.requests[0]:
  2098. raise TypeError
  2099. del self.requests[0]
  2100. # We should only resume the producer if we're not waiting for the
  2101. # transport.
  2102. if not self._waitingForTransport:
  2103. self._networkProducer.resumeProducing()
  2104. if self.persistent:
  2105. self._handlingRequest = False
  2106. if self._savedTimeOut:
  2107. self.setTimeout(self._savedTimeOut)
  2108. # Receive our buffered data, if any.
  2109. data = b"".join(self._dataBuffer)
  2110. self._dataBuffer = []
  2111. self.setLineMode(data)
  2112. else:
  2113. self.loseConnection()
  2114. def timeoutConnection(self):
  2115. self._log.info("Timing out client: {peer}", peer=str(self.transport.getPeer()))
  2116. if self.abortTimeout is not None:
  2117. # We use self.callLater because that's what TimeoutMixin does.
  2118. self._abortingCall = self.callLater(
  2119. self.abortTimeout, self.forceAbortClient
  2120. )
  2121. self.loseConnection()
  2122. def forceAbortClient(self):
  2123. """
  2124. Called if C{abortTimeout} seconds have passed since the timeout fired,
  2125. and the connection still hasn't gone away. This can really only happen
  2126. on extremely bad connections or when clients are maliciously attempting
  2127. to keep connections open.
  2128. """
  2129. self._log.info(
  2130. "Forcibly timing out client: {peer}", peer=str(self.transport.getPeer())
  2131. )
  2132. # We want to lose track of the _abortingCall so that no-one tries to
  2133. # cancel it.
  2134. self._abortingCall = None
  2135. self.transport.abortConnection()
  2136. def connectionLost(self, reason):
  2137. self.setTimeout(None)
  2138. for request in self.requests:
  2139. request.connectionLost(reason)
  2140. # If we were going to force-close the transport, we don't have to now.
  2141. if self._abortingCall is not None:
  2142. self._abortingCall.cancel()
  2143. self._abortingCall = None
  2144. def isSecure(self):
  2145. """
  2146. Return L{True} if this channel is using a secure transport.
  2147. Normally this method returns L{True} if this instance is using a
  2148. transport that implements L{interfaces.ISSLTransport}.
  2149. @returns: L{True} if this request is secure
  2150. @rtype: C{bool}
  2151. """
  2152. if interfaces.ISSLTransport(self.transport, None) is not None:
  2153. return True
  2154. return False
  2155. def writeHeaders(self, version, code, reason, headers):
  2156. """
  2157. Called by L{Request} objects to write a complete set of HTTP headers to
  2158. a transport.
  2159. @param version: The HTTP version in use.
  2160. @type version: L{bytes}
  2161. @param code: The HTTP status code to write.
  2162. @type code: L{bytes}
  2163. @param reason: The HTTP reason phrase to write.
  2164. @type reason: L{bytes}
  2165. @param headers: The headers to write to the transport.
  2166. @type headers: L{twisted.web.http_headers.Headers}
  2167. """
  2168. sanitizedHeaders = Headers()
  2169. for name, value in headers:
  2170. sanitizedHeaders.addRawHeader(name, value)
  2171. responseLine = version + b" " + code + b" " + reason + b"\r\n"
  2172. headerSequence = [responseLine]
  2173. headerSequence.extend(
  2174. name + b": " + value + b"\r\n"
  2175. for name, values in sanitizedHeaders.getAllRawHeaders()
  2176. for value in values
  2177. )
  2178. headerSequence.append(b"\r\n")
  2179. self.transport.writeSequence(headerSequence)
  2180. def write(self, data):
  2181. """
  2182. Called by L{Request} objects to write response data.
  2183. @param data: The data chunk to write to the stream.
  2184. @type data: L{bytes}
  2185. @return: L{None}
  2186. """
  2187. self.transport.write(data)
  2188. def writeSequence(self, iovec):
  2189. """
  2190. Write a list of strings to the HTTP response.
  2191. @param iovec: A list of byte strings to write to the stream.
  2192. @type iovec: L{list} of L{bytes}
  2193. @return: L{None}
  2194. """
  2195. self.transport.writeSequence(iovec)
  2196. def getPeer(self):
  2197. """
  2198. Get the remote address of this connection.
  2199. @return: An L{IAddress} provider.
  2200. """
  2201. return self.transport.getPeer()
  2202. def getHost(self):
  2203. """
  2204. Get the local address of this connection.
  2205. @return: An L{IAddress} provider.
  2206. """
  2207. return self.transport.getHost()
  2208. def loseConnection(self):
  2209. """
  2210. Closes the connection. Will write any data that is pending to be sent
  2211. on the network, but if this response has not yet been written to the
  2212. network will not write anything.
  2213. @return: L{None}
  2214. """
  2215. self._networkProducer.unregisterProducer()
  2216. return self.transport.loseConnection()
  2217. def registerProducer(self, producer, streaming):
  2218. """
  2219. Register to receive data from a producer.
  2220. This sets self to be a consumer for a producer. When this object runs
  2221. out of data (as when a send(2) call on a socket succeeds in moving the
  2222. last data from a userspace buffer into a kernelspace buffer), it will
  2223. ask the producer to resumeProducing().
  2224. For L{IPullProducer} providers, C{resumeProducing} will be called once
  2225. each time data is required.
  2226. For L{IPushProducer} providers, C{pauseProducing} will be called
  2227. whenever the write buffer fills up and C{resumeProducing} will only be
  2228. called when it empties.
  2229. @type producer: L{IProducer} provider
  2230. @param producer: The L{IProducer} that will be producing data.
  2231. @type streaming: L{bool}
  2232. @param streaming: C{True} if C{producer} provides L{IPushProducer},
  2233. C{False} if C{producer} provides L{IPullProducer}.
  2234. @raise RuntimeError: If a producer is already registered.
  2235. @return: L{None}
  2236. """
  2237. if self._requestProducer is not None:
  2238. raise RuntimeError(
  2239. "Cannot register producer %s, because producer %s was never "
  2240. "unregistered." % (producer, self._requestProducer)
  2241. )
  2242. if not streaming:
  2243. producer = _PullToPush(producer, self)
  2244. self._requestProducer = producer
  2245. self._requestProducerStreaming = streaming
  2246. if not streaming:
  2247. producer.startStreaming()
  2248. def unregisterProducer(self):
  2249. """
  2250. Stop consuming data from a producer, without disconnecting.
  2251. @return: L{None}
  2252. """
  2253. if self._requestProducer is None:
  2254. return
  2255. if not self._requestProducerStreaming:
  2256. self._requestProducer.stopStreaming()
  2257. self._requestProducer = None
  2258. self._requestProducerStreaming = None
  2259. def stopProducing(self):
  2260. """
  2261. Stop producing data.
  2262. The HTTPChannel doesn't *actually* implement this, beacuse the
  2263. assumption is that it will only be called just before C{loseConnection}
  2264. is called. There's nothing sensible we can do other than call
  2265. C{loseConnection} anyway.
  2266. """
  2267. if self._requestProducer is not None:
  2268. self._requestProducer.stopProducing()
  2269. def pauseProducing(self):
  2270. """
  2271. Pause producing data.
  2272. This will be called by the transport when the send buffers have been
  2273. filled up. We want to simultaneously pause the producing L{Request}
  2274. object and also pause our transport.
  2275. The logic behind pausing the transport is specifically to avoid issues
  2276. like https://twistedmatrix.com/trac/ticket/8868. In this case, our
  2277. inability to send does not prevent us handling more requests, which
  2278. means we increasingly queue up more responses in our send buffer
  2279. without end. The easiest way to handle this is to ensure that if we are
  2280. unable to send our responses, we will not read further data from the
  2281. connection until the client pulls some data out. This is a bit of a
  2282. blunt instrument, but it's ok.
  2283. Note that this potentially interacts with timeout handling in a
  2284. positive way. Once the transport is paused the client may run into a
  2285. timeout which will cause us to tear the connection down. That's a good
  2286. thing!
  2287. """
  2288. self._waitingForTransport = True
  2289. # The first step is to tell any producer we might currently have
  2290. # registered to stop producing. If we can slow our applications down
  2291. # we should.
  2292. if self._requestProducer is not None:
  2293. self._requestProducer.pauseProducing()
  2294. # The next step here is to pause our own transport, as discussed in the
  2295. # docstring.
  2296. if not self._handlingRequest:
  2297. self._networkProducer.pauseProducing()
  2298. def resumeProducing(self):
  2299. """
  2300. Resume producing data.
  2301. This will be called by the transport when the send buffer has dropped
  2302. enough to actually send more data. When this happens we can unpause any
  2303. outstanding L{Request} producers we have, and also unpause our
  2304. transport.
  2305. """
  2306. self._waitingForTransport = False
  2307. if self._requestProducer is not None:
  2308. self._requestProducer.resumeProducing()
  2309. # We only want to resume the network producer if we're not currently
  2310. # waiting for a response to show up.
  2311. if not self._handlingRequest:
  2312. self._networkProducer.resumeProducing()
  2313. def _send100Continue(self):
  2314. """
  2315. Sends a 100 Continue response, used to signal to clients that further
  2316. processing will be performed.
  2317. """
  2318. self.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n")
  2319. def _respondToBadRequestAndDisconnect(self):
  2320. """
  2321. This is a quick and dirty way of responding to bad requests.
  2322. As described by HTTP standard we should be patient and accept the
  2323. whole request from the client before sending a polite bad request
  2324. response, even in the case when clients send tons of data.
  2325. """
  2326. self.transport.write(b"HTTP/1.1 400 Bad Request\r\n\r\n")
  2327. self.loseConnection()
  2328. def _escape(s):
  2329. """
  2330. Return a string like python repr, but always escaped as if surrounding
  2331. quotes were double quotes.
  2332. @param s: The string to escape.
  2333. @type s: L{bytes} or L{str}
  2334. @return: An escaped string.
  2335. @rtype: L{str}
  2336. """
  2337. if not isinstance(s, bytes):
  2338. s = s.encode("ascii")
  2339. r = repr(s)
  2340. if not isinstance(r, str):
  2341. r = r.decode("ascii")
  2342. if r.startswith("b"):
  2343. r = r[1:]
  2344. if r.startswith("'"):
  2345. return r[1:-1].replace('"', '\\"').replace("\\'", "'")
  2346. return r[1:-1]
  2347. @provider(IAccessLogFormatter)
  2348. def combinedLogFormatter(timestamp, request):
  2349. """
  2350. @return: A combined log formatted log line for the given request.
  2351. @see: L{IAccessLogFormatter}
  2352. """
  2353. clientAddr = request.getClientAddress()
  2354. if isinstance(
  2355. clientAddr, (address.IPv4Address, address.IPv6Address, _XForwardedForAddress)
  2356. ):
  2357. ip = clientAddr.host
  2358. else:
  2359. ip = b"-"
  2360. referrer = _escape(request.getHeader(b"referer") or b"-")
  2361. agent = _escape(request.getHeader(b"user-agent") or b"-")
  2362. line = (
  2363. '"%(ip)s" - - %(timestamp)s "%(method)s %(uri)s %(protocol)s" '
  2364. '%(code)d %(length)s "%(referrer)s" "%(agent)s"'
  2365. % dict(
  2366. ip=_escape(ip),
  2367. timestamp=timestamp,
  2368. method=_escape(request.method),
  2369. uri=_escape(request.uri),
  2370. protocol=_escape(request.clientproto),
  2371. code=request.code,
  2372. length=request.sentLength or "-",
  2373. referrer=referrer,
  2374. agent=agent,
  2375. )
  2376. )
  2377. return line
  2378. @implementer(interfaces.IAddress)
  2379. class _XForwardedForAddress:
  2380. """
  2381. L{IAddress} which represents the client IP to log for a request, as gleaned
  2382. from an X-Forwarded-For header.
  2383. @ivar host: An IP address or C{b"-"}.
  2384. @type host: L{bytes}
  2385. @see: L{proxiedLogFormatter}
  2386. """
  2387. def __init__(self, host):
  2388. self.host = host
  2389. class _XForwardedForRequest(proxyForInterface(IRequest, "_request")): # type: ignore[misc]
  2390. """
  2391. Add a layer on top of another request that only uses the value of an
  2392. X-Forwarded-For header as the result of C{getClientAddress}.
  2393. """
  2394. def getClientAddress(self):
  2395. """
  2396. The client address (the first address) in the value of the
  2397. I{X-Forwarded-For header}. If the header is not present, the IP is
  2398. considered to be C{b"-"}.
  2399. @return: L{_XForwardedForAddress} which wraps the client address as
  2400. expected by L{combinedLogFormatter}.
  2401. """
  2402. host = (
  2403. self._request.requestHeaders.getRawHeaders(b"x-forwarded-for", [b"-"])[0]
  2404. .split(b",")[0]
  2405. .strip()
  2406. )
  2407. return _XForwardedForAddress(host)
  2408. # These are missing from the interface. Forward them manually.
  2409. @property
  2410. def clientproto(self):
  2411. """
  2412. @return: The protocol version in the request.
  2413. @rtype: L{bytes}
  2414. """
  2415. return self._request.clientproto
  2416. @property
  2417. def code(self):
  2418. """
  2419. @return: The response code for the request.
  2420. @rtype: L{int}
  2421. """
  2422. return self._request.code
  2423. @property
  2424. def sentLength(self):
  2425. """
  2426. @return: The number of bytes sent in the response body.
  2427. @rtype: L{int}
  2428. """
  2429. return self._request.sentLength
  2430. @provider(IAccessLogFormatter)
  2431. def proxiedLogFormatter(timestamp, request):
  2432. """
  2433. @return: A combined log formatted log line for the given request but use
  2434. the value of the I{X-Forwarded-For} header as the value for the client
  2435. IP address.
  2436. @see: L{IAccessLogFormatter}
  2437. """
  2438. return combinedLogFormatter(timestamp, _XForwardedForRequest(request))
  2439. class _GenericHTTPChannelProtocol(proxyForInterface(IProtocol, "_channel")): # type: ignore[misc]
  2440. """
  2441. A proxy object that wraps one of the HTTP protocol objects, and switches
  2442. between them depending on TLS negotiated protocol.
  2443. @ivar _negotiatedProtocol: The protocol negotiated with ALPN or NPN, if
  2444. any.
  2445. @type _negotiatedProtocol: Either a bytestring containing the ALPN token
  2446. for the negotiated protocol, or L{None} if no protocol has yet been
  2447. negotiated.
  2448. @ivar _channel: The object capable of behaving like a L{HTTPChannel} that
  2449. is backing this object. By default this is a L{HTTPChannel}, but if a
  2450. HTTP protocol upgrade takes place this may be a different channel
  2451. object. Must implement L{IProtocol}.
  2452. @type _channel: L{HTTPChannel}
  2453. @ivar _requestFactory: A callable to use to build L{IRequest} objects.
  2454. @type _requestFactory: L{IRequest}
  2455. @ivar _site: A reference to the creating L{twisted.web.server.Site} object.
  2456. @type _site: L{twisted.web.server.Site}
  2457. @ivar _factory: A reference to the creating L{HTTPFactory} object.
  2458. @type _factory: L{HTTPFactory}
  2459. @ivar _timeOut: A timeout value to pass to the backing channel.
  2460. @type _timeOut: L{int} or L{None}
  2461. @ivar _callLater: A value for the C{callLater} callback.
  2462. @type _callLater: L{callable}
  2463. """
  2464. _negotiatedProtocol = None
  2465. _requestFactory = Request
  2466. _factory = None
  2467. _site = None
  2468. _timeOut = None
  2469. _callLater = None
  2470. @property
  2471. def factory(self):
  2472. """
  2473. @see: L{_genericHTTPChannelProtocolFactory}
  2474. """
  2475. return self._channel.factory
  2476. @factory.setter
  2477. def factory(self, value):
  2478. self._factory = value
  2479. self._channel.factory = value
  2480. @property
  2481. def requestFactory(self):
  2482. """
  2483. A callable to use to build L{IRequest} objects.
  2484. Retries the object from the current backing channel.
  2485. """
  2486. return self._channel.requestFactory
  2487. @requestFactory.setter
  2488. def requestFactory(self, value):
  2489. """
  2490. A callable to use to build L{IRequest} objects.
  2491. Sets the object on the backing channel and also stores the value for
  2492. propagation to any new channel.
  2493. @param value: The new callable to use.
  2494. @type value: A L{callable} returning L{IRequest}
  2495. """
  2496. self._requestFactory = value
  2497. self._channel.requestFactory = value
  2498. @property
  2499. def site(self):
  2500. """
  2501. A reference to the creating L{twisted.web.server.Site} object.
  2502. Returns the site object from the backing channel.
  2503. """
  2504. return self._channel.site
  2505. @site.setter
  2506. def site(self, value):
  2507. """
  2508. A reference to the creating L{twisted.web.server.Site} object.
  2509. Sets the object on the backing channel and also stores the value for
  2510. propagation to any new channel.
  2511. @param value: The L{twisted.web.server.Site} object to set.
  2512. @type value: L{twisted.web.server.Site}
  2513. """
  2514. self._site = value
  2515. self._channel.site = value
  2516. @property
  2517. def timeOut(self):
  2518. """
  2519. The idle timeout for the backing channel.
  2520. """
  2521. return self._channel.timeOut
  2522. @timeOut.setter
  2523. def timeOut(self, value):
  2524. """
  2525. The idle timeout for the backing channel.
  2526. Sets the idle timeout on both the backing channel and stores it for
  2527. propagation to any new backing channel.
  2528. @param value: The timeout to set.
  2529. @type value: L{int} or L{float}
  2530. """
  2531. self._timeOut = value
  2532. self._channel.timeOut = value
  2533. @property
  2534. def callLater(self):
  2535. """
  2536. A value for the C{callLater} callback. This callback is used by the
  2537. L{twisted.protocols.policies.TimeoutMixin} to handle timeouts.
  2538. """
  2539. return self._channel.callLater
  2540. @callLater.setter
  2541. def callLater(self, value):
  2542. """
  2543. Sets the value for the C{callLater} callback. This callback is used by
  2544. the L{twisted.protocols.policies.TimeoutMixin} to handle timeouts.
  2545. @param value: The new callback to use.
  2546. @type value: L{callable}
  2547. """
  2548. self._callLater = value
  2549. self._channel.callLater = value
  2550. def dataReceived(self, data):
  2551. """
  2552. An override of L{IProtocol.dataReceived} that checks what protocol we're
  2553. using.
  2554. """
  2555. if self._negotiatedProtocol is None:
  2556. try:
  2557. negotiatedProtocol = self._channel.transport.negotiatedProtocol
  2558. except AttributeError:
  2559. # Plaintext HTTP, always HTTP/1.1
  2560. negotiatedProtocol = b"http/1.1"
  2561. if negotiatedProtocol is None:
  2562. negotiatedProtocol = b"http/1.1"
  2563. if negotiatedProtocol == b"h2":
  2564. if not H2_ENABLED:
  2565. raise ValueError("Negotiated HTTP/2 without support.")
  2566. # We need to make sure that the HTTPChannel is unregistered
  2567. # from the transport so that the H2Connection can register
  2568. # itself if possible.
  2569. networkProducer = self._channel._networkProducer
  2570. networkProducer.unregisterProducer()
  2571. # Cancel the old channel's timeout.
  2572. self._channel.setTimeout(None)
  2573. transport = self._channel.transport
  2574. self._channel = H2Connection()
  2575. self._channel.requestFactory = self._requestFactory
  2576. self._channel.site = self._site
  2577. self._channel.factory = self._factory
  2578. self._channel.timeOut = self._timeOut
  2579. self._channel.callLater = self._callLater
  2580. self._channel.makeConnection(transport)
  2581. # Register the H2Connection as the transport's
  2582. # producer, so that the transport can apply back
  2583. # pressure.
  2584. networkProducer.registerProducer(self._channel, True)
  2585. else:
  2586. # Only HTTP/2 and HTTP/1.1 are supported right now.
  2587. assert (
  2588. negotiatedProtocol == b"http/1.1"
  2589. ), "Unsupported protocol negotiated"
  2590. self._negotiatedProtocol = negotiatedProtocol
  2591. return self._channel.dataReceived(data)
  2592. def _genericHTTPChannelProtocolFactory(self):
  2593. """
  2594. Returns an appropriately initialized _GenericHTTPChannelProtocol.
  2595. """
  2596. return _GenericHTTPChannelProtocol(HTTPChannel())
  2597. class HTTPFactory(protocol.ServerFactory):
  2598. """
  2599. Factory for HTTP server.
  2600. @ivar _logDateTime: A cached datetime string for log messages, updated by
  2601. C{_logDateTimeCall}.
  2602. @type _logDateTime: C{str}
  2603. @ivar _logDateTimeCall: A delayed call for the next update to the cached
  2604. log datetime string.
  2605. @type _logDateTimeCall: L{IDelayedCall} provided
  2606. @ivar _logFormatter: See the C{logFormatter} parameter to L{__init__}
  2607. @ivar _nativeize: A flag that indicates whether the log file being written
  2608. to wants native strings (C{True}) or bytes (C{False}). This is only to
  2609. support writing to L{twisted.python.log} which, unfortunately, works
  2610. with native strings.
  2611. @ivar reactor: An L{IReactorTime} provider used to manage connection
  2612. timeouts and compute logging timestamps.
  2613. """
  2614. # We need to ignore the mypy error here, because
  2615. # _genericHTTPChannelProtocolFactory is a callable which returns a proxy
  2616. # to a Protocol, instead of a concrete Protocol object, as expected in
  2617. # the protocol.Factory interface
  2618. protocol = _genericHTTPChannelProtocolFactory # type: ignore[assignment]
  2619. logPath = None
  2620. timeOut = _REQUEST_TIMEOUT
  2621. def __init__(
  2622. self, logPath=None, timeout=_REQUEST_TIMEOUT, logFormatter=None, reactor=None
  2623. ):
  2624. """
  2625. @param logPath: File path to which access log messages will be written
  2626. or C{None} to disable logging.
  2627. @type logPath: L{str} or L{bytes}
  2628. @param timeout: The initial value of L{timeOut}, which defines the idle
  2629. connection timeout in seconds, or C{None} to disable the idle
  2630. timeout.
  2631. @type timeout: L{float}
  2632. @param logFormatter: An object to format requests into log lines for
  2633. the access log. L{combinedLogFormatter} when C{None} is passed.
  2634. @type logFormatter: L{IAccessLogFormatter} provider
  2635. @param reactor: An L{IReactorTime} provider used to manage connection
  2636. timeouts and compute logging timestamps. Defaults to the global
  2637. reactor.
  2638. """
  2639. if not reactor:
  2640. from twisted.internet import reactor
  2641. self.reactor = reactor
  2642. if logPath is not None:
  2643. logPath = os.path.abspath(logPath)
  2644. self.logPath = logPath
  2645. self.timeOut = timeout
  2646. if logFormatter is None:
  2647. logFormatter = combinedLogFormatter
  2648. self._logFormatter = logFormatter
  2649. # For storing the cached log datetime and the callback to update it
  2650. self._logDateTime = None
  2651. self._logDateTimeCall = None
  2652. def _updateLogDateTime(self):
  2653. """
  2654. Update log datetime periodically, so we aren't always recalculating it.
  2655. """
  2656. self._logDateTime = datetimeToLogString(self.reactor.seconds())
  2657. self._logDateTimeCall = self.reactor.callLater(1, self._updateLogDateTime)
  2658. def buildProtocol(self, addr):
  2659. p = protocol.ServerFactory.buildProtocol(self, addr)
  2660. # This is a bit of a hack to ensure that the HTTPChannel timeouts
  2661. # occur on the same reactor as the one we're using here. This could
  2662. # ideally be resolved by passing the reactor more generally to the
  2663. # HTTPChannel, but that won't work for the TimeoutMixin until we fix
  2664. # https://twistedmatrix.com/trac/ticket/8488
  2665. p.callLater = self.reactor.callLater
  2666. # timeOut needs to be on the Protocol instance cause
  2667. # TimeoutMixin expects it there
  2668. p.timeOut = self.timeOut
  2669. return p
  2670. def startFactory(self):
  2671. """
  2672. Set up request logging if necessary.
  2673. """
  2674. if self._logDateTimeCall is None:
  2675. self._updateLogDateTime()
  2676. if self.logPath:
  2677. self.logFile = self._openLogFile(self.logPath)
  2678. else:
  2679. self.logFile = log.logfile
  2680. def stopFactory(self):
  2681. if hasattr(self, "logFile"):
  2682. if self.logFile != log.logfile:
  2683. self.logFile.close()
  2684. del self.logFile
  2685. if self._logDateTimeCall is not None and self._logDateTimeCall.active():
  2686. self._logDateTimeCall.cancel()
  2687. self._logDateTimeCall = None
  2688. def _openLogFile(self, path):
  2689. """
  2690. Override in subclasses, e.g. to use L{twisted.python.logfile}.
  2691. """
  2692. f = open(path, "ab", 1)
  2693. return f
  2694. def log(self, request):
  2695. """
  2696. Write a line representing C{request} to the access log file.
  2697. @param request: The request object about which to log.
  2698. @type request: L{Request}
  2699. """
  2700. try:
  2701. logFile = self.logFile
  2702. except AttributeError:
  2703. pass
  2704. else:
  2705. line = self._logFormatter(self._logDateTime, request) + "\n"
  2706. logFile.write(line.encode("utf8"))