started work on backend
This commit is contained in:
d3m1g0d
@@ -0,0 +1,451 @@
|
||||
"""
|
||||
This module extends the functionality of `urllib.request.Request` to support multipart requests, to support passing
|
||||
instances of serial models to the `data` parameter/property for `urllib.request.Request`, and to
|
||||
support casting requests as `str` or `bytes` (typically for debugging purposes and/or to aid in producing
|
||||
non-language-specific API documentation).
|
||||
"""
|
||||
# region Backwards Compatibility
|
||||
from __future__ import nested_scopes, generators, division, absolute_import, with_statement, \
|
||||
print_function, unicode_literals
|
||||
from .utilities.compatibility import backport
|
||||
|
||||
backport() # noqa
|
||||
|
||||
from future.utils import native_str
|
||||
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
from typing import Dict, Sequence, Set, Iterable
|
||||
except ImportError:
|
||||
Dict = Sequence = Set = None
|
||||
|
||||
from serial.marshal import serialize
|
||||
from .abc.model import Model
|
||||
from .utilities import collections_abc
|
||||
|
||||
|
||||
class Headers(object):
|
||||
"""
|
||||
A dictionary of headers for a `Request`, `Part`, or `MultipartRequest` instance.
|
||||
"""
|
||||
|
||||
def __init__(self, items, request):
|
||||
# type: (Dict[str, str], Union[Part, Request]) -> None
|
||||
self._dict = {}
|
||||
self.request = request # type: Data
|
||||
self.update(items)
|
||||
|
||||
def pop(self, key, default=None):
|
||||
# type: (str, Optional[str]) -> str
|
||||
key = key.capitalize()
|
||||
if hasattr(self.request, '_boundary'):
|
||||
self.request._boundary = None
|
||||
if hasattr(self.request, '_bytes'):
|
||||
self.request._bytes = None
|
||||
return self._dict.pop(key, default=default)
|
||||
|
||||
def popitem(self):
|
||||
# type: (str, Optional[str]) -> str
|
||||
if hasattr(self.request, '_boundary'):
|
||||
self.request._boundary = None
|
||||
if hasattr(self.request, '_bytes'):
|
||||
self.request._bytes = None
|
||||
return self._dict.popitem()
|
||||
|
||||
def setdefault(self, key, default=None):
|
||||
# type: (str, Optional[str]) -> str
|
||||
key = key.capitalize()
|
||||
if hasattr(self.request, '_boundary'):
|
||||
self.request._boundary = None
|
||||
if hasattr(self.request, '_bytes'):
|
||||
self.request._bytes = None
|
||||
return self._dict.setdefault(key, default=default)
|
||||
|
||||
def update(self, iterable=None, **kwargs):
|
||||
# type: (Union[Dict[str, str], Sequence[Tuple[str, str]]], Union[Dict[str, str]]) -> None
|
||||
cd = {}
|
||||
if iterable is None:
|
||||
d = kwargs
|
||||
else:
|
||||
d = dict(iterable, **kwargs)
|
||||
for k, v in d.items():
|
||||
cd[k.capitalize()] = v
|
||||
if hasattr(self.request, '_boundary'):
|
||||
self.request._boundary = None
|
||||
if hasattr(self.request, '_bytes'):
|
||||
self.request._bytes = None
|
||||
return self._dict.update(cd)
|
||||
|
||||
def __delitem__(self, key):
|
||||
# type: (str) -> None
|
||||
key = key.capitalize()
|
||||
if hasattr(self.request, '_boundary'):
|
||||
self.request._boundary = None
|
||||
if hasattr(self.request, '_bytes'):
|
||||
self.request._bytes = None
|
||||
del self._dict[key]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
# type: (str, str) -> None
|
||||
key = key.capitalize()
|
||||
if key != 'Content-length':
|
||||
if hasattr(self.request, '_boundary'):
|
||||
self.request._boundary = None
|
||||
if hasattr(self.request, '_bytes'):
|
||||
self.request._bytes = None
|
||||
return self._dict.__setitem__(key, value)
|
||||
|
||||
def __getitem__(self, key):
|
||||
# type: (str) -> None
|
||||
key = key.capitalize()
|
||||
if key == 'Content-length':
|
||||
data = self.request.data
|
||||
if data is None:
|
||||
content_length = 0
|
||||
else:
|
||||
content_length = len(data)
|
||||
value = str(content_length)
|
||||
else:
|
||||
try:
|
||||
value = self._dict.__getitem__(key)
|
||||
except KeyError as e:
|
||||
if key == 'Content-type':
|
||||
if hasattr(self.request, 'parts') and self.request.parts:
|
||||
value = 'multipart/form-data'
|
||||
if (
|
||||
(value is not None) and
|
||||
value.strip().lower()[:9] == 'multipart' and
|
||||
hasattr(self.request, 'boundary')
|
||||
):
|
||||
value += '; boundary=' + str(self.request.boundary, encoding='utf-8')
|
||||
return value
|
||||
|
||||
def keys(self):
|
||||
# type: (...) -> Iterable[str]
|
||||
return (k for k in self)
|
||||
|
||||
def values(self):
|
||||
return (self[k] for k in self)
|
||||
|
||||
def __len__(self):
|
||||
return len(tuple(self))
|
||||
|
||||
def __iter__(self):
|
||||
# type: (...) -> Iterable[str]
|
||||
keys = set()
|
||||
for k in self._dict.keys():
|
||||
keys.add(k)
|
||||
yield k
|
||||
if type(self.request) is not Part:
|
||||
# *Always* include "Content-length"
|
||||
if 'Content-length' not in keys:
|
||||
yield 'Content-length'
|
||||
if (
|
||||
hasattr(self.request, 'parts') and
|
||||
self.request.parts and
|
||||
('Content-type' not in keys)
|
||||
):
|
||||
yield 'Content-type'
|
||||
|
||||
def __contains__(self, key):
|
||||
# type: (str) -> bool
|
||||
return True if key in self.keys() else False
|
||||
|
||||
def items(self):
|
||||
# type: (...) -> Iterable[Tuple[str, str]]
|
||||
for k in self:
|
||||
yield k, self[k]
|
||||
|
||||
def copy(self):
|
||||
# type: (...) -> Headers
|
||||
return self.__class__(
|
||||
self._dict,
|
||||
request=self.request
|
||||
)
|
||||
|
||||
def __copy__(self):
|
||||
# type: (...) -> Headers
|
||||
return self.copy()
|
||||
|
||||
|
||||
class Data(object):
|
||||
"""
|
||||
One of a multipart request's parts.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data=None, # type: Optional[Union[bytes, str, Sequence, Set, dict, Model]]
|
||||
headers=None # type: Optional[Dict[str, str]]
|
||||
):
|
||||
"""
|
||||
Parameters:
|
||||
|
||||
- data (bytes|str|collections.Sequence|collections.Set|dict|serial.abc.Model): The payload.
|
||||
|
||||
- headers ({str: str}): A dictionary of headers (for this part of the request body, not the main request).
|
||||
This should (almost) always include values for "Content-Disposition" and "Content-Type".
|
||||
"""
|
||||
self._bytes = None # type: Optional[bytes]
|
||||
self._headers = None
|
||||
self._data = None
|
||||
self.headers = headers # type: Dict[str, str]
|
||||
self.data = data # type: Optional[bytes]
|
||||
|
||||
@property
|
||||
def headers(self):
|
||||
return self._headers
|
||||
|
||||
@headers.setter
|
||||
def headers(self, headers):
|
||||
self._bytes = None
|
||||
if headers is None:
|
||||
headers = Headers({}, self)
|
||||
elif isinstance(headers, Headers):
|
||||
headers.request = self
|
||||
else:
|
||||
headers = Headers(headers, self)
|
||||
self._headers = headers
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
return self._data
|
||||
|
||||
@data.setter
|
||||
def data(self, data):
|
||||
# type: (Optional[Union[bytes, str, Sequence, Set, dict, Model]]) -> None
|
||||
self._bytes = None
|
||||
if data is not None:
|
||||
serialize_type = None
|
||||
if 'Content-type' in self.headers:
|
||||
ct = self.headers['Content-type']
|
||||
if re.search(r'/json\b', ct) is not None:
|
||||
serialize_type = 'json'
|
||||
if re.search(r'/xml\b', ct) is not None:
|
||||
serialize_type = 'xml'
|
||||
if re.search(r'/yaml\b', ct) is not None:
|
||||
serialize_type = 'yaml'
|
||||
if isinstance(data, (Model, dict)) or (
|
||||
isinstance(data, (collections_abc.Sequence, collections_abc.Set)) and not
|
||||
isinstance(data, (str, bytes))
|
||||
):
|
||||
data = serialize(data, serialize_type or 'json')
|
||||
if isinstance(data, str):
|
||||
data = bytes(data, encoding='utf-8')
|
||||
self._data = data
|
||||
|
||||
def __bytes__(self):
|
||||
if self._bytes is None:
|
||||
lines = []
|
||||
for k, v in self.headers.items():
|
||||
lines.append(bytes(
|
||||
'%s: %s' % (k, v),
|
||||
encoding='utf-8'
|
||||
))
|
||||
lines.append(b'')
|
||||
data = self.data
|
||||
if data:
|
||||
lines.append(self.data)
|
||||
self._bytes = b'\r\n'.join(lines) + b'\r\n'
|
||||
return self._bytes
|
||||
|
||||
def __str__(self):
|
||||
b = self.__bytes__()
|
||||
if not isinstance(b, native_str):
|
||||
b = repr(b)[2:-1].replace('\\r\\n', '\r\n').replace('\\n', '\n')
|
||||
return b
|
||||
|
||||
|
||||
class Part(Data):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data=None, # type: Optional[Union[bytes, str, Sequence, Set, dict, Model]]
|
||||
headers=None, # type: Optional[Dict[str, str]]
|
||||
parts=None # type: Optional[Sequence[Part]]
|
||||
):
|
||||
"""
|
||||
Parameters:
|
||||
|
||||
- data (bytes|str|collections.Sequence|collections.Set|dict|serial.abc.Model): The payload.
|
||||
|
||||
- headers ({str: str}): A dictionary of headers (for this part of the request body, not the main request).
|
||||
This should (almost) always include values for "Content-Disposition" and "Content-Type".
|
||||
"""
|
||||
self._boundary = None # type: Optional[bytes]
|
||||
self._parts = None # type: Optional[Parts]
|
||||
self.parts = parts
|
||||
Data.__init__(self, data=data, headers=headers)
|
||||
|
||||
@property
|
||||
def boundary(self):
|
||||
"""
|
||||
Calculates a boundary which is not contained in any of the request parts.
|
||||
"""
|
||||
if self._boundary is None:
|
||||
data = b'\r\n'.join(
|
||||
[self._data or b''] +
|
||||
[bytes(p) for p in self.parts]
|
||||
)
|
||||
boundary = b''.join(
|
||||
bytes(
|
||||
random.choice(string.digits + string.ascii_letters),
|
||||
encoding='utf-8'
|
||||
)
|
||||
for i in range(16)
|
||||
)
|
||||
while boundary in data:
|
||||
boundary += bytes(
|
||||
random.choice(string.digits + string.ascii_letters),
|
||||
encoding='utf-8'
|
||||
)
|
||||
self._boundary = boundary
|
||||
return self._boundary
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
# type: (bytes) -> None
|
||||
if self.parts:
|
||||
data = (b'\r\n--' + self.boundary + b'\r\n').join(
|
||||
[self._data or b''] +
|
||||
[bytes(p).rstrip() for p in self.parts]
|
||||
) + (b'\r\n--' + self.boundary + b'--')
|
||||
else:
|
||||
data = self._data
|
||||
return data
|
||||
|
||||
@data.setter
|
||||
def data(self, data):
|
||||
return Data.data.__set__(self, data)
|
||||
|
||||
@property
|
||||
def parts(self):
|
||||
# type: (...) -> Parts
|
||||
return self._parts
|
||||
|
||||
@parts.setter
|
||||
def parts(self, parts):
|
||||
# type: (Optional[Sequence[Part]]) -> None
|
||||
if parts is None:
|
||||
parts = Parts([], request=self)
|
||||
elif isinstance(parts, Parts):
|
||||
parts.request = self
|
||||
else:
|
||||
parts = Parts(parts, request=self)
|
||||
self._boundary = None
|
||||
self._parts = parts
|
||||
|
||||
|
||||
class Parts(list):
|
||||
|
||||
def __init__(self, items, request):
|
||||
# type: (typing.Sequence[Part], MultipartRequest) -> None
|
||||
self.request = request
|
||||
super().__init__(items)
|
||||
|
||||
def append(self, item):
|
||||
# type: (Part) -> None
|
||||
self.request._boundary = None
|
||||
self.request._bytes = None
|
||||
super().append(item)
|
||||
|
||||
def clear(self):
|
||||
# type: (...) -> None
|
||||
self.request._boundary = None
|
||||
self.request._bytes = None
|
||||
super().clear()
|
||||
|
||||
def extend(self, items):
|
||||
# type: (Iterable[Part]) -> None
|
||||
self.request._boundary = None
|
||||
self.request._bytes = None
|
||||
super().extend(items)
|
||||
|
||||
def reverse(self):
|
||||
# type: (...) -> None
|
||||
self.request._boundary = None
|
||||
self.request._bytes = None
|
||||
super().reverse()
|
||||
|
||||
def __delitem__(self, key):
|
||||
# type: (str) -> None
|
||||
self.request._boundary = None
|
||||
self.request._bytes = None
|
||||
super().__delitem__(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
# type: (str) -> None
|
||||
self.request._boundary = None
|
||||
self.request._bytes = None
|
||||
super().__setitem__(key, value)
|
||||
|
||||
|
||||
class Request(Data, urllib.request.Request):
|
||||
"""
|
||||
A sub-class of `urllib.request.Request` which accommodates additional data types, and serializes `data` in
|
||||
accordance with what is indicated by the request's "Content-Type" header.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url,
|
||||
data=None, # type: Optional[Union[bytes, str, Sequence, Set, dict, Model]]
|
||||
headers=None, # type: Optional[Dict[str, str]]
|
||||
origin_req_host=None, # type: Optional[str]
|
||||
unverifiable=False, # type: bool
|
||||
method=None # type: Optional[str]
|
||||
):
|
||||
# type: (...) -> None
|
||||
self._bytes = None # type: Optional[bytes]
|
||||
self._headers = None
|
||||
self._data = None
|
||||
self.headers = headers
|
||||
urllib.request.Request.__init__(
|
||||
self,
|
||||
url,
|
||||
data=data,
|
||||
headers=headers,
|
||||
origin_req_host=origin_req_host,
|
||||
unverifiable=unverifiable,
|
||||
method=method
|
||||
)
|
||||
|
||||
|
||||
class MultipartRequest(Part, Request):
|
||||
"""
|
||||
A sub-class of `Request` which adds a property (and initialization parameter) to hold the `parts` of a
|
||||
multipart request.
|
||||
|
||||
https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url,
|
||||
data=None, # type: Optional[Union[bytes, str, Sequence, Set, dict, Model]]
|
||||
headers=None, # type: Optional[Dict[str, str]]
|
||||
origin_req_host=None, # type: Optional[str]
|
||||
unverifiable=False, # type: bool
|
||||
method=None, # type: Optional[str]
|
||||
parts=None # type: Optional[Sequence[Part]]
|
||||
):
|
||||
# type: (...) -> None
|
||||
Part.__init__(
|
||||
self,
|
||||
data=data,
|
||||
headers=headers,
|
||||
parts=parts
|
||||
)
|
||||
Request.__init__(
|
||||
self,
|
||||
url,
|
||||
data=data,
|
||||
headers=headers,
|
||||
origin_req_host=origin_req_host,
|
||||
unverifiable=unverifiable,
|
||||
method=method
|
||||
)
|
||||
Reference in New Issue
Block a user