705 lines
22 KiB
Python
705 lines
22 KiB
Python
# Tell the linters what's up:
|
|
# pylint:disable=wrong-import-position,consider-using-enumerate,useless-object-inheritance
|
|
# mccabe:options:max-complexity=999
|
|
from __future__ import nested_scopes, generators, division, absolute_import, with_statement, \
|
|
print_function, unicode_literals
|
|
|
|
from .utilities.compatibility import backport
|
|
|
|
backport() # noqa
|
|
|
|
from future.utils import native_str
|
|
|
|
from copy import deepcopy
|
|
|
|
import json
|
|
from decimal import Decimal
|
|
from base64 import b64encode
|
|
from numbers import Number
|
|
|
|
import yaml
|
|
from datetime import date, datetime
|
|
|
|
from itertools import chain
|
|
|
|
import serial
|
|
from . import utilities, abc, properties, errors, meta, hooks
|
|
from .utilities import Generator, qualified_name, read, collections, collections_abc
|
|
|
|
|
|
try:
|
|
from typing import Union, Optional, Sequence, Any, Callable
|
|
except ImportError:
|
|
Union = Optional = Sequence = Any = Callable = None
|
|
|
|
try:
|
|
from abc import ABC
|
|
except ImportError:
|
|
ABC = None
|
|
|
|
|
|
UNMARSHALLABLE_TYPES = tuple({
|
|
str, bytes, native_str, Number, Decimal, date, datetime, bool,
|
|
dict, collections.OrderedDict,
|
|
collections_abc.Set, collections_abc.Sequence, Generator,
|
|
abc.model.Model, properties.Null, properties.NoneType
|
|
})
|
|
|
|
|
|
def ab2c(abc_or_property):
|
|
# type: (ABC) -> Union[model.Object, model.Dict, model.Array, properties.Property]
|
|
"""
|
|
Get the corresponding model class from an abstract base class, or if none exists--return the original class or type
|
|
"""
|
|
|
|
class_or_property = abc_or_property # type: (type)
|
|
|
|
if isinstance(abc_or_property, type):
|
|
|
|
if abc_or_property in {abc.model.Dictionary, abc.model.Array, abc.model.Object}:
|
|
|
|
class_or_property = getattr(
|
|
serial.model,
|
|
abc_or_property.__name__.split('.')[-1]
|
|
)
|
|
|
|
elif isinstance(abc_or_property, properties.Property):
|
|
|
|
class_or_property = deepcopy(abc_or_property)
|
|
|
|
for types_attribute in ('types', 'value_types', 'item_types'):
|
|
|
|
if hasattr(class_or_property, types_attribute):
|
|
|
|
setattr(
|
|
class_or_property,
|
|
types_attribute,
|
|
tuple(
|
|
ab2c(type_) for type_ in getattr(class_or_property, types_attribute)
|
|
)
|
|
)
|
|
|
|
return class_or_property
|
|
|
|
|
|
def ab2cs(abcs_or_properties):
|
|
# type: (Sequence[Union[ABC, properties.Property]]) -> Sequence[model.Model, properties.Property]
|
|
for abc_or_property in abcs_or_properties:
|
|
yield ab2c(abc_or_property)
|
|
|
|
|
|
class _Marshal(object):
|
|
|
|
pass
|
|
|
|
|
|
def marshal(
|
|
data, # type: Any
|
|
types=None, # type: Optional[Sequence[Union[type, properties.Property, Callable]]]
|
|
value_types=None, # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
item_types=None, # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
):
|
|
# type: (...) -> Any
|
|
|
|
"""
|
|
Recursively converts instances of `serial.abc.model.Model` into JSON/YAML/XML serializable objects.
|
|
"""
|
|
|
|
if hasattr(data, '_marshal'):
|
|
return data._marshal() # noqa - this is *our* protected member, so linters can buzz off
|
|
|
|
if data is None:
|
|
return data
|
|
|
|
if callable(types):
|
|
types = types(data)
|
|
|
|
# If data types have been provided, validate the un-marshalled data by attempting to initialize the provided type(s)
|
|
# with `data`
|
|
|
|
if types is not None:
|
|
|
|
if (str in types) and (native_str is not str) and (native_str not in types):
|
|
|
|
types = tuple(chain(*(
|
|
((type_, native_str) if (type_ is str) else (type_,))
|
|
for type_ in types
|
|
)))
|
|
|
|
matched = False
|
|
|
|
for type_ in types:
|
|
|
|
if isinstance(type_, properties.Property):
|
|
|
|
try:
|
|
data = type_.marshal(data)
|
|
matched = True
|
|
break
|
|
except TypeError:
|
|
pass
|
|
|
|
elif isinstance(type_, type) and isinstance(data, type_):
|
|
|
|
matched = True
|
|
|
|
break
|
|
|
|
if not matched:
|
|
raise TypeError(
|
|
'%s cannot be interpreted as any of the designated types: %s' % (
|
|
repr(data),
|
|
repr(types)
|
|
)
|
|
)
|
|
|
|
if value_types is not None:
|
|
for k, v in data.items():
|
|
data[k] = marshal(v, types=value_types)
|
|
|
|
if item_types is not None:
|
|
|
|
for i in range(len(data)):
|
|
data[i] = marshal(data[i], types=item_types)
|
|
|
|
if isinstance(data, Decimal):
|
|
return float(data)
|
|
|
|
if isinstance(data, (date, datetime)):
|
|
return data.isoformat()
|
|
|
|
if isinstance(data, native_str):
|
|
return data
|
|
|
|
if isinstance(data, (bytes, bytearray)):
|
|
return str(b64encode(data), 'ascii')
|
|
|
|
if hasattr(data, '__bytes__'):
|
|
return str(b64encode(bytes(data)), 'ascii')
|
|
|
|
return data
|
|
|
|
|
|
class _Unmarshal(object):
|
|
"""
|
|
This class should be used exclusively by `serial.marshal.unmarshal`.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
data, # type: Any
|
|
types=None, # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
value_types=None, # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
item_types=None # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
):
|
|
# type: (...) -> None
|
|
|
|
if not isinstance(
|
|
data,
|
|
UNMARSHALLABLE_TYPES
|
|
):
|
|
# Verify that the data can be parsed before attempting to un-marshall it
|
|
|
|
raise errors.UnmarshalTypeError(
|
|
'%s, an instance of `%s`, cannot be un-marshalled. ' % (repr(data), type(data).__name__) +
|
|
'Acceptable types are: ' + ', '.join((
|
|
qualified_name(data_type)
|
|
for data_type in UNMARSHALLABLE_TYPES
|
|
))
|
|
)
|
|
|
|
# If only one type was passed for any of the following parameters--we convert it to a tuple
|
|
# If any parameters are abstract base classes--we convert them to the corresponding models
|
|
|
|
if types is not None:
|
|
|
|
if not isinstance(types, collections_abc.Sequence):
|
|
types = (types,)
|
|
|
|
if value_types is not None:
|
|
|
|
if not isinstance(value_types, collections_abc.Sequence):
|
|
value_types = (value_types,)
|
|
|
|
if item_types is not None:
|
|
|
|
if not isinstance(item_types, collections_abc.Sequence):
|
|
item_types = (item_types,)
|
|
|
|
# Member data
|
|
|
|
self.data = data # type: Any
|
|
self.types = types # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
self.value_types = value_types # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
self.item_types = item_types # type: Optional[Sequence[Union[type, properties.Property]]]
|
|
self.meta = None # type: Optional[meta.Meta]
|
|
|
|
def __call__(self):
|
|
# type: (...) -> Any
|
|
"""
|
|
Return `self.data` unmarshalled
|
|
"""
|
|
|
|
unmarshalled_data = self.data
|
|
|
|
if (
|
|
(self.data is not None) and
|
|
(self.data is not properties.NULL)
|
|
):
|
|
# If the data is a serial `Model`, get it's metadata
|
|
if isinstance(self.data, abc.model.Model):
|
|
|
|
self.meta = meta.read(self.data)
|
|
|
|
if self.meta is None: # Only un-marshall models if they have no metadata yet (are generic)
|
|
|
|
# If `types` is a function, it should be one which expects to receive marshalled data and returns a list
|
|
# of types which are applicable
|
|
if callable(self.types):
|
|
self.types = self.types(self.data)
|
|
|
|
# If the data provided is a `Generator`, make it static by casting the data into a tuple
|
|
if isinstance(self.data, Generator):
|
|
self.data = tuple(self.data)
|
|
|
|
if self.types is None:
|
|
|
|
# If no types are provided, we unmarshal the data into one of serial's generic container types
|
|
unmarshalled_data = self.as_container_or_simple_type
|
|
|
|
else:
|
|
|
|
self.backport_types()
|
|
|
|
unmarshalled_data = None
|
|
successfully_unmarshalled = False
|
|
|
|
first_error = None # type: Optional[Exception]
|
|
|
|
# Attempt to un-marshal the data as each type, in the order provided
|
|
for type_ in self.types:
|
|
|
|
error = None # type: Optional[Union[AttributeError, KeyError, TypeError, ValueError]]
|
|
|
|
try:
|
|
|
|
unmarshalled_data = self.as_type(type_)
|
|
|
|
# if (self.data is not None) and (unmarshalled_data is None):
|
|
# raise RuntimeError(self.data)
|
|
|
|
# If the data is un-marshalled successfully, we do not need to try any further types
|
|
successfully_unmarshalled = True
|
|
break
|
|
|
|
except (AttributeError, KeyError, TypeError, ValueError) as e:
|
|
|
|
error = e
|
|
|
|
if (first_error is None) and (error is not None):
|
|
first_error = error
|
|
|
|
if not successfully_unmarshalled:
|
|
|
|
if (first_error is None) or isinstance(first_error, TypeError):
|
|
|
|
raise errors.UnmarshalTypeError(
|
|
self.data,
|
|
types=self.types,
|
|
value_types=self.value_types,
|
|
item_types=self.item_types
|
|
)
|
|
|
|
elif isinstance(first_error, ValueError):
|
|
|
|
raise errors.UnmarshalValueError(
|
|
self.data,
|
|
types=self.types,
|
|
value_types=self.value_types,
|
|
item_types=self.item_types
|
|
)
|
|
|
|
else:
|
|
|
|
raise first_error # noqa - pylint erroneously identifies this as raising `None`
|
|
|
|
return unmarshalled_data
|
|
|
|
@property
|
|
def as_container_or_simple_type(self):
|
|
# type: (...) -> Any
|
|
|
|
"""
|
|
This function unmarshalls and returns the data into one of serial's container types, or if the data is of a
|
|
simple data type--it returns that data unmodified
|
|
"""
|
|
|
|
unmarshalled_data = self.data
|
|
|
|
if isinstance(self.data, abc.model.Dictionary):
|
|
|
|
type_ = type(self.data())
|
|
|
|
if self.value_types is not None:
|
|
unmarshalled_data = type_(self.data, value_types=self.value_types)
|
|
|
|
elif isinstance(self.data, abc.model.Array):
|
|
|
|
if self.item_types is not None:
|
|
unmarshalled_data = serial.model.Array(self.data, item_types=self.item_types)
|
|
|
|
elif isinstance(self.data, (dict, collections.OrderedDict)):
|
|
|
|
unmarshalled_data = serial.model.Dictionary(self.data, value_types=self.value_types)
|
|
|
|
elif (
|
|
isinstance(self.data, (collections_abc.Set, collections_abc.Sequence))
|
|
) and (
|
|
not isinstance(self.data, (str, bytes, native_str))
|
|
):
|
|
|
|
unmarshalled_data = serial.model.Array(self.data, item_types=self.item_types)
|
|
|
|
elif not isinstance(self.data, (str, bytes, native_str, Number, Decimal, date, datetime, bool, abc.model.Model)):
|
|
|
|
raise errors.UnmarshalValueError(
|
|
'%s cannot be un-marshalled' % repr(self.data)
|
|
)
|
|
|
|
return unmarshalled_data
|
|
|
|
def backport_types(self):
|
|
# type: (...) -> None
|
|
"""
|
|
This examines a set of types passed to `unmarshal`, and resolves any compatibility issues with the python
|
|
version being utilized
|
|
"""
|
|
|
|
if (str in self.types) and (native_str is not str) and (native_str not in self.types):
|
|
|
|
self.types = tuple(chain(*(
|
|
((type_, native_str) if (type_ is str) else (type_,))
|
|
for type_ in self.types
|
|
))) # type: Tuple[Union[type, properties.Property], ...]
|
|
|
|
def as_type(
|
|
self,
|
|
type_, # type: Union[type, properties.Property]
|
|
):
|
|
# type: (...) -> bool
|
|
|
|
unmarshalled_data = None # type: Union[abc.model.Model, Number, str, bytes, date, datetime]
|
|
|
|
if isinstance(
|
|
type_,
|
|
properties.Property
|
|
):
|
|
|
|
unmarshalled_data = type_.unmarshal(self.data)
|
|
|
|
elif isinstance(type_, type):
|
|
|
|
if isinstance(
|
|
self.data,
|
|
(dict, collections.OrderedDict, abc.model.Model)
|
|
):
|
|
|
|
if issubclass(type_, abc.model.Object):
|
|
|
|
unmarshalled_data = type_(self.data)
|
|
|
|
elif issubclass(
|
|
type_,
|
|
abc.model.Dictionary
|
|
):
|
|
|
|
unmarshalled_data = type_(self.data, value_types=self.value_types)
|
|
|
|
elif issubclass(
|
|
type_,
|
|
(dict, collections.OrderedDict)
|
|
):
|
|
|
|
unmarshalled_data = serial.model.Dictionary(self.data, value_types=self.value_types)
|
|
|
|
else:
|
|
|
|
raise TypeError(self.data)
|
|
|
|
elif (
|
|
isinstance(self.data, (collections_abc.Set, collections_abc.Sequence, abc.model.Array)) and
|
|
(not isinstance(self.data, (str, bytes, native_str)))
|
|
):
|
|
|
|
if issubclass(type_, abc.model.Array):
|
|
|
|
unmarshalled_data = type_(self.data, item_types=self.item_types)
|
|
|
|
elif issubclass(
|
|
type_,
|
|
(collections_abc.Set, collections_abc.Sequence)
|
|
) and not issubclass(
|
|
type_,
|
|
(str, bytes, native_str)
|
|
):
|
|
|
|
unmarshalled_data = serial.model.Array(self.data, item_types=self.item_types)
|
|
|
|
else:
|
|
|
|
raise TypeError('%s is not of type `%s`' % (repr(self.data), repr(type_)))
|
|
|
|
elif isinstance(self.data, type_):
|
|
|
|
if isinstance(self.data, Decimal):
|
|
unmarshalled_data = float(self.data)
|
|
else:
|
|
unmarshalled_data = self.data
|
|
|
|
else:
|
|
|
|
raise TypeError(self.data)
|
|
|
|
return unmarshalled_data
|
|
|
|
|
|
def unmarshal(
|
|
data, # type: Any
|
|
types=None, # type: Optional[Union[Sequence[Union[type, properties.Property]], type, properties.Property]]
|
|
value_types=None, # type: Optional[Union[Sequence[Union[type, properties.Property]], type, properties.Property]]
|
|
item_types=None, # type: Optional[Union[Sequence[Union[type, properties.Property]], type, properties.Property]]
|
|
):
|
|
# type: (...) -> Optional[Union[abc.model., str, Number, date, datetime]]
|
|
"""
|
|
Converts `data` into an instance of a serial model, and recursively does the same for all member data.
|
|
|
|
Parameters:
|
|
|
|
- data ([type|serial.properties.Property]): One or more data types. Each type
|
|
|
|
This is done by attempting to cast that data into a series of `types`.
|
|
|
|
to "un-marshal" data which has been deserialized from bytes or text, but is still represented
|
|
by generic containers
|
|
"""
|
|
|
|
unmarshalled_data = _Unmarshal(
|
|
data,
|
|
types=types,
|
|
value_types=value_types,
|
|
item_types=item_types
|
|
)()
|
|
|
|
return unmarshalled_data
|
|
|
|
|
|
def serialize(data, format_='json'):
|
|
# type: (Union[abc.model.Model, str, Number], Optional[str]) -> str
|
|
"""
|
|
Serializes instances of `serial.model.Object` as JSON or YAML.
|
|
"""
|
|
instance_hooks = None
|
|
|
|
if isinstance(data, abc.model.Model):
|
|
|
|
instance_hooks = hooks.read(data)
|
|
|
|
if (instance_hooks is not None) and (instance_hooks.before_serialize is not None):
|
|
data = instance_hooks.before_serialize(data)
|
|
|
|
if format_ not in ('json', 'yaml'): # , 'xml'
|
|
|
|
format_ = format_.lower()
|
|
|
|
if format_ not in ('json', 'yaml'):
|
|
|
|
raise ValueError(
|
|
'Supported `serial.model.serialize()` `format_` values include "json" and "yaml" (not "%s").' %
|
|
format_
|
|
)
|
|
|
|
if format_ == 'json':
|
|
data = json.dumps(marshal(data))
|
|
elif format_ == 'yaml':
|
|
data = yaml.dump(marshal(data))
|
|
|
|
if (instance_hooks is not None) and (instance_hooks.after_serialize is not None):
|
|
data = instance_hooks.after_serialize(data)
|
|
|
|
if not isinstance(data, str):
|
|
if isinstance(data, native_str):
|
|
data = str(data)
|
|
|
|
return data
|
|
|
|
|
|
def deserialize(data, format_):
|
|
# type: (Optional[Union[str, IOBase, addbase]], str) -> Any
|
|
"""
|
|
Parameters:
|
|
|
|
- data (str|io.IOBase|io.addbase):
|
|
|
|
This can be a string or file-like object containing JSON, YAML, or XML serialized inforation.
|
|
|
|
- format_ (str):
|
|
|
|
This can be "json", "yaml" or "xml".
|
|
|
|
Returns:
|
|
|
|
A deserialized representation of the information you provided.
|
|
"""
|
|
if format_ not in ('json', 'yaml'): # , 'xml'
|
|
raise NotImplementedError(
|
|
'Deserialization of data in the format %s is not currently supported.' % repr(format_)
|
|
)
|
|
if not isinstance(data, (str, bytes)):
|
|
data = read(data)
|
|
if isinstance(data, bytes):
|
|
data = str(data, encoding='utf-8')
|
|
if isinstance(data, str):
|
|
if format_ == 'json':
|
|
data = json.loads(
|
|
data,
|
|
object_hook=collections.OrderedDict,
|
|
object_pairs_hook=collections.OrderedDict
|
|
)
|
|
elif format_ == 'yaml':
|
|
data = yaml.load(data)
|
|
return data
|
|
|
|
|
|
def detect_format(data):
|
|
# type: (Optional[Union[str, IOBase, addbase]]) -> Tuple[Any, str]
|
|
"""
|
|
Parameters:
|
|
|
|
- data (str|io.IOBase|io.addbase):
|
|
|
|
This can be a string or file-like object containing JSON, YAML, or XML serialized inforation.
|
|
|
|
Returns:
|
|
|
|
A tuple containing the deserialized information and a string indicating the format of that information.
|
|
"""
|
|
if not isinstance(data, str):
|
|
try:
|
|
data = utilities.read(data)
|
|
except TypeError:
|
|
return data, None
|
|
formats = ('json', 'yaml') # , 'xml'
|
|
format_ = None
|
|
for potential_format in formats:
|
|
try:
|
|
data = deserialize(data, potential_format)
|
|
format_ = potential_format
|
|
break
|
|
except (ValueError, yaml.YAMLError):
|
|
pass
|
|
if format is None:
|
|
raise ValueError(
|
|
'The data provided could not be parsed:\n' + repr(data)
|
|
)
|
|
return data, format_
|
|
|
|
|
|
def validate(
|
|
data, # type: Optional[abc.model.Model]
|
|
types=None, # type: Optional[Union[type, properties.Property, model.Object, Callable]]
|
|
raise_errors=True # type: bool
|
|
):
|
|
# type: (...) -> Sequence[str]
|
|
"""
|
|
This function verifies that all properties/items/values in an instance of `serial.abc.model.Model` are of the
|
|
correct data type(s), and that all required attributes are present (if applicable). If `raise_errors` is `True`
|
|
(this is the default)--violations will result in a validation error. If `raise_errors` is `False`--a list of error
|
|
messages will be returned if invalid/missing information is found, or an empty list otherwise.
|
|
"""
|
|
|
|
if isinstance(data, Generator):
|
|
data = tuple(data)
|
|
|
|
error_messages = []
|
|
|
|
error_message = None
|
|
|
|
if types is not None:
|
|
|
|
if callable(types):
|
|
types = types(data)
|
|
|
|
if (str in types) and (native_str is not str) and (native_str not in types):
|
|
|
|
types = tuple(chain(*(
|
|
((type_, native_str) if (type_ is str) else (type_,))
|
|
for type_ in types
|
|
)))
|
|
|
|
valid = False
|
|
|
|
for type_ in types:
|
|
|
|
if isinstance(type_, type) and isinstance(data, type_):
|
|
|
|
valid = True
|
|
break
|
|
|
|
elif isinstance(type_, properties.Property):
|
|
|
|
if type_.types is None:
|
|
|
|
valid = True
|
|
break
|
|
|
|
try:
|
|
|
|
validate(data, type_.types, raise_errors=True)
|
|
valid = True
|
|
break
|
|
|
|
except errors.ValidationError:
|
|
|
|
pass
|
|
|
|
if not valid:
|
|
|
|
error_message = (
|
|
'Invalid data:\n\n%s\n\nThe data must be one of the following types:\n\n%s' % (
|
|
'\n'.join(
|
|
' ' + line
|
|
for line in repr(data).split('\n')
|
|
),
|
|
'\n'.join(chain(
|
|
(' (',),
|
|
(
|
|
' %s,' % '\n'.join(
|
|
' ' + line
|
|
for line in repr(type_).split('\n')
|
|
).strip()
|
|
for type_ in types
|
|
),
|
|
(' )',)
|
|
))
|
|
)
|
|
)
|
|
|
|
if error_message is not None:
|
|
|
|
if (not error_messages) or (error_message not in error_messages):
|
|
|
|
error_messages.append(error_message)
|
|
|
|
if ('_validate' in dir(data)) and callable(data._validate):
|
|
|
|
error_messages.extend(
|
|
error_message for error_message in
|
|
data._validate(raise_errors=False)
|
|
if error_message not in error_messages
|
|
)
|
|
|
|
if raise_errors and error_messages:
|
|
raise errors.ValidationError('\n' + '\n\n'.join(error_messages))
|
|
|
|
return error_messages
|