demo + utils venv

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/common.py

@@ -0,0 +1,9 @@
+from pandas.compat import PY3
+
+# array compat
+if PY3:
+    frombytes = lambda obj, data: obj.frombytes(data)
+    tobytes = lambda obj: obj.tobytes()
+else:
+    frombytes = lambda obj, data: obj.fromstring(data)
+    tobytes = lambda obj: obj.tostring()

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_buffer.py

@@ -0,0 +1,21 @@
+# coding: utf-8
+
+from pandas.io.msgpack import packb, unpackb
+
+from .common import frombytes
+
+
+def test_unpack_buffer():
+    from array import array
+    buf = array('b')
+    frombytes(buf, packb((b'foo', b'bar')))
+    obj = unpackb(buf, use_list=1)
+    assert [b'foo', b'bar'] == obj
+
+
+def test_unpack_bytearray():
+    buf = bytearray(packb(('foo', 'bar')))
+    obj = unpackb(buf, use_list=1)
+    assert [b'foo', b'bar'] == obj
+    expected_type = bytes
+    assert all(type(s) == expected_type for s in obj)

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_case.py

@@ -0,0 +1,115 @@
+# coding: utf-8
+
+from pandas.io.msgpack import packb, unpackb
+
+
+def check(length, obj):
+    v = packb(obj)
+    assert len(v) == length, \
+        "%r length should be %r but get %r" % (obj, length, len(v))
+    assert unpackb(v, use_list=0) == obj
+
+
+def test_1():
+    for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
+              -((1 << 5) - 1), -(1 << 5)]:
+        check(1, o)
+
+
+def test_2():
+    for o in [1 << 7, (1 << 8) - 1, -((1 << 5) + 1), -(1 << 7)]:
+        check(2, o)
+
+
+def test_3():
+    for o in [1 << 8, (1 << 16) - 1, -((1 << 7) + 1), -(1 << 15)]:
+        check(3, o)
+
+
+def test_5():
+    for o in [1 << 16, (1 << 32) - 1, -((1 << 15) + 1), -(1 << 31)]:
+        check(5, o)
+
+
+def test_9():
+    for o in [1 << 32, (1 << 64) - 1, -((1 << 31) + 1), -(1 << 63), 1.0, 0.1,
+              -0.1, -1.0]:
+        check(9, o)
+
+
+def check_raw(overhead, num):
+    check(num + overhead, b" " * num)
+
+
+def test_fixraw():
+    check_raw(1, 0)
+    check_raw(1, (1 << 5) - 1)
+
+
+def test_raw16():
+    check_raw(3, 1 << 5)
+    check_raw(3, (1 << 16) - 1)
+
+
+def test_raw32():
+    check_raw(5, 1 << 16)
+
+
+def check_array(overhead, num):
+    check(num + overhead, (None, ) * num)
+
+
+def test_fixarray():
+    check_array(1, 0)
+    check_array(1, (1 << 4) - 1)
+
+
+def test_array16():
+    check_array(3, 1 << 4)
+    check_array(3, (1 << 16) - 1)
+
+
+def test_array32():
+    check_array(5, (1 << 16))
+
+
+def match(obj, buf):
+    assert packb(obj) == buf
+    assert unpackb(buf, use_list=0) == obj
+
+
+def test_match():
+    cases = [
+        (None, b'\xc0'),
+        (False, b'\xc2'),
+        (True, b'\xc3'),
+        (0, b'\x00'),
+        (127, b'\x7f'),
+        (128, b'\xcc\x80'),
+        (256, b'\xcd\x01\x00'),
+        (-1, b'\xff'),
+        (-33, b'\xd0\xdf'),
+        (-129, b'\xd1\xff\x7f'),
+        ({1: 1}, b'\x81\x01\x01'),
+        (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
+        ((), b'\x90'),
+        (tuple(range(15)), (b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09"
+                            b"\x0a\x0b\x0c\x0d\x0e")),
+        (tuple(range(16)), (b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07"
+                            b"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f")),
+        ({}, b'\x80'),
+        ({x: x for x in range(15)},
+         (b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07'
+          b'\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e')),
+        ({x: x for x in range(16)},
+         (b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06'
+          b'\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'
+          b'\x0f\x0f')),
+    ]
+
+    for v, p in cases:
+        match(v, p)
+
+
+def test_unicode():
+    assert unpackb(packb('foobar'), use_list=1) == b'foobar'

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_except.py

@@ -0,0 +1,39 @@
+# coding: utf-8
+
+from datetime import datetime
+
+import pytest
+
+from pandas.io.msgpack import packb, unpackb
+
+
+class DummyException(Exception):
+    pass
+
+
+class TestExceptions(object):
+
+    def test_raise_on_find_unsupported_value(self):
+        msg = "can\'t serialize datetime"
+        with pytest.raises(TypeError, match=msg):
+            packb(datetime.now())
+
+    def test_raise_from_object_hook(self):
+        def hook(_):
+            raise DummyException()
+
+        with pytest.raises(DummyException):
+            unpackb(packb({}), object_hook=hook)
+        with pytest.raises(DummyException):
+            unpackb(packb({'fizz': 'buzz'}), object_hook=hook)
+        with pytest.raises(DummyException):
+            unpackb(packb({'fizz': 'buzz'}), object_pairs_hook=hook)
+        with pytest.raises(DummyException):
+            unpackb(packb({'fizz': {'buzz': 'spam'}}), object_hook=hook)
+        with pytest.raises(DummyException):
+            unpackb(packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook)
+
+    def test_invalid_value(self):
+        msg = "Unpack failed: error"
+        with pytest.raises(ValueError, match=msg):
+            unpackb(b"\xd9\x97#DL_")

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_extension.py

@@ -0,0 +1,63 @@
+from __future__ import print_function
+
+import array
+
+import pandas.io.msgpack as msgpack
+from pandas.io.msgpack import ExtType
+
+from .common import frombytes, tobytes
+
+
+def test_pack_ext_type():
+    def p(s):
+        packer = msgpack.Packer()
+        packer.pack_ext_type(0x42, s)
+        return packer.bytes()
+
+    assert p(b'A') == b'\xd4\x42A'  # fixext 1
+    assert p(b'AB') == b'\xd5\x42AB'  # fixext 2
+    assert p(b'ABCD') == b'\xd6\x42ABCD'  # fixext 4
+    assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH'  # fixext 8
+    assert p(b'A' * 16) == b'\xd8\x42' + b'A' * 16  # fixext 16
+    assert p(b'ABC') == b'\xc7\x03\x42ABC'  # ext 8
+    assert p(b'A' * 0x0123) == b'\xc8\x01\x23\x42' + b'A' * 0x0123  # ext 16
+    assert (p(b'A' * 0x00012345) ==
+            b'\xc9\x00\x01\x23\x45\x42' + b'A' * 0x00012345)  # ext 32
+
+
+def test_unpack_ext_type():
+    def check(b, expected):
+        assert msgpack.unpackb(b) == expected
+
+    check(b'\xd4\x42A', ExtType(0x42, b'A'))  # fixext 1
+    check(b'\xd5\x42AB', ExtType(0x42, b'AB'))  # fixext 2
+    check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD'))  # fixext 4
+    check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH'))  # fixext 8
+    check(b'\xd8\x42' + b'A' * 16, ExtType(0x42, b'A' * 16))  # fixext 16
+    check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC'))  # ext 8
+    check(b'\xc8\x01\x23\x42' + b'A' * 0x0123,
+          ExtType(0x42, b'A' * 0x0123))  # ext 16
+    check(b'\xc9\x00\x01\x23\x45\x42' + b'A' * 0x00012345,
+          ExtType(0x42, b'A' * 0x00012345))  # ext 32
+
+
+def test_extension_type():
+    def default(obj):
+        print('default called', obj)
+        if isinstance(obj, array.array):
+            typecode = 123  # application specific typecode
+            data = tobytes(obj)
+            return ExtType(typecode, data)
+        raise TypeError("Unknown type object %r" % (obj, ))
+
+    def ext_hook(code, data):
+        print('ext_hook called', code, data)
+        assert code == 123
+        obj = array.array('d')
+        frombytes(obj, data)
+        return obj
+
+    obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
+    s = msgpack.packb(obj, default=default)
+    obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
+    assert obj == obj2

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_format.py

@@ -0,0 +1,91 @@
+# coding: utf-8
+
+from pandas.io.msgpack import unpackb
+
+
+def check(src, should, use_list=0):
+    assert unpackb(src, use_list=use_list) == should
+
+
+def testSimpleValue():
+    check(b"\x93\xc0\xc2\xc3", (None, False, True, ))
+
+
+def testFixnum():
+    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0,
+                                                     64,
+                                                     127, ),
+                                                    (-32,
+                                                     -16,
+                                                     -1, ), ))
+
+
+def testFixArray():
+    check(b"\x92\x90\x91\x91\xc0", ((), ((None, ), ), ), )
+
+
+def testFixRaw():
+    check(b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def", ), )
+
+
+def testFixMap():
+    check(b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
+          {False: {None: None},
+           True: {None: {}}}, )
+
+
+def testUnsignedInt():
+    check(b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
+          b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
+          b"\xce\xff\xff\xff\xff",
+          (0,
+           128,
+           255,
+           0,
+           32768,
+           65535,
+           0,
+           2147483648,
+           4294967295, ), )
+
+
+def testSignedInt():
+    check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
+          b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
+          b"\xd2\xff\xff\xff\xff", (0,
+                                    -128,
+                                    -1,
+                                    0,
+                                    -32768,
+                                    -1,
+                                    0,
+                                    -2147483648,
+                                    -1, ))
+
+
+def testRaw():
+    check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+          b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
+          (b"", b"a", b"ab", b"", b"a", b"ab"))
+
+
+def testArray():
+    check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
+          b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
+          b"\xc2\xc3", ((), (None, ), (False, True), (), (None, ),
+                        (False, True)))
+
+
+def testMap():
+    check(b"\x96"
+          b"\xde\x00\x00"
+          b"\xde\x00\x01\xc0\xc2"
+          b"\xde\x00\x02\xc0\xc2\xc3\xc2"
+          b"\xdf\x00\x00\x00\x00"
+          b"\xdf\x00\x00\x00\x01\xc0\xc2"
+          b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", ({}, {None: False},
+                                                    {True: False,
+                                                     None: False}, {},
+                                                    {None: False},
+                                                    {True: False,
+                                                     None: False}))

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_limits.py

@@ -0,0 +1,109 @@
+# coding: utf-8
+from __future__ import (
+    absolute_import, division, print_function, unicode_literals)
+
+import pytest
+
+from pandas.io.msgpack import ExtType, Packer, Unpacker, packb, unpackb
+
+
+class TestLimits(object):
+
+    def test_integer(self):
+        x = -(2 ** 63)
+        assert unpackb(packb(x)) == x
+        msg = (r"((long |Python )?(int )?too (big|large) to convert"
+               r"( to C (unsigned )?long))?")
+        with pytest.raises((OverflowError, ValueError), match=msg):
+            packb(x - 1)
+        x = 2 ** 64 - 1
+        assert unpackb(packb(x)) == x
+        with pytest.raises((OverflowError, ValueError), match=msg):
+            packb(x + 1)
+
+    def test_array_header(self):
+        packer = Packer()
+        packer.pack_array_header(2 ** 32 - 1)
+        with pytest.raises((OverflowError, ValueError)):
+            packer.pack_array_header(2 ** 32)
+
+    def test_map_header(self):
+        packer = Packer()
+        packer.pack_map_header(2 ** 32 - 1)
+        with pytest.raises((OverflowError, ValueError)):
+            packer.pack_array_header(2 ** 32)
+
+    def test_max_str_len(self):
+        d = 'x' * 3
+        packed = packb(d)
+
+        unpacker = Unpacker(max_str_len=3, encoding='utf-8')
+        unpacker.feed(packed)
+        assert unpacker.unpack() == d
+
+        unpacker = Unpacker(max_str_len=2, encoding='utf-8')
+        unpacker.feed(packed)
+
+        msg = "3 exceeds max_str_len"
+        with pytest.raises(ValueError, match=msg):
+            unpacker.unpack()
+
+    def test_max_bin_len(self):
+        d = b'x' * 3
+        packed = packb(d, use_bin_type=True)
+
+        unpacker = Unpacker(max_bin_len=3)
+        unpacker.feed(packed)
+        assert unpacker.unpack() == d
+
+        unpacker = Unpacker(max_bin_len=2)
+        unpacker.feed(packed)
+
+        msg = "3 exceeds max_bin_len"
+        with pytest.raises(ValueError, match=msg):
+            unpacker.unpack()
+
+    def test_max_array_len(self):
+        d = [1, 2, 3]
+        packed = packb(d)
+
+        unpacker = Unpacker(max_array_len=3)
+        unpacker.feed(packed)
+        assert unpacker.unpack() == d
+
+        unpacker = Unpacker(max_array_len=2)
+        unpacker.feed(packed)
+
+        msg = "3 exceeds max_array_len"
+        with pytest.raises(ValueError, match=msg):
+            unpacker.unpack()
+
+    def test_max_map_len(self):
+        d = {1: 2, 3: 4, 5: 6}
+        packed = packb(d)
+
+        unpacker = Unpacker(max_map_len=3)
+        unpacker.feed(packed)
+        assert unpacker.unpack() == d
+
+        unpacker = Unpacker(max_map_len=2)
+        unpacker.feed(packed)
+
+        msg = "3 exceeds max_map_len"
+        with pytest.raises(ValueError, match=msg):
+            unpacker.unpack()
+
+    def test_max_ext_len(self):
+        d = ExtType(42, b"abc")
+        packed = packb(d)
+
+        unpacker = Unpacker(max_ext_len=3)
+        unpacker.feed(packed)
+        assert unpacker.unpack() == d
+
+        unpacker = Unpacker(max_ext_len=2)
+        unpacker.feed(packed)
+
+        msg = "4 exceeds max_ext_len"
+        with pytest.raises(ValueError, match=msg):
+            unpacker.unpack()

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_newspec.py

@@ -0,0 +1,92 @@
+# coding: utf-8
+
+from pandas.io.msgpack import ExtType, packb, unpackb
+
+
+def test_str8():
+    header = b'\xd9'
+    data = b'x' * 32
+    b = packb(data.decode(), use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\x20'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 255
+    b = packb(data.decode(), use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\xff'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+
+def test_bin8():
+    header = b'\xc4'
+    data = b''
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\x00'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 255
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\xff'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+
+def test_bin16():
+    header = b'\xc5'
+    data = b'x' * 256
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 3
+    assert b[0:1] == header
+    assert b[1:3] == b'\x01\x00'
+    assert b[3:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 65535
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 3
+    assert b[0:1] == header
+    assert b[1:3] == b'\xff\xff'
+    assert b[3:] == data
+    assert unpackb(b) == data
+
+
+def test_bin32():
+    header = b'\xc6'
+    data = b'x' * 65536
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 5
+    assert b[0:1] == header
+    assert b[1:5] == b'\x00\x01\x00\x00'
+    assert b[5:] == data
+    assert unpackb(b) == data
+
+
+def test_ext():
+    def check(ext, packed):
+        assert packb(ext) == packed
+        assert unpackb(packed) == ext
+
+    check(ExtType(0x42, b'Z'), b'\xd4\x42Z')  # fixext 1
+    check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ')  # fixext 2
+    check(ExtType(0x42, b'Z' * 4), b'\xd6\x42' + b'Z' * 4)  # fixext 4
+    check(ExtType(0x42, b'Z' * 8), b'\xd7\x42' + b'Z' * 8)  # fixext 8
+    check(ExtType(0x42, b'Z' * 16), b'\xd8\x42' + b'Z' * 16)  # fixext 16
+    # ext 8
+    check(ExtType(0x42, b''), b'\xc7\x00\x42')
+    check(ExtType(0x42, b'Z' * 255), b'\xc7\xff\x42' + b'Z' * 255)
+    # ext 16
+    check(ExtType(0x42, b'Z' * 256), b'\xc8\x01\x00\x42' + b'Z' * 256)
+    check(ExtType(0x42, b'Z' * 0xffff), b'\xc8\xff\xff\x42' + b'Z' * 0xffff)
+    # ext 32
+    check(
+        ExtType(0x42, b'Z' *
+                0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z' * 0x10000)
+    # needs large memory
+    # check(ExtType(0x42, b'Z'*0xffffffff),
+    #              b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff)

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_obj.py

@@ -0,0 +1,74 @@
+# coding: utf-8
+
+import pytest
+
+from pandas.io.msgpack import packb, unpackb
+
+
+class DecodeError(Exception):
+    pass
+
+
+class TestObj(object):
+
+    def _arr_to_str(self, arr):
+        return ''.join(str(c) for c in arr)
+
+    def bad_complex_decoder(self, o):
+        raise DecodeError("Ooops!")
+
+    def _decode_complex(self, obj):
+        if b'__complex__' in obj:
+            return complex(obj[b'real'], obj[b'imag'])
+        return obj
+
+    def _encode_complex(self, obj):
+        if isinstance(obj, complex):
+            return {b'__complex__': True, b'real': 1, b'imag': 2}
+        return obj
+
+    def test_encode_hook(self):
+        packed = packb([3, 1 + 2j], default=self._encode_complex)
+        unpacked = unpackb(packed, use_list=1)
+        assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2}
+
+    def test_decode_hook(self):
+        packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
+        unpacked = unpackb(packed, object_hook=self._decode_complex,
+                           use_list=1)
+        assert unpacked[1] == 1 + 2j
+
+    def test_decode_pairs_hook(self):
+        packed = packb([3, {1: 2, 3: 4}])
+        prod_sum = 1 * 2 + 3 * 4
+        unpacked = unpackb(
+            packed, object_pairs_hook=lambda l: sum(k * v for k, v in l),
+            use_list=1)
+        assert unpacked[1] == prod_sum
+
+    def test_only_one_obj_hook(self):
+        msg = "object_pairs_hook and object_hook are mutually exclusive"
+        with pytest.raises(TypeError, match=msg):
+            unpackb(b'', object_hook=lambda x: x,
+                    object_pairs_hook=lambda x: x)
+
+    def test_bad_hook(self):
+        msg = r"can't serialize \(1\+2j\)"
+        with pytest.raises(TypeError, match=msg):
+            packed = packb([3, 1 + 2j], default=lambda o: o)
+            unpacked = unpackb(packed, use_list=1)  # noqa
+
+    def test_array_hook(self):
+        packed = packb([1, 2, 3])
+        unpacked = unpackb(packed, list_hook=self._arr_to_str, use_list=1)
+        assert unpacked == '123'
+
+    def test_an_exception_in_objecthook1(self):
+        with pytest.raises(DecodeError, match='Ooops!'):
+            packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
+            unpackb(packed, object_hook=self.bad_complex_decoder)
+
+    def test_an_exception_in_objecthook2(self):
+        with pytest.raises(DecodeError, match='Ooops!'):
+            packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
+            unpackb(packed, list_hook=self.bad_complex_decoder, use_list=1)

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_pack.py

@@ -0,0 +1,162 @@
+# coding: utf-8
+
+import struct
+
+import pytest
+
+from pandas.compat import OrderedDict, u
+
+from pandas import compat
+
+from pandas.io.msgpack import Packer, Unpacker, packb, unpackb
+
+
+class TestPack(object):
+
+    def check(self, data, use_list=False):
+        re = unpackb(packb(data), use_list=use_list)
+        assert re == data
+
+    def testPack(self):
+        test_data = [
+            0, 1, 127, 128, 255, 256, 65535, 65536,
+            -1, -32, -33, -128, -129, -32768, -32769,
+            1.0,
+            b"", b"a", b"a" * 31, b"a" * 32,
+            None, True, False,
+            (), ((),), ((), None,),
+            {None: 0},
+            (1 << 23),
+        ]
+        for td in test_data:
+            self.check(td)
+
+    def testPackUnicode(self):
+        test_data = [u(""), u("abcd"), [u("defgh")], u("Русский текст"), ]
+        for td in test_data:
+            re = unpackb(
+                packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
+            assert re == td
+            packer = Packer(encoding='utf-8')
+            data = packer.pack(td)
+            re = Unpacker(
+                compat.BytesIO(data), encoding='utf-8', use_list=1).unpack()
+            assert re == td
+
+    def testPackUTF32(self):
+        test_data = [
+            compat.u(""),
+            compat.u("abcd"),
+            [compat.u("defgh")],
+            compat.u("Русский текст"),
+        ]
+        for td in test_data:
+            re = unpackb(
+                packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
+            assert re == td
+
+    def testPackBytes(self):
+        test_data = [b"", b"abcd", (b"defgh", ), ]
+        for td in test_data:
+            self.check(td)
+
+    def testIgnoreUnicodeErrors(self):
+        re = unpackb(
+            packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore',
+            use_list=1)
+        assert re == "abcdef"
+
+    def testStrictUnicodeUnpack(self):
+        msg = (r"'utf-*8' codec can't decode byte 0xed in position 3:"
+               " invalid continuation byte")
+        with pytest.raises(UnicodeDecodeError, match=msg):
+            unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
+
+    def testStrictUnicodePack(self):
+        msg = (r"'ascii' codec can't encode character u*'\\xed' in position 3:"
+               r" ordinal not in range\(128\)")
+        with pytest.raises(UnicodeEncodeError, match=msg):
+            packb(compat.u("abc\xeddef"), encoding='ascii',
+                  unicode_errors='strict')
+
+    def testIgnoreErrorsPack(self):
+        re = unpackb(
+            packb(
+                compat.u("abcФФФdef"), encoding='ascii',
+                unicode_errors='ignore'), encoding='utf-8', use_list=1)
+        assert re == compat.u("abcdef")
+
+    def testNoEncoding(self):
+        msg = "Can't encode unicode string: no encoding is specified"
+        with pytest.raises(TypeError, match=msg):
+            packb(compat.u("abc"), encoding=None)
+
+    def testDecodeBinary(self):
+        re = unpackb(packb("abc"), encoding=None, use_list=1)
+        assert re == b"abc"
+
+    def testPackFloat(self):
+        assert packb(1.0,
+                     use_single_float=True) == b'\xca' + struct.pack('>f', 1.0)
+        assert packb(
+            1.0, use_single_float=False) == b'\xcb' + struct.pack('>d', 1.0)
+
+    def testArraySize(self, sizes=[0, 5, 50, 1000]):
+        bio = compat.BytesIO()
+        packer = Packer()
+        for size in sizes:
+            bio.write(packer.pack_array_header(size))
+            for i in range(size):
+                bio.write(packer.pack(i))
+
+        bio.seek(0)
+        unpacker = Unpacker(bio, use_list=1)
+        for size in sizes:
+            assert unpacker.unpack() == list(range(size))
+
+    def test_manualreset(self, sizes=[0, 5, 50, 1000]):
+        packer = Packer(autoreset=False)
+        for size in sizes:
+            packer.pack_array_header(size)
+            for i in range(size):
+                packer.pack(i)
+
+        bio = compat.BytesIO(packer.bytes())
+        unpacker = Unpacker(bio, use_list=1)
+        for size in sizes:
+            assert unpacker.unpack() == list(range(size))
+
+        packer.reset()
+        assert packer.bytes() == b''
+
+    def testMapSize(self, sizes=[0, 5, 50, 1000]):
+        bio = compat.BytesIO()
+        packer = Packer()
+        for size in sizes:
+            bio.write(packer.pack_map_header(size))
+            for i in range(size):
+                bio.write(packer.pack(i))  # key
+                bio.write(packer.pack(i * 2))  # value
+
+        bio.seek(0)
+        unpacker = Unpacker(bio)
+        for size in sizes:
+            assert unpacker.unpack() == {i: i * 2 for i in range(size)}
+
+    def test_odict(self):
+        seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
+        od = OrderedDict(seq)
+        assert unpackb(packb(od), use_list=1) == dict(seq)
+
+        def pair_hook(seq):
+            return list(seq)
+
+        assert unpackb(
+            packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
+
+    def test_pairlist(self):
+        pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')]
+        packer = Packer()
+        packed = packer.pack_map_pairs(pairlist)
+        unpacked = unpackb(packed, object_pairs_hook=list)
+        assert pairlist == unpacked

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_read_size.py

@@ -0,0 +1,71 @@
+"""Test Unpacker's read_array_header and read_map_header methods"""
+from pandas.io.msgpack import OutOfData, Unpacker, packb
+
+UnexpectedTypeException = ValueError
+
+
+def test_read_array_header():
+    unpacker = Unpacker()
+    unpacker.feed(packb(['a', 'b', 'c']))
+    assert unpacker.read_array_header() == 3
+    assert unpacker.unpack() == b'a'
+    assert unpacker.unpack() == b'b'
+    assert unpacker.unpack() == b'c'
+    try:
+        unpacker.unpack()
+        assert 0, 'should raise exception'
+    except OutOfData:
+        assert 1, 'okay'
+
+
+def test_read_map_header():
+    unpacker = Unpacker()
+    unpacker.feed(packb({'a': 'A'}))
+    assert unpacker.read_map_header() == 1
+    assert unpacker.unpack() == B'a'
+    assert unpacker.unpack() == B'A'
+    try:
+        unpacker.unpack()
+        assert 0, 'should raise exception'
+    except OutOfData:
+        assert 1, 'okay'
+
+
+def test_incorrect_type_array():
+    unpacker = Unpacker()
+    unpacker.feed(packb(1))
+    try:
+        unpacker.read_array_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+
+def test_incorrect_type_map():
+    unpacker = Unpacker()
+    unpacker.feed(packb(1))
+    try:
+        unpacker.read_map_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+
+def test_correct_type_nested_array():
+    unpacker = Unpacker()
+    unpacker.feed(packb({'a': ['b', 'c', 'd']}))
+    try:
+        unpacker.read_array_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+
+def test_incorrect_type_nested_map():
+    unpacker = Unpacker()
+    unpacker.feed(packb([{'a': 'b'}]))
+    try:
+        unpacker.read_map_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_seq.py

@@ -0,0 +1,47 @@
+# coding: utf-8
+
+import io
+
+import pandas.io.msgpack as msgpack
+
+binarydata = bytes(bytearray(range(256)))
+
+
+def gen_binary_data(idx):
+    return binarydata[:idx % 300]
+
+
+def test_exceeding_unpacker_read_size():
+    dumpf = io.BytesIO()
+
+    packer = msgpack.Packer()
+
+    NUMBER_OF_STRINGS = 6
+    read_size = 16
+
+    # 5 ok for read_size=16, while 6 glibc detected *** python: double free or
+    # corruption (fasttop):
+
+    # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python:
+    # double free or corruption (!prev)
+
+    # 40 ok for read_size=1024, while 50 introduces errors
+    # 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected ***
+    # python: double free or corruption (!prev):
+
+    for idx in range(NUMBER_OF_STRINGS):
+        data = gen_binary_data(idx)
+        dumpf.write(packer.pack(data))
+
+    f = io.BytesIO(dumpf.getvalue())
+    dumpf.close()
+
+    unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1)
+
+    read_count = 0
+    for idx, o in enumerate(unpacker):
+        assert type(o) == bytes
+        assert o == gen_binary_data(idx)
+        read_count += 1
+
+    assert read_count == NUMBER_OF_STRINGS

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_sequnpack.py

@@ -0,0 +1,104 @@
+# coding: utf-8
+
+import pytest
+
+from pandas import compat
+
+from pandas.io.msgpack import BufferFull, OutOfData, Unpacker
+
+
+class TestPack(object):
+
+    def test_partial_data(self):
+        unpacker = Unpacker()
+        msg = "No more data to unpack"
+
+        for data in [b"\xa5", b"h", b"a", b"l", b"l"]:
+            unpacker.feed(data)
+            with pytest.raises(StopIteration, match=msg):
+                next(iter(unpacker))
+
+        unpacker.feed(b"o")
+        assert next(iter(unpacker)) == b"hallo"
+
+    def test_foobar(self):
+        unpacker = Unpacker(read_size=3, use_list=1)
+        unpacker.feed(b'foobar')
+        assert unpacker.unpack() == ord(b'f')
+        assert unpacker.unpack() == ord(b'o')
+        assert unpacker.unpack() == ord(b'o')
+        assert unpacker.unpack() == ord(b'b')
+        assert unpacker.unpack() == ord(b'a')
+        assert unpacker.unpack() == ord(b'r')
+        msg = "No more data to unpack"
+        with pytest.raises(OutOfData, match=msg):
+            unpacker.unpack()
+
+        unpacker.feed(b'foo')
+        unpacker.feed(b'bar')
+
+        k = 0
+        for o, e in zip(unpacker, 'foobarbaz'):
+            assert o == ord(e)
+            k += 1
+        assert k == len(b'foobar')
+
+    def test_foobar_skip(self):
+        unpacker = Unpacker(read_size=3, use_list=1)
+        unpacker.feed(b'foobar')
+        assert unpacker.unpack() == ord(b'f')
+        unpacker.skip()
+        assert unpacker.unpack() == ord(b'o')
+        unpacker.skip()
+        assert unpacker.unpack() == ord(b'a')
+        unpacker.skip()
+        msg = "No more data to unpack"
+        with pytest.raises(OutOfData, match=msg):
+            unpacker.unpack()
+
+    def test_maxbuffersize_read_size_exceeds_max_buffer_size(self):
+        msg = "read_size should be less or equal to max_buffer_size"
+        with pytest.raises(ValueError, match=msg):
+            Unpacker(read_size=5, max_buffer_size=3)
+
+    def test_maxbuffersize_bufferfull(self):
+        unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
+        unpacker.feed(b'foo')
+        with pytest.raises(BufferFull, match=r'^$'):
+            unpacker.feed(b'b')
+
+    def test_maxbuffersize(self):
+        unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
+        unpacker.feed(b'foo')
+        assert ord('f') == next(unpacker)
+        unpacker.feed(b'b')
+        assert ord('o') == next(unpacker)
+        assert ord('o') == next(unpacker)
+        assert ord('b') == next(unpacker)
+
+    def test_readbytes(self):
+        unpacker = Unpacker(read_size=3)
+        unpacker.feed(b'foobar')
+        assert unpacker.unpack() == ord(b'f')
+        assert unpacker.read_bytes(3) == b'oob'
+        assert unpacker.unpack() == ord(b'a')
+        assert unpacker.unpack() == ord(b'r')
+
+        # Test buffer refill
+        unpacker = Unpacker(compat.BytesIO(b'foobar'), read_size=3)
+        assert unpacker.unpack() == ord(b'f')
+        assert unpacker.read_bytes(3) == b'oob'
+        assert unpacker.unpack() == ord(b'a')
+        assert unpacker.unpack() == ord(b'r')
+
+    def test_issue124(self):
+        unpacker = Unpacker()
+        unpacker.feed(b'\xa1?\xa1!')
+        assert tuple(unpacker) == (b'?', b'!')
+        assert tuple(unpacker) == ()
+        unpacker.feed(b"\xa1?\xa1")
+        assert tuple(unpacker) == (b'?', )
+        assert tuple(unpacker) == ()
+        unpacker.feed(b"!")
+        assert tuple(unpacker) == (b'!', )
+        assert tuple(unpacker) == ()

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_subtype.py

@@ -0,0 +1,26 @@
+# coding: utf-8
+
+from collections import namedtuple
+
+from pandas.io.msgpack import packb
+
+
+class MyList(list):
+    pass
+
+
+class MyDict(dict):
+    pass
+
+
+class MyTuple(tuple):
+    pass
+
+
+MyNamedTuple = namedtuple('MyNamedTuple', 'x y')
+
+
+def test_types():
+    assert packb(MyDict()) == packb(dict())
+    assert packb(MyList()) == packb(list())
+    assert packb(MyNamedTuple(1, 2)) == packb((1, 2))

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_unpack.py

@@ -0,0 +1,67 @@
+from io import BytesIO
+import sys
+
+import pytest
+
+from pandas.io.msgpack import ExtType, OutOfData, Unpacker, packb
+
+
+class TestUnpack(object):
+
+    def test_unpack_array_header_from_file(self):
+        f = BytesIO(packb([1, 2, 3, 4]))
+        unpacker = Unpacker(f)
+        assert unpacker.read_array_header() == 4
+        assert unpacker.unpack() == 1
+        assert unpacker.unpack() == 2
+        assert unpacker.unpack() == 3
+        assert unpacker.unpack() == 4
+        msg = "No more data to unpack"
+        with pytest.raises(OutOfData, match=msg):
+            unpacker.unpack()
+
+    def test_unpacker_hook_refcnt(self):
+        if not hasattr(sys, 'getrefcount'):
+            pytest.skip('no sys.getrefcount()')
+        result = []
+
+        def hook(x):
+            result.append(x)
+            return x
+
+        basecnt = sys.getrefcount(hook)
+
+        up = Unpacker(object_hook=hook, list_hook=hook)
+
+        assert sys.getrefcount(hook) >= basecnt + 2
+
+        up.feed(packb([{}]))
+        up.feed(packb([{}]))
+        assert up.unpack() == [{}]
+        assert up.unpack() == [{}]
+        assert result == [{}, [{}], {}, [{}]]
+
+        del up
+
+        assert sys.getrefcount(hook) == basecnt
+
+    def test_unpacker_ext_hook(self):
+        class MyUnpacker(Unpacker):
+
+            def __init__(self):
+                super(MyUnpacker, self).__init__(ext_hook=self._hook,
+                                                 encoding='utf-8')
+
+            def _hook(self, code, data):
+                if code == 1:
+                    return int(data)
+                else:
+                    return ExtType(code, data)
+
+        unpacker = MyUnpacker()
+        unpacker.feed(packb({'a': 1}, encoding='utf-8'))
+        assert unpacker.unpack() == {'a': 1}
+        unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8'))
+        assert unpacker.unpack() == {'a': 123}
+        unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8'))
+        assert unpacker.unpack() == {'a': ExtType(2, b'321')}

utils/venv/lib/python3.6/site-packages/pandas/tests/io/msgpack/test_unpack_raw.py

@@ -0,0 +1,30 @@
+"""Tests for cases where the user seeks to obtain packed msgpack objects"""
+
+import io
+
+from pandas.io.msgpack import Unpacker, packb
+
+
+def test_write_bytes():
+    unpacker = Unpacker()
+    unpacker.feed(b'abc')
+    f = io.BytesIO()
+    assert unpacker.unpack(f.write) == ord('a')
+    assert f.getvalue() == b'a'
+    f = io.BytesIO()
+    assert unpacker.skip(f.write) is None
+    assert f.getvalue() == b'b'
+    f = io.BytesIO()
+    assert unpacker.skip() is None
+    assert f.getvalue() == b''
+
+
+def test_write_bytes_multi_buffer():
+    long_val = (5) * 100
+    expected = packb(long_val)
+    unpacker = Unpacker(io.BytesIO(expected), read_size=3, max_buffer_size=3)
+
+    f = io.BytesIO()
+    unpacked = unpacker.unpack(f.write)
+    assert unpacked == long_val
+    assert f.getvalue() == expected