kristjank/TeraHz
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

pruned venvs

Browse Source
This commit is contained in:
d3m1g0d
2019-03-12 21:57:16 +01:00
parent 33f0511081
commit e441f4f7f7
5988 changed files with 0 additions and 1353666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/venv/lib/python3.6/site-packages/scipy/io/__init__.py
-113
View File
@@ -1,113 +0,0 @@
# -*- encoding:utf-8 -*-
"""
==================================
Input and output (:mod:`scipy.io`)
==================================
.. currentmodule:: scipy.io
SciPy has many modules, classes, and functions available to read data
from and write data to a variety of file formats.
.. seealso:: :ref:`numpy-reference.routines.io` (in Numpy)
MATLAB® files
=============
.. autosummary::
:toctree: generated/
loadmat - Read a MATLAB style mat file (version 4 through 7.1)
savemat - Write a MATLAB style mat file (version 4 through 7.1)
whosmat - List contents of a MATLAB style mat file (version 4 through 7.1)
IDL® files
==========
.. autosummary::
:toctree: generated/
readsav - Read an IDL 'save' file
Matrix Market files
===================
.. autosummary::
:toctree: generated/
mminfo - Query matrix info from Matrix Market formatted file
mmread - Read matrix from Matrix Market formatted file
mmwrite - Write matrix to Matrix Market formatted file
Unformatted Fortran files
===============================
.. autosummary::
:toctree: generated/
FortranFile - A file object for unformatted sequential Fortran files
Netcdf
======
.. autosummary::
:toctree: generated/
netcdf_file - A file object for NetCDF data
netcdf_variable - A data object for the netcdf module
Harwell-Boeing files
====================
.. autosummary::
:toctree: generated/
hb_read -- read H-B file
hb_write -- write H-B file
Wav sound files (:mod:`scipy.io.wavfile`)
=========================================
.. module:: scipy.io.wavfile
.. autosummary::
:toctree: generated/
read
write
WavFileWarning
Arff files (:mod:`scipy.io.arff`)
=================================
.. module:: scipy.io.arff
.. autosummary::
:toctree: generated/
loadarff
MetaData
ArffError
ParseArffError
"""
from __future__ import division, print_function, absolute_import
# matfile read and write
from .matlab import loadmat, savemat, whosmat, byteordercodes
# netCDF file support
from .netcdf import netcdf_file, netcdf_variable
# Fortran file support
from ._fortran import FortranFile
from .mmio import mminfo, mmread, mmwrite
from .idl import readsav
from .harwell_boeing import hb_read, hb_write
__all__ = [s for s in dir() if not s.startswith('_')]
from scipy._lib._testutils import PytestTester
test = PytestTester(__name__)
del PytestTester
utils/venv/lib/python3.6/site-packages/scipy/io/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/__pycache__/_fortran.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/__pycache__/idl.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/__pycache__/mmio.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/__pycache__/netcdf.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/__pycache__/setup.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/__pycache__/wavfile.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/_fortran.py
-317
View File
@@ -1,317 +0,0 @@
"""
Module to read / write Fortran unformatted sequential files.
This is in the spirit of code written by Neil Martinsen-Burrell and Joe Zuntz.
"""
from __future__ import division, print_function, absolute_import
import warnings
import numpy as np
__all__ = ['FortranFile']
class FortranFile(object):
"""
A file object for unformatted sequential files from Fortran code.
Parameters
----------
filename : file or str
Open file object or filename.
mode : {'r', 'w'}, optional
Read-write mode, default is 'r'.
header_dtype : dtype, optional
Data type of the header. Size and endiness must match the input/output file.
Notes
-----
These files are broken up into records of unspecified types. The size of
each record is given at the start (although the size of this header is not
standard) and the data is written onto disk without any formatting. Fortran
compilers supporting the BACKSPACE statement will write a second copy of
the size to facilitate backwards seeking.
This class only supports files written with both sizes for the record.
It also does not support the subrecords used in Intel and gfortran compilers
for records which are greater than 2GB with a 4-byte header.
An example of an unformatted sequential file in Fortran would be written as::
OPEN(1, FILE=myfilename, FORM='unformatted')
WRITE(1) myvariable
Since this is a non-standard file format, whose contents depend on the
compiler and the endianness of the machine, caution is advised. Files from
gfortran 4.8.0 and gfortran 4.1.2 on x86_64 are known to work.
Consider using Fortran direct-access files or files from the newer Stream
I/O, which can be easily read by `numpy.fromfile`.
Examples
--------
To create an unformatted sequential Fortran file:
>>> from scipy.io import FortranFile
>>> f = FortranFile('test.unf', 'w')
>>> f.write_record(np.array([1,2,3,4,5], dtype=np.int32))
>>> f.write_record(np.linspace(0,1,20).reshape((5,4)).T)
>>> f.close()
To read this file:
>>> f = FortranFile('test.unf', 'r')
>>> print(f.read_ints(np.int32))
[1 2 3 4 5]
>>> print(f.read_reals(float).reshape((5,4), order="F"))
[[0. 0.05263158 0.10526316 0.15789474]
[0.21052632 0.26315789 0.31578947 0.36842105]
[0.42105263 0.47368421 0.52631579 0.57894737]
[0.63157895 0.68421053 0.73684211 0.78947368]
[0.84210526 0.89473684 0.94736842 1. ]]
>>> f.close()
Or, in Fortran::
integer :: a(5), i
double precision :: b(5,4)
open(1, file='test.unf', form='unformatted')
read(1) a
read(1) b
close(1)
write(*,*) a
do i = 1, 5
write(*,*) b(i,:)
end do
"""
def __init__(self, filename, mode='r', header_dtype=np.uint32):
if header_dtype is None:
raise ValueError('Must specify dtype')
header_dtype = np.dtype(header_dtype)
if header_dtype.kind != 'u':
warnings.warn("Given a dtype which is not unsigned.")
if mode not in 'rw' or len(mode) != 1:
raise ValueError('mode must be either r or w')
if hasattr(filename, 'seek'):
self._fp = filename
else:
self._fp = open(filename, '%sb' % mode)
self._header_dtype = header_dtype
def _read_size(self):
return int(np.fromfile(self._fp, dtype=self._header_dtype, count=1))
def write_record(self, *items):
"""
Write a record (including sizes) to the file.
Parameters
----------
*items : array_like
The data arrays to write.
Notes
-----
Writes data items to a file::
write_record(a.T, b.T, c.T, ...)
write(1) a, b, c, ...
Note that data in multidimensional arrays is written in
row-major order --- to make them read correctly by Fortran
programs, you need to transpose the arrays yourself when
writing them.
"""
items = tuple(np.asarray(item) for item in items)
total_size = sum(item.nbytes for item in items)
nb = np.array([total_size], dtype=self._header_dtype)
nb.tofile(self._fp)
for item in items:
item.tofile(self._fp)
nb.tofile(self._fp)
def read_record(self, *dtypes, **kwargs):
"""
Reads a record of a given type from the file.
Parameters
----------
*dtypes : dtypes, optional
Data type(s) specifying the size and endiness of the data.
Returns
-------
data : ndarray
A one-dimensional array object.
Notes
-----
If the record contains a multi-dimensional array, you can specify
the size in the dtype. For example::
INTEGER var(5,4)
can be read with::
read_record('(4,5)i4').T
Note that this function does **not** assume the file data is in Fortran
column major order, so you need to (i) swap the order of dimensions
when reading and (ii) transpose the resulting array.
Alternatively, you can read the data as a 1D array and handle the
ordering yourself. For example::
read_record('i4').reshape(5, 4, order='F')
For records that contain several variables or mixed types (as opposed
to single scalar or array types), give them as separate arguments::
double precision :: a
integer :: b
write(1) a, b
record = f.read_record('<f4', '<i4')
a = record[0] # first number
b = record[1] # second number
and if any of the variables are arrays, the shape can be specified as
the third item in the relevant dtype::
double precision :: a
integer :: b(3,4)
write(1) a, b
record = f.read_record('<f4', np.dtype(('<i4', (4, 3))))
a = record[0]
b = record[1].T
Numpy also supports a short syntax for this kind of type::
record = f.read_record('<f4', '(3,3)<i4')
See Also
--------
read_reals
read_ints
"""
dtype = kwargs.pop('dtype', None)
if kwargs:
raise ValueError("Unknown keyword arguments {}".format(tuple(kwargs.keys())))
if dtype is not None:
dtypes = dtypes + (dtype,)
elif not dtypes:
raise ValueError('Must specify at least one dtype')
first_size = self._read_size()
dtypes = tuple(np.dtype(dtype) for dtype in dtypes)
block_size = sum(dtype.itemsize for dtype in dtypes)
num_blocks, remainder = divmod(first_size, block_size)
if remainder != 0:
raise ValueError('Size obtained ({0}) is not a multiple of the '
'dtypes given ({1}).'.format(first_size, block_size))
if len(dtypes) != 1 and first_size != block_size:
# Fortran does not write mixed type array items in interleaved order,
# and it's not possible to guess the sizes of the arrays that were written.
# The user must specify the exact sizes of each of the arrays.
raise ValueError('Size obtained ({0}) does not match with the expected '
'size ({1}) of multi-item record'.format(first_size, block_size))
data = []
for dtype in dtypes:
r = np.fromfile(self._fp, dtype=dtype, count=num_blocks)
if dtype.shape != ():
# Squeeze outmost block dimension for array items
if num_blocks == 1:
assert r.shape == (1,) + dtype.shape
r = r[0]
data.append(r)
second_size = self._read_size()
if first_size != second_size:
raise IOError('Sizes do not agree in the header and footer for '
'this record - check header dtype')
# Unpack result
if len(dtypes) == 1:
return data[0]
else:
return tuple(data)
def read_ints(self, dtype='i4'):
"""
Reads a record of a given type from the file, defaulting to an integer
type (``INTEGER*4`` in Fortran).
Parameters
----------
dtype : dtype, optional
Data type specifying the size and endiness of the data.
Returns
-------
data : ndarray
A one-dimensional array object.
See Also
--------
read_reals
read_record
"""
return self.read_record(dtype)
def read_reals(self, dtype='f8'):
"""
Reads a record of a given type from the file, defaulting to a floating
point number (``real*8`` in Fortran).
Parameters
----------
dtype : dtype, optional
Data type specifying the size and endiness of the data.
Returns
-------
data : ndarray
A one-dimensional array object.
See Also
--------
read_ints
read_record
"""
return self.read_record(dtype)
def close(self):
"""
Closes the file. It is unsupported to call any other methods off this
object after closing it. Note that this class supports the 'with'
statement in modern versions of Python, to call this automatically
"""
self._fp.close()
def __enter__(self):
return self
def __exit__(self, type, value, tb):
self.close()
utils/venv/lib/python3.6/site-packages/scipy/io/_test_fortran.cpython-36m-x86_64-linux-gnu.so
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/arff/__init__.py
-26
View File
@@ -1,26 +0,0 @@
"""
Module to read ARFF files, which are the standard data format for WEKA.
ARFF is a text file format which support numerical, string and data values.
The format can also represent missing data and sparse data.
Notes
-----
The ARFF support in ``scipy.io`` provides file reading functionality only.
For more extensive ARFF functionality, see `liac-arff
<https://github.com/renatopp/liac-arff>`_.
See the `WEKA website <http://weka.wikispaces.com/ARFF>`_
for more details about the ARFF format and available datasets.
"""
from __future__ import division, print_function, absolute_import
from .arffread import *
from . import arffread
__all__ = arffread.__all__
from scipy._lib._testutils import PytestTester
test = PytestTester(__name__)
del PytestTester
utils/venv/lib/python3.6/site-packages/scipy/io/arff/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/arff/__pycache__/arffread.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/arff/__pycache__/setup.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/arff/arffread.py
-670
View File
@@ -1,670 +0,0 @@
# Last Change: Mon Aug 20 08:00 PM 2007 J
from __future__ import division, print_function, absolute_import
import re
import itertools
import datetime
from functools import partial
import numpy as np
from scipy._lib.six import next
"""A module to read arff files."""
__all__ = ['MetaData', 'loadarff', 'ArffError', 'ParseArffError']
# An Arff file is basically two parts:
# - header
# - data
#
# A header has each of its components starting by @META where META is one of
# the keyword (attribute of relation, for now).
# TODO:
# - both integer and reals are treated as numeric -> the integer info
# is lost!
# - Replace ValueError by ParseError or something
# We know can handle the following:
# - numeric and nominal attributes
# - missing values for numeric attributes
r_meta = re.compile(r'^\s*@')
# Match a comment
r_comment = re.compile(r'^%')
# Match an empty line
r_empty = re.compile(r'^\s+$')
# Match a header line, that is a line which starts by @ + a word
r_headerline = re.compile(r'^@\S*')
r_datameta = re.compile(r'^@[Dd][Aa][Tt][Aa]')
r_relation = re.compile(r'^@[Rr][Ee][Ll][Aa][Tt][Ii][Oo][Nn]\s*(\S*)')
r_attribute = re.compile(r'^@[Aa][Tt][Tt][Rr][Ii][Bb][Uu][Tt][Ee]\s*(..*$)')
# To get attributes name enclosed with ''
r_comattrval = re.compile(r"'(..+)'\s+(..+$)")
# To get normal attributes
r_wcomattrval = re.compile(r"(\S+)\s+(..+$)")
#-------------------------
# Module defined exception
#-------------------------
class ArffError(IOError):
pass
class ParseArffError(ArffError):
pass
#------------------
# Various utilities
#------------------
# An attribute is defined as @attribute name value
def parse_type(attrtype):
"""Given an arff attribute value (meta data), returns its type.
Expect the value to be a name."""
uattribute = attrtype.lower().strip()
if uattribute[0] == '{':
return 'nominal'
elif uattribute[:len('real')] == 'real':
return 'numeric'
elif uattribute[:len('integer')] == 'integer':
return 'numeric'
elif uattribute[:len('numeric')] == 'numeric':
return 'numeric'
elif uattribute[:len('string')] == 'string':
return 'string'
elif uattribute[:len('relational')] == 'relational':
return 'relational'
elif uattribute[:len('date')] == 'date':
return 'date'
else:
raise ParseArffError("unknown attribute %s" % uattribute)
def get_nominal(attribute):
"""If attribute is nominal, returns a list of the values"""
return attribute.split(',')
def read_data_list(ofile):
"""Read each line of the iterable and put it in a list."""
data = [next(ofile)]
if data[0].strip()[0] == '{':
raise ValueError("This looks like a sparse ARFF: not supported yet")
data.extend([i for i in ofile])
return data
def get_ndata(ofile):
"""Read the whole file to get number of data attributes."""
data = [next(ofile)]
loc = 1
if data[0].strip()[0] == '{':
raise ValueError("This looks like a sparse ARFF: not supported yet")
for i in ofile:
loc += 1
return loc
def maxnomlen(atrv):
"""Given a string containing a nominal type definition, returns the
string len of the biggest component.
A nominal type is defined as seomthing framed between brace ({}).
Parameters
----------
atrv : str
Nominal type definition
Returns
-------
slen : int
length of longest component
Examples
--------
maxnomlen("{floup, bouga, fl, ratata}") returns 6 (the size of
ratata, the longest nominal value).
>>> maxnomlen("{floup, bouga, fl, ratata}")
6
"""
nomtp = get_nom_val(atrv)
return max(len(i) for i in nomtp)
def get_nom_val(atrv):
"""Given a string containing a nominal type, returns a tuple of the
possible values.
A nominal type is defined as something framed between braces ({}).
Parameters
----------
atrv : str
Nominal type definition
Returns
-------
poss_vals : tuple
possible values
Examples
--------
>>> get_nom_val("{floup, bouga, fl, ratata}")
('floup', 'bouga', 'fl', 'ratata')
"""
r_nominal = re.compile('{(.+)}')
m = r_nominal.match(atrv)
if m:
return tuple(i.strip() for i in m.group(1).split(','))
else:
raise ValueError("This does not look like a nominal string")
def get_date_format(atrv):
r_date = re.compile(r"[Dd][Aa][Tt][Ee]\s+[\"']?(.+?)[\"']?$")
m = r_date.match(atrv)
if m:
pattern = m.group(1).strip()
# convert time pattern from Java's SimpleDateFormat to C's format
datetime_unit = None
if "yyyy" in pattern:
pattern = pattern.replace("yyyy", "%Y")
datetime_unit = "Y"
elif "yy":
pattern = pattern.replace("yy", "%y")
datetime_unit = "Y"
if "MM" in pattern:
pattern = pattern.replace("MM", "%m")
datetime_unit = "M"
if "dd" in pattern:
pattern = pattern.replace("dd", "%d")
datetime_unit = "D"
if "HH" in pattern:
pattern = pattern.replace("HH", "%H")
datetime_unit = "h"
if "mm" in pattern:
pattern = pattern.replace("mm", "%M")
datetime_unit = "m"
if "ss" in pattern:
pattern = pattern.replace("ss", "%S")
datetime_unit = "s"
if "z" in pattern or "Z" in pattern:
raise ValueError("Date type attributes with time zone not "
"supported, yet")
if datetime_unit is None:
raise ValueError("Invalid or unsupported date format")
return pattern, datetime_unit
else:
raise ValueError("Invalid or no date format")
def go_data(ofile):
"""Skip header.
the first next() call of the returned iterator will be the @data line"""
return itertools.dropwhile(lambda x: not r_datameta.match(x), ofile)
#----------------
# Parsing header
#----------------
def tokenize_attribute(iterable, attribute):
"""Parse a raw string in header (eg starts by @attribute).
Given a raw string attribute, try to get the name and type of the
attribute. Constraints:
* The first line must start with @attribute (case insensitive, and
space like characters before @attribute are allowed)
* Works also if the attribute is spread on multilines.
* Works if empty lines or comments are in between
Parameters
----------
attribute : str
the attribute string.
Returns
-------
name : str
name of the attribute
value : str
value of the attribute
next : str
next line to be parsed
Examples
--------
If attribute is a string defined in python as r"floupi real", will
return floupi as name, and real as value.
>>> iterable = iter([0] * 10) # dummy iterator
>>> tokenize_attribute(iterable, r"@attribute floupi real")
('floupi', 'real', 0)
If attribute is r"'floupi 2' real", will return 'floupi 2' as name,
and real as value.
>>> tokenize_attribute(iterable, r" @attribute 'floupi 2' real ")
('floupi 2', 'real', 0)
"""
sattr = attribute.strip()
mattr = r_attribute.match(sattr)
if mattr:
# atrv is everything after @attribute
atrv = mattr.group(1)
if r_comattrval.match(atrv):
name, type = tokenize_single_comma(atrv)
next_item = next(iterable)
elif r_wcomattrval.match(atrv):
name, type = tokenize_single_wcomma(atrv)
next_item = next(iterable)
else:
# Not sure we should support this, as it does not seem supported by
# weka.
raise ValueError("multi line not supported yet")
#name, type, next_item = tokenize_multilines(iterable, atrv)
else:
raise ValueError("First line unparsable: %s" % sattr)
if type == 'relational':
raise ValueError("relational attributes not supported yet")
return name, type, next_item
def tokenize_single_comma(val):
# XXX we match twice the same string (here and at the caller level). It is
# stupid, but it is easier for now...
m = r_comattrval.match(val)
if m:
try:
name = m.group(1).strip()
type = m.group(2).strip()
except IndexError:
raise ValueError("Error while tokenizing attribute")
else:
raise ValueError("Error while tokenizing single %s" % val)
return name, type
def tokenize_single_wcomma(val):
# XXX we match twice the same string (here and at the caller level). It is
# stupid, but it is easier for now...
m = r_wcomattrval.match(val)
if m:
try:
name = m.group(1).strip()
type = m.group(2).strip()
except IndexError:
raise ValueError("Error while tokenizing attribute")
else:
raise ValueError("Error while tokenizing single %s" % val)
return name, type
def read_header(ofile):
"""Read the header of the iterable ofile."""
i = next(ofile)
# Pass first comments
while r_comment.match(i):
i = next(ofile)
# Header is everything up to DATA attribute ?
relation = None
attributes = []
while not r_datameta.match(i):
m = r_headerline.match(i)
if m:
isattr = r_attribute.match(i)
if isattr:
name, type, i = tokenize_attribute(ofile, i)
attributes.append((name, type))
else:
isrel = r_relation.match(i)
if isrel:
relation = isrel.group(1)
else:
raise ValueError("Error parsing line %s" % i)
i = next(ofile)
else:
i = next(ofile)
return relation, attributes
#--------------------
# Parsing actual data
#--------------------
def safe_float(x):
"""given a string x, convert it to a float. If the stripped string is a ?,
return a Nan (missing value).
Parameters
----------
x : str
string to convert
Returns
-------
f : float
where float can be nan
Examples
--------
>>> safe_float('1')
1.0
>>> safe_float('1\\n')
1.0
>>> safe_float('?\\n')
nan
"""
if '?' in x:
return np.nan
else:
return float(x)
def safe_nominal(value, pvalue):
svalue = value.strip()
if svalue in pvalue:
return svalue
elif svalue == '?':
return svalue
else:
raise ValueError("%s value not in %s" % (str(svalue), str(pvalue)))
def safe_date(value, date_format, datetime_unit):
date_str = value.strip().strip("'").strip('"')
if date_str == '?':
return np.datetime64('NaT', datetime_unit)
else:
dt = datetime.datetime.strptime(date_str, date_format)
return np.datetime64(dt).astype("datetime64[%s]" % datetime_unit)
class MetaData(object):
"""Small container to keep useful information on a ARFF dataset.
Knows about attributes names and types.
Examples
--------
::
data, meta = loadarff('iris.arff')
# This will print the attributes names of the iris.arff dataset
for i in meta:
print(i)
# This works too
meta.names()
# Getting attribute type
types = meta.types()
Notes
-----
Also maintains the list of attributes in order, i.e. doing for i in
meta, where meta is an instance of MetaData, will return the
different attribute names in the order they were defined.
"""
def __init__(self, rel, attr):
self.name = rel
# We need the dictionary to be ordered
# XXX: may be better to implement an ordered dictionary
self._attributes = {}
self._attrnames = []
for name, value in attr:
tp = parse_type(value)
self._attrnames.append(name)
if tp == 'nominal':
self._attributes[name] = (tp, get_nom_val(value))
elif tp == 'date':
self._attributes[name] = (tp, get_date_format(value)[0])
else:
self._attributes[name] = (tp, None)
def __repr__(self):
msg = ""
msg += "Dataset: %s\n" % self.name
for i in self._attrnames:
msg += "\t%s's type is %s" % (i, self._attributes[i][0])
if self._attributes[i][1]:
msg += ", range is %s" % str(self._attributes[i][1])
msg += '\n'
return msg
def __iter__(self):
return iter(self._attrnames)
def __getitem__(self, key):
return self._attributes[key]
def names(self):
"""Return the list of attribute names."""
return self._attrnames
def types(self):
"""Return the list of attribute types."""
attr_types = [self._attributes[name][0] for name in self._attrnames]
return attr_types
def loadarff(f):
"""
Read an arff file.
The data is returned as a record array, which can be accessed much like
a dictionary of numpy arrays. For example, if one of the attributes is
called 'pressure', then its first 10 data points can be accessed from the
``data`` record array like so: ``data['pressure'][0:10]``
Parameters
----------
f : file-like or str
File-like object to read from, or filename to open.
Returns
-------
data : record array
The data of the arff file, accessible by attribute names.
meta : `MetaData`
Contains information about the arff file such as name and
type of attributes, the relation (name of the dataset), etc...
Raises
------
ParseArffError
This is raised if the given file is not ARFF-formatted.
NotImplementedError
The ARFF file has an attribute which is not supported yet.
Notes
-----
This function should be able to read most arff files. Not
implemented functionality include:
* date type attributes
* string type attributes
It can read files with numeric and nominal attributes. It cannot read
files with sparse data ({} in the file). However, this function can
read files with missing data (? in the file), representing the data
points as NaNs.
Examples
--------
>>> from scipy.io import arff
>>> from io import StringIO
>>> content = \"\"\"
... @relation foo
... @attribute width numeric
... @attribute height numeric
... @attribute color {red,green,blue,yellow,black}
... @data
... 5.0,3.25,blue
... 4.5,3.75,green
... 3.0,4.00,red
... \"\"\"
>>> f = StringIO(content)
>>> data, meta = arff.loadarff(f)
>>> data
array([(5.0, 3.25, 'blue'), (4.5, 3.75, 'green'), (3.0, 4.0, 'red')],
dtype=[('width', '<f8'), ('height', '<f8'), ('color', '|S6')])
>>> meta
Dataset: foo
\twidth's type is numeric
\theight's type is numeric
\tcolor's type is nominal, range is ('red', 'green', 'blue', 'yellow', 'black')
"""
if hasattr(f, 'read'):
ofile = f
else:
ofile = open(f, 'rt')
try:
return _loadarff(ofile)
finally:
if ofile is not f: # only close what we opened
ofile.close()
def _loadarff(ofile):
# Parse the header file
try:
rel, attr = read_header(ofile)
except ValueError as e:
msg = "Error while parsing header, error was: " + str(e)
raise ParseArffError(msg)
# Check whether we have a string attribute (not supported yet)
hasstr = False
for name, value in attr:
type = parse_type(value)
if type == 'string':
hasstr = True
meta = MetaData(rel, attr)
# XXX The following code is not great
# Build the type descriptor descr and the list of convertors to convert
# each attribute to the suitable type (which should match the one in
# descr).
# This can be used once we want to support integer as integer values and
# not as numeric anymore (using masked arrays ?).
acls2dtype = {'real': float, 'integer': float, 'numeric': float}
acls2conv = {'real': safe_float,
'integer': safe_float,
'numeric': safe_float}
descr = []
convertors = []
if not hasstr:
for name, value in attr:
type = parse_type(value)
if type == 'date':
date_format, datetime_unit = get_date_format(value)
descr.append((name, "datetime64[%s]" % datetime_unit))
convertors.append(partial(safe_date, date_format=date_format,
datetime_unit=datetime_unit))
elif type == 'nominal':
n = maxnomlen(value)
descr.append((name, 'S%d' % n))
pvalue = get_nom_val(value)
convertors.append(partial(safe_nominal, pvalue=pvalue))
else:
descr.append((name, acls2dtype[type]))
convertors.append(safe_float)
#dc.append(acls2conv[type])
#sdescr.append((name, acls2sdtype[type]))
else:
# How to support string efficiently ? Ideally, we should know the max
# size of the string before allocating the numpy array.
raise NotImplementedError("String attributes not supported yet, sorry")
ni = len(convertors)
def generator(row_iter, delim=','):
# TODO: this is where we are spending times (~80%). I think things
# could be made more efficiently:
# - We could for example "compile" the function, because some values
# do not change here.
# - The function to convert a line to dtyped values could also be
# generated on the fly from a string and be executed instead of
# looping.
# - The regex are overkill: for comments, checking that a line starts
# by % should be enough and faster, and for empty lines, same thing
# --> this does not seem to change anything.
# 'compiling' the range since it does not change
# Note, I have already tried zipping the converters and
# row elements and got slightly worse performance.
elems = list(range(ni))
for raw in row_iter:
# We do not abstract skipping comments and empty lines for
# performance reasons.
if r_comment.match(raw) or r_empty.match(raw):
continue
row = raw.split(delim)
yield tuple([convertors[i](row[i]) for i in elems])
a = generator(ofile)
# No error should happen here: it is a bug otherwise
data = np.fromiter(a, descr)
return data, meta
#-----
# Misc
#-----
def basic_stats(data):
nbfac = data.size * 1. / (data.size - 1)
return np.nanmin(data), np.nanmax(data), np.mean(data), np.std(data) * nbfac
def print_attribute(name, tp, data):
type = tp[0]
if type == 'numeric' or type == 'real' or type == 'integer':
min, max, mean, std = basic_stats(data)
print("%s,%s,%f,%f,%f,%f" % (name, type, min, max, mean, std))
else:
msg = name + ",{"
for i in range(len(tp[1])-1):
msg += tp[1][i] + ","
msg += tp[1][-1]
msg += "}"
print(msg)
def test_weka(filename):
data, meta = loadarff(filename)
print(len(data.dtype))
print(data.size)
for i in meta:
print_attribute(i, meta[i], data[i])
# make sure nose does not find this as a test
test_weka.__test__ = False
if __name__ == '__main__':
import sys
filename = sys.argv[1]
test_weka(filename)
utils/venv/lib/python3.6/site-packages/scipy/io/arff/setup.py
-13
View File
@@ -1,13 +0,0 @@
from __future__ import division, print_function, absolute_import
def configuration(parent_package='io',top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('arff', parent_package, top_path)
config.add_data_dir('tests')
return config
if __name__ == '__main__':
from numpy.distutils.core import setup
setup(**configuration(top_path='').todict())
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/__init__.py
View File
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/__pycache__/test_arffread.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/iris.arff
-225
View File
@@ -1,225 +0,0 @@
% 1. Title: Iris Plants Database
%
% 2. Sources:
% (a) Creator: R.A. Fisher
% (b) Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov)
% (c) Date: July, 1988
%
% 3. Past Usage:
% - Publications: too many to mention!!! Here are a few.
% 1. Fisher,R.A. "The use of multiple measurements in taxonomic problems"
% Annual Eugenics, 7, Part II, 179-188 (1936); also in "Contributions
% to Mathematical Statistics" (John Wiley, NY, 1950).
% 2. Duda,R.O., & Hart,P.E. (1973) Pattern Classification and Scene Analysis.
% (Q327.D83) John Wiley & Sons. ISBN 0-471-22361-1. See page 218.
% 3. Dasarathy, B.V. (1980) "Nosing Around the Neighborhood: A New System
% Structure and Classification Rule for Recognition in Partially Exposed
% Environments". IEEE Transactions on Pattern Analysis and Machine
% Intelligence, Vol. PAMI-2, No. 1, 67-71.
% -- Results:
% -- very low misclassification rates (0% for the setosa class)
% 4. Gates, G.W. (1972) "The Reduced Nearest Neighbor Rule". IEEE
% Transactions on Information Theory, May 1972, 431-433.
% -- Results:
% -- very low misclassification rates again
% 5. See also: 1988 MLC Proceedings, 54-64. Cheeseman et al's AUTOCLASS II
% conceptual clustering system finds 3 classes in the data.
%
% 4. Relevant Information:
% --- This is perhaps the best known database to be found in the pattern
% recognition literature. Fisher's paper is a classic in the field
% and is referenced frequently to this day. (See Duda & Hart, for
% example.) The data set contains 3 classes of 50 instances each,
% where each class refers to a type of iris plant. One class is
% linearly separable from the other 2; the latter are NOT linearly
% separable from each other.
% --- Predicted attribute: class of iris plant.
% --- This is an exceedingly simple domain.
%
% 5. Number of Instances: 150 (50 in each of three classes)
%
% 6. Number of Attributes: 4 numeric, predictive attributes and the class
%
% 7. Attribute Information:
% 1. sepal length in cm
% 2. sepal width in cm
% 3. petal length in cm
% 4. petal width in cm
% 5. class:
% -- Iris Setosa
% -- Iris Versicolour
% -- Iris Virginica
%
% 8. Missing Attribute Values: None
%
% Summary Statistics:
% Min Max Mean SD Class Correlation
% sepal length: 4.3 7.9 5.84 0.83 0.7826
% sepal width: 2.0 4.4 3.05 0.43 -0.4194
% petal length: 1.0 6.9 3.76 1.76 0.9490 (high!)
% petal width: 0.1 2.5 1.20 0.76 0.9565 (high!)
%
% 9. Class Distribution: 33.3% for each of 3 classes.
@RELATION iris
@ATTRIBUTE sepallength REAL
@ATTRIBUTE sepalwidth REAL
@ATTRIBUTE petallength REAL
@ATTRIBUTE petalwidth REAL
@ATTRIBUTE class {Iris-setosa,Iris-versicolor,Iris-virginica}
@DATA
5.1,3.5,1.4,0.2,Iris-setosa
4.9,3.0,1.4,0.2,Iris-setosa
4.7,3.2,1.3,0.2,Iris-setosa
4.6,3.1,1.5,0.2,Iris-setosa
5.0,3.6,1.4,0.2,Iris-setosa
5.4,3.9,1.7,0.4,Iris-setosa
4.6,3.4,1.4,0.3,Iris-setosa
5.0,3.4,1.5,0.2,Iris-setosa
4.4,2.9,1.4,0.2,Iris-setosa
4.9,3.1,1.5,0.1,Iris-setosa
5.4,3.7,1.5,0.2,Iris-setosa
4.8,3.4,1.6,0.2,Iris-setosa
4.8,3.0,1.4,0.1,Iris-setosa
4.3,3.0,1.1,0.1,Iris-setosa
5.8,4.0,1.2,0.2,Iris-setosa
5.7,4.4,1.5,0.4,Iris-setosa
5.4,3.9,1.3,0.4,Iris-setosa
5.1,3.5,1.4,0.3,Iris-setosa
5.7,3.8,1.7,0.3,Iris-setosa
5.1,3.8,1.5,0.3,Iris-setosa
5.4,3.4,1.7,0.2,Iris-setosa
5.1,3.7,1.5,0.4,Iris-setosa
4.6,3.6,1.0,0.2,Iris-setosa
5.1,3.3,1.7,0.5,Iris-setosa
4.8,3.4,1.9,0.2,Iris-setosa
5.0,3.0,1.6,0.2,Iris-setosa
5.0,3.4,1.6,0.4,Iris-setosa
5.2,3.5,1.5,0.2,Iris-setosa
5.2,3.4,1.4,0.2,Iris-setosa
4.7,3.2,1.6,0.2,Iris-setosa
4.8,3.1,1.6,0.2,Iris-setosa
5.4,3.4,1.5,0.4,Iris-setosa
5.2,4.1,1.5,0.1,Iris-setosa
5.5,4.2,1.4,0.2,Iris-setosa
4.9,3.1,1.5,0.1,Iris-setosa
5.0,3.2,1.2,0.2,Iris-setosa
5.5,3.5,1.3,0.2,Iris-setosa
4.9,3.1,1.5,0.1,Iris-setosa
4.4,3.0,1.3,0.2,Iris-setosa
5.1,3.4,1.5,0.2,Iris-setosa
5.0,3.5,1.3,0.3,Iris-setosa
4.5,2.3,1.3,0.3,Iris-setosa
4.4,3.2,1.3,0.2,Iris-setosa
5.0,3.5,1.6,0.6,Iris-setosa
5.1,3.8,1.9,0.4,Iris-setosa
4.8,3.0,1.4,0.3,Iris-setosa
5.1,3.8,1.6,0.2,Iris-setosa
4.6,3.2,1.4,0.2,Iris-setosa
5.3,3.7,1.5,0.2,Iris-setosa
5.0,3.3,1.4,0.2,Iris-setosa
7.0,3.2,4.7,1.4,Iris-versicolor
6.4,3.2,4.5,1.5,Iris-versicolor
6.9,3.1,4.9,1.5,Iris-versicolor
5.5,2.3,4.0,1.3,Iris-versicolor
6.5,2.8,4.6,1.5,Iris-versicolor
5.7,2.8,4.5,1.3,Iris-versicolor
6.3,3.3,4.7,1.6,Iris-versicolor
4.9,2.4,3.3,1.0,Iris-versicolor
6.6,2.9,4.6,1.3,Iris-versicolor
5.2,2.7,3.9,1.4,Iris-versicolor
5.0,2.0,3.5,1.0,Iris-versicolor
5.9,3.0,4.2,1.5,Iris-versicolor
6.0,2.2,4.0,1.0,Iris-versicolor
6.1,2.9,4.7,1.4,Iris-versicolor
5.6,2.9,3.6,1.3,Iris-versicolor
6.7,3.1,4.4,1.4,Iris-versicolor
5.6,3.0,4.5,1.5,Iris-versicolor
5.8,2.7,4.1,1.0,Iris-versicolor
6.2,2.2,4.5,1.5,Iris-versicolor
5.6,2.5,3.9,1.1,Iris-versicolor
5.9,3.2,4.8,1.8,Iris-versicolor
6.1,2.8,4.0,1.3,Iris-versicolor
6.3,2.5,4.9,1.5,Iris-versicolor
6.1,2.8,4.7,1.2,Iris-versicolor
6.4,2.9,4.3,1.3,Iris-versicolor
6.6,3.0,4.4,1.4,Iris-versicolor
6.8,2.8,4.8,1.4,Iris-versicolor
6.7,3.0,5.0,1.7,Iris-versicolor
6.0,2.9,4.5,1.5,Iris-versicolor
5.7,2.6,3.5,1.0,Iris-versicolor
5.5,2.4,3.8,1.1,Iris-versicolor
5.5,2.4,3.7,1.0,Iris-versicolor
5.8,2.7,3.9,1.2,Iris-versicolor
6.0,2.7,5.1,1.6,Iris-versicolor
5.4,3.0,4.5,1.5,Iris-versicolor
6.0,3.4,4.5,1.6,Iris-versicolor
6.7,3.1,4.7,1.5,Iris-versicolor
6.3,2.3,4.4,1.3,Iris-versicolor
5.6,3.0,4.1,1.3,Iris-versicolor
5.5,2.5,4.0,1.3,Iris-versicolor
5.5,2.6,4.4,1.2,Iris-versicolor
6.1,3.0,4.6,1.4,Iris-versicolor
5.8,2.6,4.0,1.2,Iris-versicolor
5.0,2.3,3.3,1.0,Iris-versicolor
5.6,2.7,4.2,1.3,Iris-versicolor
5.7,3.0,4.2,1.2,Iris-versicolor
5.7,2.9,4.2,1.3,Iris-versicolor
6.2,2.9,4.3,1.3,Iris-versicolor
5.1,2.5,3.0,1.1,Iris-versicolor
5.7,2.8,4.1,1.3,Iris-versicolor
6.3,3.3,6.0,2.5,Iris-virginica
5.8,2.7,5.1,1.9,Iris-virginica
7.1,3.0,5.9,2.1,Iris-virginica
6.3,2.9,5.6,1.8,Iris-virginica
6.5,3.0,5.8,2.2,Iris-virginica
7.6,3.0,6.6,2.1,Iris-virginica
4.9,2.5,4.5,1.7,Iris-virginica
7.3,2.9,6.3,1.8,Iris-virginica
6.7,2.5,5.8,1.8,Iris-virginica
7.2,3.6,6.1,2.5,Iris-virginica
6.5,3.2,5.1,2.0,Iris-virginica
6.4,2.7,5.3,1.9,Iris-virginica
6.8,3.0,5.5,2.1,Iris-virginica
5.7,2.5,5.0,2.0,Iris-virginica
5.8,2.8,5.1,2.4,Iris-virginica
6.4,3.2,5.3,2.3,Iris-virginica
6.5,3.0,5.5,1.8,Iris-virginica
7.7,3.8,6.7,2.2,Iris-virginica
7.7,2.6,6.9,2.3,Iris-virginica
6.0,2.2,5.0,1.5,Iris-virginica
6.9,3.2,5.7,2.3,Iris-virginica
5.6,2.8,4.9,2.0,Iris-virginica
7.7,2.8,6.7,2.0,Iris-virginica
6.3,2.7,4.9,1.8,Iris-virginica
6.7,3.3,5.7,2.1,Iris-virginica
7.2,3.2,6.0,1.8,Iris-virginica
6.2,2.8,4.8,1.8,Iris-virginica
6.1,3.0,4.9,1.8,Iris-virginica
6.4,2.8,5.6,2.1,Iris-virginica
7.2,3.0,5.8,1.6,Iris-virginica
7.4,2.8,6.1,1.9,Iris-virginica
7.9,3.8,6.4,2.0,Iris-virginica
6.4,2.8,5.6,2.2,Iris-virginica
6.3,2.8,5.1,1.5,Iris-virginica
6.1,2.6,5.6,1.4,Iris-virginica
7.7,3.0,6.1,2.3,Iris-virginica
6.3,3.4,5.6,2.4,Iris-virginica
6.4,3.1,5.5,1.8,Iris-virginica
6.0,3.0,4.8,1.8,Iris-virginica
6.9,3.1,5.4,2.1,Iris-virginica
6.7,3.1,5.6,2.4,Iris-virginica
6.9,3.1,5.1,2.3,Iris-virginica
5.8,2.7,5.1,1.9,Iris-virginica
6.8,3.2,5.9,2.3,Iris-virginica
6.7,3.3,5.7,2.5,Iris-virginica
6.7,3.0,5.2,2.3,Iris-virginica
6.3,2.5,5.0,1.9,Iris-virginica
6.5,3.0,5.2,2.0,Iris-virginica
6.2,3.4,5.4,2.3,Iris-virginica
5.9,3.0,5.1,1.8,Iris-virginica
%
%
%
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/missing.arff
-8
View File
@@ -1,8 +0,0 @@
% This arff file contains some missing data
@relation missing
@attribute yop real
@attribute yap real
@data
1,5
2,4
?,?
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/nodata.arff
-11
View File
@@ -1,11 +0,0 @@
@RELATION iris
@ATTRIBUTE sepallength REAL
@ATTRIBUTE sepalwidth REAL
@ATTRIBUTE petallength REAL
@ATTRIBUTE petalwidth REAL
@ATTRIBUTE class {Iris-setosa,Iris-versicolor,Iris-virginica}
@DATA
% This file has no data
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test1.arff
-10
View File
@@ -1,10 +0,0 @@
@RELATION test1
@ATTRIBUTE attr0 REAL
@ATTRIBUTE attr1 REAL
@ATTRIBUTE attr2 REAL
@ATTRIBUTE attr3 REAL
@ATTRIBUTE class {class0, class1, class2, class3}
@DATA
0.1, 0.2, 0.3, 0.4,class1
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test2.arff
-15
View File
@@ -1,15 +0,0 @@
@RELATION test2
@ATTRIBUTE attr0 REAL
@ATTRIBUTE attr1 real
@ATTRIBUTE attr2 integer
@ATTRIBUTE attr3 Integer
@ATTRIBUTE attr4 Numeric
@ATTRIBUTE attr5 numeric
@ATTRIBUTE attr6 string
@ATTRIBUTE attr7 STRING
@ATTRIBUTE attr8 {bla}
@ATTRIBUTE attr9 {bla, bla}
@DATA
0.1, 0.2, 0.3, 0.4,class1
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test3.arff
-6
View File
@@ -1,6 +0,0 @@
@RELATION test3
@ATTRIBUTE attr0 crap
@DATA
0.1, 0.2, 0.3, 0.4,class1
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test4.arff
-11
View File
@@ -1,11 +0,0 @@
@RELATION test5
@ATTRIBUTE attr0 REAL
@ATTRIBUTE attr1 REAL
@ATTRIBUTE attr2 REAL
@ATTRIBUTE attr3 REAL
@ATTRIBUTE class {class0, class1, class2, class3}
@DATA
0.1, 0.2, 0.3, 0.4,class1
-0.1, -0.2, -0.3, -0.4,class2
1, 2, 3, 4,class3
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test5.arff
-26
View File
@@ -1,26 +0,0 @@
@RELATION test4
@ATTRIBUTE attr0 REAL
@ATTRIBUTE attr1 REAL
@ATTRIBUTE attr2 REAL
@ATTRIBUTE attr3 REAL
@ATTRIBUTE class {class0, class1, class2, class3}
@DATA
% lsdflkjhaksjdhf
% lsdflkjhaksjdhf
0.1, 0.2, 0.3, 0.4,class1
% laksjdhf
% lsdflkjhaksjdhf
-0.1, -0.2, -0.3, -0.4,class2
% lsdflkjhaksjdhf
% lsdflkjhaksjdhf
% lsdflkjhaksjdhf
1, 2, 3, 4,class3
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test6.arff
-12
View File
@@ -1,12 +0,0 @@
@RELATION test6
@ATTRIBUTE attr0 REAL
@ATTRIBUTE attr1 REAL
@ATTRIBUTE attr2 REAL
@ATTRIBUTE attr3 REAL
@ATTRIBUTE class {C}
@DATA
0.1, 0.2, 0.3, 0.4,C
-0.1, -0.2, -0.3, -0.4,C
1, 2, 3, 4,C
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test7.arff
-15
View File
@@ -1,15 +0,0 @@
@RELATION test7
@ATTRIBUTE attr_year DATE yyyy
@ATTRIBUTE attr_month DATE yyyy-MM
@ATTRIBUTE attr_date DATE yyyy-MM-dd
@ATTRIBUTE attr_datetime_local DATE "yyyy-MM-dd HH:mm"
@ATTRIBUTE attr_datetime_missing DATE "yyyy-MM-dd HH:mm"
@DATA
1999,1999-01,1999-01-31,"1999-01-31 00:01",?
2004,2004-12,2004-12-01,"2004-12-01 23:59","2004-12-01 23:59"
1817,1817-04,1817-04-28,"1817-04-28 13:00",?
2100,2100-09,2100-09-10,"2100-09-10 12:00",?
2013,2013-11,2013-11-30,"2013-11-30 04:55","2013-11-30 04:55"
1631,1631-10,1631-10-15,"1631-10-15 20:04","1631-10-15 20:04"
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/data/test8.arff
-12
View File
@@ -1,12 +0,0 @@
@RELATION test8
@ATTRIBUTE attr_datetime_utc DATE "yyyy-MM-dd HH:mm Z"
@ATTRIBUTE attr_datetime_full DATE "yy-MM-dd HH:mm:ss z"
@DATA
"1999-01-31 00:01 UTC","99-01-31 00:01:08 +0430"
"2004-12-01 23:59 UTC","04-12-01 23:59:59 -0800"
"1817-04-28 13:00 UTC","17-04-28 13:00:33 +1000"
"2100-09-10 12:00 UTC","21-09-10 12:00:21 -0300"
"2013-11-30 04:55 UTC","13-11-30 04:55:48 -1100"
"1631-10-15 20:04 UTC","31-10-15 20:04:10 +0000"
utils/venv/lib/python3.6/site-packages/scipy/io/arff/tests/test_arffread.py
-259
View File
@@ -1,259 +0,0 @@
from __future__ import division, print_function, absolute_import
import datetime
import os
import sys
from os.path import join as pjoin
if sys.version_info[0] >= 3:
from io import StringIO
else:
from cStringIO import StringIO
import numpy as np
from numpy.testing import (assert_array_almost_equal,
assert_array_equal, assert_equal, assert_)
import pytest
from pytest import raises as assert_raises
from scipy.io.arff.arffread import loadarff
from scipy.io.arff.arffread import read_header, parse_type, ParseArffError
data_path = pjoin(os.path.dirname(__file__), 'data')
test1 = pjoin(data_path, 'test1.arff')
test2 = pjoin(data_path, 'test2.arff')
test3 = pjoin(data_path, 'test3.arff')
test4 = pjoin(data_path, 'test4.arff')
test5 = pjoin(data_path, 'test5.arff')
test6 = pjoin(data_path, 'test6.arff')
test7 = pjoin(data_path, 'test7.arff')
test8 = pjoin(data_path, 'test8.arff')
expect4_data = [(0.1, 0.2, 0.3, 0.4, 'class1'),
(-0.1, -0.2, -0.3, -0.4, 'class2'),
(1, 2, 3, 4, 'class3')]
expected_types = ['numeric', 'numeric', 'numeric', 'numeric', 'nominal']
missing = pjoin(data_path, 'missing.arff')
expect_missing_raw = np.array([[1, 5], [2, 4], [np.nan, np.nan]])
expect_missing = np.empty(3, [('yop', float), ('yap', float)])
expect_missing['yop'] = expect_missing_raw[:, 0]
expect_missing['yap'] = expect_missing_raw[:, 1]
class TestData(object):
def test1(self):
# Parsing trivial file with nothing.
self._test(test4)
def test2(self):
# Parsing trivial file with some comments in the data section.
self._test(test5)
def test3(self):
# Parsing trivial file with nominal attribute of 1 character.
self._test(test6)
def _test(self, test_file):
data, meta = loadarff(test_file)
for i in range(len(data)):
for j in range(4):
assert_array_almost_equal(expect4_data[i][j], data[i][j])
assert_equal(meta.types(), expected_types)
def test_filelike(self):
# Test reading from file-like object (StringIO)
f1 = open(test1)
data1, meta1 = loadarff(f1)
f1.close()
f2 = open(test1)
data2, meta2 = loadarff(StringIO(f2.read()))
f2.close()
assert_(data1 == data2)
assert_(repr(meta1) == repr(meta2))
@pytest.mark.skipif(sys.version_info < (3, 6),
reason='Passing path-like objects to IO functions requires Python >= 3.6')
def test_path(self):
# Test reading from `pathlib.Path` object
from pathlib import Path
with open(test1) as f1:
data1, meta1 = loadarff(f1)
data2, meta2 = loadarff(Path(test1))
assert_(data1 == data2)
assert_(repr(meta1) == repr(meta2))
class TestMissingData(object):
def test_missing(self):
data, meta = loadarff(missing)
for i in ['yop', 'yap']:
assert_array_almost_equal(data[i], expect_missing[i])
class TestNoData(object):
def test_nodata(self):
# The file nodata.arff has no data in the @DATA section.
# Reading it should result in an array with length 0.
nodata_filename = os.path.join(data_path, 'nodata.arff')
data, meta = loadarff(nodata_filename)
expected_dtype = np.dtype([('sepallength', '<f8'),
('sepalwidth', '<f8'),
('petallength', '<f8'),
('petalwidth', '<f8'),
('class', 'S15')])
assert_equal(data.dtype, expected_dtype)
assert_equal(data.size, 0)
class TestHeader(object):
def test_type_parsing(self):
# Test parsing type of attribute from their value.
ofile = open(test2)
rel, attrs = read_header(ofile)
ofile.close()
expected = ['numeric', 'numeric', 'numeric', 'numeric', 'numeric',
'numeric', 'string', 'string', 'nominal', 'nominal']
for i in range(len(attrs)):
assert_(parse_type(attrs[i][1]) == expected[i])
def test_badtype_parsing(self):
# Test parsing wrong type of attribute from their value.
ofile = open(test3)
rel, attrs = read_header(ofile)
ofile.close()
for name, value in attrs:
assert_raises(ParseArffError, parse_type, value)
def test_fullheader1(self):
# Parsing trivial header with nothing.
ofile = open(test1)
rel, attrs = read_header(ofile)
ofile.close()
# Test relation
assert_(rel == 'test1')
# Test numerical attributes
assert_(len(attrs) == 5)
for i in range(4):
assert_(attrs[i][0] == 'attr%d' % i)
assert_(attrs[i][1] == 'REAL')
# Test nominal attribute
assert_(attrs[4][0] == 'class')
assert_(attrs[4][1] == '{class0, class1, class2, class3}')
def test_dateheader(self):
ofile = open(test7)
rel, attrs = read_header(ofile)
ofile.close()
assert_(rel == 'test7')
assert_(len(attrs) == 5)
assert_(attrs[0][0] == 'attr_year')
assert_(attrs[0][1] == 'DATE yyyy')
assert_(attrs[1][0] == 'attr_month')
assert_(attrs[1][1] == 'DATE yyyy-MM')
assert_(attrs[2][0] == 'attr_date')
assert_(attrs[2][1] == 'DATE yyyy-MM-dd')
assert_(attrs[3][0] == 'attr_datetime_local')
assert_(attrs[3][1] == 'DATE "yyyy-MM-dd HH:mm"')
assert_(attrs[4][0] == 'attr_datetime_missing')
assert_(attrs[4][1] == 'DATE "yyyy-MM-dd HH:mm"')
def test_dateheader_unsupported(self):
ofile = open(test8)
rel, attrs = read_header(ofile)
ofile.close()
assert_(rel == 'test8')
assert_(len(attrs) == 2)
assert_(attrs[0][0] == 'attr_datetime_utc')
assert_(attrs[0][1] == 'DATE "yyyy-MM-dd HH:mm Z"')
assert_(attrs[1][0] == 'attr_datetime_full')
assert_(attrs[1][1] == 'DATE "yy-MM-dd HH:mm:ss z"')
class TestDateAttribute(object):
def setup_method(self):
self.data, self.meta = loadarff(test7)
def test_year_attribute(self):
expected = np.array([
'1999',
'2004',
'1817',
'2100',
'2013',
'1631'
], dtype='datetime64[Y]')
assert_array_equal(self.data["attr_year"], expected)
def test_month_attribute(self):
expected = np.array([
'1999-01',
'2004-12',
'1817-04',
'2100-09',
'2013-11',
'1631-10'
], dtype='datetime64[M]')
assert_array_equal(self.data["attr_month"], expected)
def test_date_attribute(self):
expected = np.array([
'1999-01-31',
'2004-12-01',
'1817-04-28',
'2100-09-10',
'2013-11-30',
'1631-10-15'
], dtype='datetime64[D]')
assert_array_equal(self.data["attr_date"], expected)
def test_datetime_local_attribute(self):
expected = np.array([
datetime.datetime(year=1999, month=1, day=31, hour=0, minute=1),
datetime.datetime(year=2004, month=12, day=1, hour=23, minute=59),
datetime.datetime(year=1817, month=4, day=28, hour=13, minute=0),
datetime.datetime(year=2100, month=9, day=10, hour=12, minute=0),
datetime.datetime(year=2013, month=11, day=30, hour=4, minute=55),
datetime.datetime(year=1631, month=10, day=15, hour=20, minute=4)
], dtype='datetime64[m]')
assert_array_equal(self.data["attr_datetime_local"], expected)
def test_datetime_missing(self):
expected = np.array([
'nat',
'2004-12-01T23:59',
'nat',
'nat',
'2013-11-30T04:55',
'1631-10-15T20:04'
], dtype='datetime64[m]')
assert_array_equal(self.data["attr_datetime_missing"], expected)
def test_datetime_timezone(self):
assert_raises(ValueError, loadarff, test8)
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/__init__.py
-4
View File
@@ -1,4 +0,0 @@
from __future__ import division, print_function, absolute_import
from scipy.io.harwell_boeing.hb import MalformedHeader, HBInfo, HBFile, \
HBMatrixType, hb_read, hb_write
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/__pycache__/_fortran_format_parser.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/__pycache__/hb.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/__pycache__/setup.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/_fortran_format_parser.py
-312
View File
@@ -1,312 +0,0 @@
"""
Preliminary module to handle fortran formats for IO. Does not use this outside
scipy.sparse io for now, until the API is deemed reasonable.
The *Format classes handle conversion between fortran and python format, and
FortranFormatParser can create *Format instances from raw fortran format
strings (e.g. '(3I4)', '(10I3)', etc...)
"""
from __future__ import division, print_function, absolute_import
import re
import numpy as np
__all__ = ["BadFortranFormat", "FortranFormatParser", "IntFormat", "ExpFormat"]
TOKENS = {
"LPAR": r"\(",
"RPAR": r"\)",
"INT_ID": r"I",
"EXP_ID": r"E",
"INT": r"\d+",
"DOT": r"\.",
}
class BadFortranFormat(SyntaxError):
pass
def number_digits(n):
return int(np.floor(np.log10(np.abs(n))) + 1)
class IntFormat(object):
@classmethod
def from_number(cls, n, min=None):
"""Given an integer, returns a "reasonable" IntFormat instance to represent
any number between 0 and n if n > 0, -n and n if n < 0
Parameters
----------
n : int
max number one wants to be able to represent
min : int
minimum number of characters to use for the format
Returns
-------
res : IntFormat
IntFormat instance with reasonable (see Notes) computed width
Notes
-----
Reasonable should be understood as the minimal string length necessary
without losing precision. For example, IntFormat.from_number(1) will
return an IntFormat instance of width 2, so that any 0 and 1 may be
represented as 1-character strings without loss of information.
"""
width = number_digits(n) + 1
if n < 0:
width += 1
repeat = 80 // width
return cls(width, min, repeat=repeat)
def __init__(self, width, min=None, repeat=None):
self.width = width
self.repeat = repeat
self.min = min
def __repr__(self):
r = "IntFormat("
if self.repeat:
r += "%d" % self.repeat
r += "I%d" % self.width
if self.min:
r += ".%d" % self.min
return r + ")"
@property
def fortran_format(self):
r = "("
if self.repeat:
r += "%d" % self.repeat
r += "I%d" % self.width
if self.min:
r += ".%d" % self.min
return r + ")"
@property
def python_format(self):
return "%" + str(self.width) + "d"
class ExpFormat(object):
@classmethod
def from_number(cls, n, min=None):
"""Given a float number, returns a "reasonable" ExpFormat instance to
represent any number between -n and n.
Parameters
----------
n : float
max number one wants to be able to represent
min : int
minimum number of characters to use for the format
Returns
-------
res : ExpFormat
ExpFormat instance with reasonable (see Notes) computed width
Notes
-----
Reasonable should be understood as the minimal string length necessary
to avoid losing precision.
"""
# len of one number in exp format: sign + 1|0 + "." +
# number of digit for fractional part + 'E' + sign of exponent +
# len of exponent
finfo = np.finfo(n.dtype)
# Number of digits for fractional part
n_prec = finfo.precision + 1
# Number of digits for exponential part
n_exp = number_digits(np.max(np.abs([finfo.maxexp, finfo.minexp])))
width = 1 + 1 + n_prec + 1 + n_exp + 1
if n < 0:
width += 1
repeat = int(np.floor(80 / width))
return cls(width, n_prec, min, repeat=repeat)
def __init__(self, width, significand, min=None, repeat=None):
"""\
Parameters
----------
width : int
number of characters taken by the string (includes space).
"""
self.width = width
self.significand = significand
self.repeat = repeat
self.min = min
def __repr__(self):
r = "ExpFormat("
if self.repeat:
r += "%d" % self.repeat
r += "E%d.%d" % (self.width, self.significand)
if self.min:
r += "E%d" % self.min
return r + ")"
@property
def fortran_format(self):
r = "("
if self.repeat:
r += "%d" % self.repeat
r += "E%d.%d" % (self.width, self.significand)
if self.min:
r += "E%d" % self.min
return r + ")"
@property
def python_format(self):
return "%" + str(self.width-1) + "." + str(self.significand) + "E"
class Token(object):
def __init__(self, type, value, pos):
self.type = type
self.value = value
self.pos = pos
def __str__(self):
return """Token('%s', "%s")""" % (self.type, self.value)
def __repr__(self):
return self.__str__()
class Tokenizer(object):
def __init__(self):
self.tokens = list(TOKENS.keys())
self.res = [re.compile(TOKENS[i]) for i in self.tokens]
def input(self, s):
self.data = s
self.curpos = 0
self.len = len(s)
def next_token(self):
curpos = self.curpos
tokens = self.tokens
while curpos < self.len:
for i, r in enumerate(self.res):
m = r.match(self.data, curpos)
if m is None:
continue
else:
self.curpos = m.end()
return Token(self.tokens[i], m.group(), self.curpos)
raise SyntaxError("Unknown character at position %d (%s)"
% (self.curpos, self.data[curpos]))
# Grammar for fortran format:
# format : LPAR format_string RPAR
# format_string : repeated | simple
# repeated : repeat simple
# simple : int_fmt | exp_fmt
# int_fmt : INT_ID width
# exp_fmt : simple_exp_fmt
# simple_exp_fmt : EXP_ID width DOT significand
# extended_exp_fmt : EXP_ID width DOT significand EXP_ID ndigits
# repeat : INT
# width : INT
# significand : INT
# ndigits : INT
# Naive fortran formatter - parser is hand-made
class FortranFormatParser(object):
"""Parser for fortran format strings. The parse method returns a *Format
instance.
Notes
-----
Only ExpFormat (exponential format for floating values) and IntFormat
(integer format) for now.
"""
def __init__(self):
self.tokenizer = Tokenizer()
def parse(self, s):
self.tokenizer.input(s)
tokens = []
try:
while True:
t = self.tokenizer.next_token()
if t is None:
break
else:
tokens.append(t)
return self._parse_format(tokens)
except SyntaxError as e:
raise BadFortranFormat(str(e))
def _get_min(self, tokens):
next = tokens.pop(0)
if not next.type == "DOT":
raise SyntaxError()
next = tokens.pop(0)
return next.value
def _expect(self, token, tp):
if not token.type == tp:
raise SyntaxError()
def _parse_format(self, tokens):
if not tokens[0].type == "LPAR":
raise SyntaxError("Expected left parenthesis at position "
"%d (got '%s')" % (0, tokens[0].value))
elif not tokens[-1].type == "RPAR":
raise SyntaxError("Expected right parenthesis at position "
"%d (got '%s')" % (len(tokens), tokens[-1].value))
tokens = tokens[1:-1]
types = [t.type for t in tokens]
if types[0] == "INT":
repeat = int(tokens.pop(0).value)
else:
repeat = None
next = tokens.pop(0)
if next.type == "INT_ID":
next = self._next(tokens, "INT")
width = int(next.value)
if tokens:
min = int(self._get_min(tokens))
else:
min = None
return IntFormat(width, min, repeat)
elif next.type == "EXP_ID":
next = self._next(tokens, "INT")
width = int(next.value)
next = self._next(tokens, "DOT")
next = self._next(tokens, "INT")
significand = int(next.value)
if tokens:
next = self._next(tokens, "EXP_ID")
next = self._next(tokens, "INT")
min = int(next.value)
else:
min = None
return ExpFormat(width, significand, min, repeat)
else:
raise SyntaxError("Invalid formater type %s" % next.value)
def _next(self, tokens, tp):
if not len(tokens) > 0:
raise SyntaxError()
next = tokens.pop(0)
self._expect(next, tp)
return next
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/hb.py
-547
View File
@@ -1,547 +0,0 @@
"""
Implementation of Harwell-Boeing read/write.
At the moment not the full Harwell-Boeing format is supported. Supported
features are:
- assembled, non-symmetric, real matrices
- integer for pointer/indices
- exponential format for float values, and int format
"""
from __future__ import division, print_function, absolute_import
# TODO:
# - Add more support (symmetric/complex matrices, non-assembled matrices ?)
# XXX: reading is reasonably efficient (>= 85 % is in numpy.fromstring), but
# takes a lot of memory. Being faster would require compiled code.
# write is not efficient. Although not a terribly exciting task,
# having reusable facilities to efficiently read/write fortran-formatted files
# would be useful outside this module.
import warnings
import numpy as np
from scipy.sparse import csc_matrix
from scipy.io.harwell_boeing._fortran_format_parser import \
FortranFormatParser, IntFormat, ExpFormat
__all__ = ["MalformedHeader", "hb_read", "hb_write", "HBInfo", "HBFile",
"HBMatrixType"]
class MalformedHeader(Exception):
pass
class LineOverflow(Warning):
pass
def _nbytes_full(fmt, nlines):
"""Return the number of bytes to read to get every full lines for the
given parsed fortran format."""
return (fmt.repeat * fmt.width + 1) * (nlines - 1)
class HBInfo(object):
@classmethod
def from_data(cls, m, title="Default title", key="0", mxtype=None, fmt=None):
"""Create a HBInfo instance from an existing sparse matrix.
Parameters
----------
m : sparse matrix
the HBInfo instance will derive its parameters from m
title : str
Title to put in the HB header
key : str
Key
mxtype : HBMatrixType
type of the input matrix
fmt : dict
not implemented
Returns
-------
hb_info : HBInfo instance
"""
m = m.tocsc(copy=False)
pointer = m.indptr
indices = m.indices
values = m.data
nrows, ncols = m.shape
nnon_zeros = m.nnz
if fmt is None:
# +1 because HB use one-based indexing (Fortran), and we will write
# the indices /pointer as such
pointer_fmt = IntFormat.from_number(np.max(pointer+1))
indices_fmt = IntFormat.from_number(np.max(indices+1))
if values.dtype.kind in np.typecodes["AllFloat"]:
values_fmt = ExpFormat.from_number(-np.max(np.abs(values)))
elif values.dtype.kind in np.typecodes["AllInteger"]:
values_fmt = IntFormat.from_number(-np.max(np.abs(values)))
else:
raise NotImplementedError("type %s not implemented yet" % values.dtype.kind)
else:
raise NotImplementedError("fmt argument not supported yet.")
if mxtype is None:
if not np.isrealobj(values):
raise ValueError("Complex values not supported yet")
if values.dtype.kind in np.typecodes["AllInteger"]:
tp = "integer"
elif values.dtype.kind in np.typecodes["AllFloat"]:
tp = "real"
else:
raise NotImplementedError("type %s for values not implemented"
% values.dtype)
mxtype = HBMatrixType(tp, "unsymmetric", "assembled")
else:
raise ValueError("mxtype argument not handled yet.")
def _nlines(fmt, size):
nlines = size // fmt.repeat
if nlines * fmt.repeat != size:
nlines += 1
return nlines
pointer_nlines = _nlines(pointer_fmt, pointer.size)
indices_nlines = _nlines(indices_fmt, indices.size)
values_nlines = _nlines(values_fmt, values.size)
total_nlines = pointer_nlines + indices_nlines + values_nlines
return cls(title, key,
total_nlines, pointer_nlines, indices_nlines, values_nlines,
mxtype, nrows, ncols, nnon_zeros,
pointer_fmt.fortran_format, indices_fmt.fortran_format,
values_fmt.fortran_format)
@classmethod
def from_file(cls, fid):
"""Create a HBInfo instance from a file object containing a matrix in the
HB format.
Parameters
----------
fid : file-like matrix
File or file-like object containing a matrix in the HB format.
Returns
-------
hb_info : HBInfo instance
"""
# First line
line = fid.readline().strip("\n")
if not len(line) > 72:
raise ValueError("Expected at least 72 characters for first line, "
"got: \n%s" % line)
title = line[:72]
key = line[72:]
# Second line
line = fid.readline().strip("\n")
if not len(line.rstrip()) >= 56:
raise ValueError("Expected at least 56 characters for second line, "
"got: \n%s" % line)
total_nlines = _expect_int(line[:14])
pointer_nlines = _expect_int(line[14:28])
indices_nlines = _expect_int(line[28:42])
values_nlines = _expect_int(line[42:56])
rhs_nlines = line[56:72].strip()
if rhs_nlines == '':
rhs_nlines = 0
else:
rhs_nlines = _expect_int(rhs_nlines)
if not rhs_nlines == 0:
raise ValueError("Only files without right hand side supported for "
"now.")
# Third line
line = fid.readline().strip("\n")
if not len(line) >= 70:
raise ValueError("Expected at least 72 character for third line, got:\n"
"%s" % line)
mxtype_s = line[:3].upper()
if not len(mxtype_s) == 3:
raise ValueError("mxtype expected to be 3 characters long")
mxtype = HBMatrixType.from_fortran(mxtype_s)
if mxtype.value_type not in ["real", "integer"]:
raise ValueError("Only real or integer matrices supported for "
"now (detected %s)" % mxtype)
if not mxtype.structure == "unsymmetric":
raise ValueError("Only unsymmetric matrices supported for "
"now (detected %s)" % mxtype)
if not mxtype.storage == "assembled":
raise ValueError("Only assembled matrices supported for now")
if not line[3:14] == " " * 11:
raise ValueError("Malformed data for third line: %s" % line)
nrows = _expect_int(line[14:28])
ncols = _expect_int(line[28:42])
nnon_zeros = _expect_int(line[42:56])
nelementals = _expect_int(line[56:70])
if not nelementals == 0:
raise ValueError("Unexpected value %d for nltvl (last entry of line 3)"
% nelementals)
# Fourth line
line = fid.readline().strip("\n")
ct = line.split()
if not len(ct) == 3:
raise ValueError("Expected 3 formats, got %s" % ct)
return cls(title, key,
total_nlines, pointer_nlines, indices_nlines, values_nlines,
mxtype, nrows, ncols, nnon_zeros,
ct[0], ct[1], ct[2],
rhs_nlines, nelementals)
def __init__(self, title, key,
total_nlines, pointer_nlines, indices_nlines, values_nlines,
mxtype, nrows, ncols, nnon_zeros,
pointer_format_str, indices_format_str, values_format_str,
right_hand_sides_nlines=0, nelementals=0):
"""Do not use this directly, but the class ctrs (from_* functions)."""
self.title = title
self.key = key
if title is None:
title = "No Title"
if len(title) > 72:
raise ValueError("title cannot be > 72 characters")
if key is None:
key = "|No Key"
if len(key) > 8:
warnings.warn("key is > 8 characters (key is %s)" % key, LineOverflow)
self.total_nlines = total_nlines
self.pointer_nlines = pointer_nlines
self.indices_nlines = indices_nlines
self.values_nlines = values_nlines
parser = FortranFormatParser()
pointer_format = parser.parse(pointer_format_str)
if not isinstance(pointer_format, IntFormat):
raise ValueError("Expected int format for pointer format, got %s"
% pointer_format)
indices_format = parser.parse(indices_format_str)
if not isinstance(indices_format, IntFormat):
raise ValueError("Expected int format for indices format, got %s" %
indices_format)
values_format = parser.parse(values_format_str)
if isinstance(values_format, ExpFormat):
if mxtype.value_type not in ["real", "complex"]:
raise ValueError("Inconsistency between matrix type %s and "
"value type %s" % (mxtype, values_format))
values_dtype = np.float64
elif isinstance(values_format, IntFormat):
if mxtype.value_type not in ["integer"]:
raise ValueError("Inconsistency between matrix type %s and "
"value type %s" % (mxtype, values_format))
# XXX: fortran int -> dtype association ?
values_dtype = int
else:
raise ValueError("Unsupported format for values %r" % (values_format,))
self.pointer_format = pointer_format
self.indices_format = indices_format
self.values_format = values_format
self.pointer_dtype = np.int32
self.indices_dtype = np.int32
self.values_dtype = values_dtype
self.pointer_nlines = pointer_nlines
self.pointer_nbytes_full = _nbytes_full(pointer_format, pointer_nlines)
self.indices_nlines = indices_nlines
self.indices_nbytes_full = _nbytes_full(indices_format, indices_nlines)
self.values_nlines = values_nlines
self.values_nbytes_full = _nbytes_full(values_format, values_nlines)
self.nrows = nrows
self.ncols = ncols
self.nnon_zeros = nnon_zeros
self.nelementals = nelementals
self.mxtype = mxtype
def dump(self):
"""Gives the header corresponding to this instance as a string."""
header = [self.title.ljust(72) + self.key.ljust(8)]
header.append("%14d%14d%14d%14d" %
(self.total_nlines, self.pointer_nlines,
self.indices_nlines, self.values_nlines))
header.append("%14s%14d%14d%14d%14d" %
(self.mxtype.fortran_format.ljust(14), self.nrows,
self.ncols, self.nnon_zeros, 0))
pffmt = self.pointer_format.fortran_format
iffmt = self.indices_format.fortran_format
vffmt = self.values_format.fortran_format
header.append("%16s%16s%20s" %
(pffmt.ljust(16), iffmt.ljust(16), vffmt.ljust(20)))
return "\n".join(header)
def _expect_int(value, msg=None):
try:
return int(value)
except ValueError:
if msg is None:
msg = "Expected an int, got %s"
raise ValueError(msg % value)
def _read_hb_data(content, header):
# XXX: look at a way to reduce memory here (big string creation)
ptr_string = "".join([content.read(header.pointer_nbytes_full),
content.readline()])
ptr = np.fromstring(ptr_string,
dtype=int, sep=' ')
ind_string = "".join([content.read(header.indices_nbytes_full),
content.readline()])
ind = np.fromstring(ind_string,
dtype=int, sep=' ')
val_string = "".join([content.read(header.values_nbytes_full),
content.readline()])
val = np.fromstring(val_string,
dtype=header.values_dtype, sep=' ')
try:
return csc_matrix((val, ind-1, ptr-1),
shape=(header.nrows, header.ncols))
except ValueError as e:
raise e
def _write_data(m, fid, header):
m = m.tocsc(copy=False)
def write_array(f, ar, nlines, fmt):
# ar_nlines is the number of full lines, n is the number of items per
# line, ffmt the fortran format
pyfmt = fmt.python_format
pyfmt_full = pyfmt * fmt.repeat
# for each array to write, we first write the full lines, and special
# case for partial line
full = ar[:(nlines - 1) * fmt.repeat]
for row in full.reshape((nlines-1, fmt.repeat)):
f.write(pyfmt_full % tuple(row) + "\n")
nremain = ar.size - full.size
if nremain > 0:
f.write((pyfmt * nremain) % tuple(ar[ar.size - nremain:]) + "\n")
fid.write(header.dump())
fid.write("\n")
# +1 is for fortran one-based indexing
write_array(fid, m.indptr+1, header.pointer_nlines,
header.pointer_format)
write_array(fid, m.indices+1, header.indices_nlines,
header.indices_format)
write_array(fid, m.data, header.values_nlines,
header.values_format)
class HBMatrixType(object):
"""Class to hold the matrix type."""
# q2f* translates qualified names to fortran character
_q2f_type = {
"real": "R",
"complex": "C",
"pattern": "P",
"integer": "I",
}
_q2f_structure = {
"symmetric": "S",
"unsymmetric": "U",
"hermitian": "H",
"skewsymmetric": "Z",
"rectangular": "R"
}
_q2f_storage = {
"assembled": "A",
"elemental": "E",
}
_f2q_type = dict([(j, i) for i, j in _q2f_type.items()])
_f2q_structure = dict([(j, i) for i, j in _q2f_structure.items()])
_f2q_storage = dict([(j, i) for i, j in _q2f_storage.items()])
@classmethod
def from_fortran(cls, fmt):
if not len(fmt) == 3:
raise ValueError("Fortran format for matrix type should be 3 "
"characters long")
try:
value_type = cls._f2q_type[fmt[0]]
structure = cls._f2q_structure[fmt[1]]
storage = cls._f2q_storage[fmt[2]]
return cls(value_type, structure, storage)
except KeyError:
raise ValueError("Unrecognized format %s" % fmt)
def __init__(self, value_type, structure, storage="assembled"):
self.value_type = value_type
self.structure = structure
self.storage = storage
if value_type not in self._q2f_type:
raise ValueError("Unrecognized type %s" % value_type)
if structure not in self._q2f_structure:
raise ValueError("Unrecognized structure %s" % structure)
if storage not in self._q2f_storage:
raise ValueError("Unrecognized storage %s" % storage)
@property
def fortran_format(self):
return self._q2f_type[self.value_type] + \
self._q2f_structure[self.structure] + \
self._q2f_storage[self.storage]
def __repr__(self):
return "HBMatrixType(%s, %s, %s)" % \
(self.value_type, self.structure, self.storage)
class HBFile(object):
def __init__(self, file, hb_info=None):
"""Create a HBFile instance.
Parameters
----------
file : file-object
StringIO work as well
hb_info : HBInfo, optional
Should be given as an argument for writing, in which case the file
should be writable.
"""
self._fid = file
if hb_info is None:
self._hb_info = HBInfo.from_file(file)
else:
#raise IOError("file %s is not writable, and hb_info "
# "was given." % file)
self._hb_info = hb_info
@property
def title(self):
return self._hb_info.title
@property
def key(self):
return self._hb_info.key
@property
def type(self):
return self._hb_info.mxtype.value_type
@property
def structure(self):
return self._hb_info.mxtype.structure
@property
def storage(self):
return self._hb_info.mxtype.storage
def read_matrix(self):
return _read_hb_data(self._fid, self._hb_info)
def write_matrix(self, m):
return _write_data(m, self._fid, self._hb_info)
def hb_read(path_or_open_file):
"""Read HB-format file.
Parameters
----------
path_or_open_file : path-like or file-like
If a file-like object, it is used as-is. Otherwise it is opened
before reading.
Returns
-------
data : scipy.sparse.csc_matrix instance
The data read from the HB file as a sparse matrix.
Notes
-----
At the moment not the full Harwell-Boeing format is supported. Supported
features are:
- assembled, non-symmetric, real matrices
- integer for pointer/indices
- exponential format for float values, and int format
"""
def _get_matrix(fid):
hb = HBFile(fid)
return hb.read_matrix()
if hasattr(path_or_open_file, 'read'):
return _get_matrix(path_or_open_file)
else:
with open(path_or_open_file) as f:
return _get_matrix(f)
def hb_write(path_or_open_file, m, hb_info=None):
"""Write HB-format file.
Parameters
----------
path_or_open_file : path-like or file-like
If a file-like object, it is used as-is. Otherwise it is opened
before writing.
m : sparse-matrix
the sparse matrix to write
hb_info : HBInfo
contains the meta-data for write
Returns
-------
None
Notes
-----
At the moment not the full Harwell-Boeing format is supported. Supported
features are:
- assembled, non-symmetric, real matrices
- integer for pointer/indices
- exponential format for float values, and int format
"""
m = m.tocsc(copy=False)
if hb_info is None:
hb_info = HBInfo.from_data(m)
def _set_matrix(fid):
hb = HBFile(fid, hb_info)
return hb.write_matrix(m)
if hasattr(path_or_open_file, 'write'):
return _set_matrix(path_or_open_file)
else:
with open(path_or_open_file, 'w') as f:
return _set_matrix(f)
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/setup.py
-14
View File
@@ -1,14 +0,0 @@
from __future__ import division, print_function, absolute_import
def configuration(parent_package='',top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('harwell_boeing',parent_package,top_path)
config.add_data_dir('tests')
return config
if __name__ == '__main__':
from numpy.distutils.core import setup
setup(**configuration(top_path='').todict())
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/tests/__init__.py
View File
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/tests/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/tests/__pycache__/test_fortran_format.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/tests/__pycache__/test_hb.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/tests/test_fortran_format.py
-77
View File
@@ -1,77 +0,0 @@
from __future__ import division, print_function, absolute_import
import numpy as np
from numpy.testing import assert_equal
from pytest import raises as assert_raises
from scipy.io.harwell_boeing._fortran_format_parser import (
FortranFormatParser, IntFormat, ExpFormat, BadFortranFormat,
number_digits)
class TestFortranFormatParser(object):
def setup_method(self):
self.parser = FortranFormatParser()
def _test_equal(self, format, ref):
ret = self.parser.parse(format)
assert_equal(ret.__dict__, ref.__dict__)
def test_simple_int(self):
self._test_equal("(I4)", IntFormat(4))
def test_simple_repeated_int(self):
self._test_equal("(3I4)", IntFormat(4, repeat=3))
def test_simple_exp(self):
self._test_equal("(E4.3)", ExpFormat(4, 3))
def test_exp_exp(self):
self._test_equal("(E8.3E3)", ExpFormat(8, 3, 3))
def test_repeat_exp(self):
self._test_equal("(2E4.3)", ExpFormat(4, 3, repeat=2))
def test_repeat_exp_exp(self):
self._test_equal("(2E8.3E3)", ExpFormat(8, 3, 3, repeat=2))
def test_wrong_formats(self):
def _test_invalid(bad_format):
assert_raises(BadFortranFormat, lambda: self.parser.parse(bad_format))
_test_invalid("I4")
_test_invalid("(E4)")
_test_invalid("(E4.)")
_test_invalid("(E4.E3)")
class TestIntFormat(object):
def test_to_fortran(self):
f = [IntFormat(10), IntFormat(12, 10), IntFormat(12, 10, 3)]
res = ["(I10)", "(I12.10)", "(3I12.10)"]
for i, j in zip(f, res):
assert_equal(i.fortran_format, j)
def test_from_number(self):
f = [10, -12, 123456789]
r_f = [IntFormat(3, repeat=26), IntFormat(4, repeat=20),
IntFormat(10, repeat=8)]
for i, j in zip(f, r_f):
assert_equal(IntFormat.from_number(i).__dict__, j.__dict__)
class TestExpFormat(object):
def test_to_fortran(self):
f = [ExpFormat(10, 5), ExpFormat(12, 10), ExpFormat(12, 10, min=3),
ExpFormat(10, 5, repeat=3)]
res = ["(E10.5)", "(E12.10)", "(E12.10E3)", "(3E10.5)"]
for i, j in zip(f, res):
assert_equal(i.fortran_format, j)
def test_from_number(self):
f = np.array([1.0, -1.2])
r_f = [ExpFormat(24, 16, repeat=3), ExpFormat(25, 16, repeat=3)]
for i, j in zip(f, r_f):
assert_equal(ExpFormat.from_number(i).__dict__, j.__dict__)
utils/venv/lib/python3.6/site-packages/scipy/io/harwell_boeing/tests/test_hb.py
-71
View File
@@ -1,71 +0,0 @@
from __future__ import division, print_function, absolute_import
import sys
if sys.version_info[0] >= 3:
from io import StringIO
else:
from StringIO import StringIO
import tempfile
import numpy as np
from numpy.testing import assert_equal, \
assert_array_almost_equal_nulp
from scipy.sparse import coo_matrix, csc_matrix, rand
from scipy.io import hb_read, hb_write
SIMPLE = """\
No Title |No Key
9 4 1 4
RUA 100 100 10 0
(26I3) (26I3) (3E23.15)
1 2 2 2 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3
3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3
3 3 3 3 3 3 3 4 4 4 6 6 6 6 6 6 6 6 6 6 6 8 9 9 9 9
9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 11
37 71 89 18 30 45 70 19 25 52
2.971243799687726e-01 3.662366682877375e-01 4.786962174699534e-01
6.490068647991184e-01 6.617490424831662e-02 8.870370343191623e-01
4.196478590163001e-01 5.649603072111251e-01 9.934423887087086e-01
6.912334991524289e-01
"""
SIMPLE_MATRIX = coo_matrix(
((0.297124379969, 0.366236668288, 0.47869621747, 0.649006864799,
0.0661749042483, 0.887037034319, 0.419647859016,
0.564960307211, 0.993442388709, 0.691233499152,),
(np.array([[36, 70, 88, 17, 29, 44, 69, 18, 24, 51],
[0, 4, 58, 61, 61, 72, 72, 73, 99, 99]]))))
def assert_csc_almost_equal(r, l):
r = csc_matrix(r)
l = csc_matrix(l)
assert_equal(r.indptr, l.indptr)
assert_equal(r.indices, l.indices)
assert_array_almost_equal_nulp(r.data, l.data, 10000)
class TestHBReader(object):
def test_simple(self):
m = hb_read(StringIO(SIMPLE))
assert_csc_almost_equal(m, SIMPLE_MATRIX)
class TestHBReadWrite(object):
def check_save_load(self, value):
with tempfile.NamedTemporaryFile(mode='w+t') as file:
hb_write(file, value)
file.file.seek(0)
value_loaded = hb_read(file)
assert_csc_almost_equal(value, value_loaded)
def test_simple(self):
random_matrix = rand(10, 100, 0.1)
for matrix_format in ('coo', 'csc', 'csr', 'bsr', 'dia', 'dok', 'lil'):
matrix = random_matrix.asformat(matrix_format, copy=False)
self.check_save_load(matrix)
utils/venv/lib/python3.6/site-packages/scipy/io/idl.py
-884
View File
@@ -1,884 +0,0 @@
# IDLSave - a python module to read IDL 'save' files
# Copyright (c) 2010 Thomas P. Robitaille
# Many thanks to Craig Markwardt for publishing the Unofficial Format
# Specification for IDL .sav files, without which this Python module would not
# exist (http://cow.physics.wisc.edu/~craigm/idl/savefmt).
# This code was developed by with permission from ITT Visual Information
# Systems. IDL(r) is a registered trademark of ITT Visual Information Systems,
# Inc. for their Interactive Data Language software.
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
from __future__ import division, print_function, absolute_import
__all__ = ['readsav']
import struct
import numpy as np
from numpy.compat import asstr
import tempfile
import zlib
import warnings
# Define the different data types that can be found in an IDL save file
DTYPE_DICT = {1: '>u1',
2: '>i2',
3: '>i4',
4: '>f4',
5: '>f8',
6: '>c8',
7: '|O',
8: '|O',
9: '>c16',
10: '|O',
11: '|O',
12: '>u2',
13: '>u4',
14: '>i8',
15: '>u8'}
# Define the different record types that can be found in an IDL save file
RECTYPE_DICT = {0: "START_MARKER",
1: "COMMON_VARIABLE",
2: "VARIABLE",
3: "SYSTEM_VARIABLE",
6: "END_MARKER",
10: "TIMESTAMP",
12: "COMPILED",
13: "IDENTIFICATION",
14: "VERSION",
15: "HEAP_HEADER",
16: "HEAP_DATA",
17: "PROMOTE64",
19: "NOTICE",
20: "DESCRIPTION"}
# Define a dictionary to contain structure definitions
STRUCT_DICT = {}
def _align_32(f):
'''Align to the next 32-bit position in a file'''
pos = f.tell()
if pos % 4 != 0:
f.seek(pos + 4 - pos % 4)
return
def _skip_bytes(f, n):
'''Skip `n` bytes'''
f.read(n)
return
def _read_bytes(f, n):
'''Read the next `n` bytes'''
return f.read(n)
def _read_byte(f):
'''Read a single byte'''
return np.uint8(struct.unpack('>B', f.read(4)[:1])[0])
def _read_long(f):
'''Read a signed 32-bit integer'''
return np.int32(struct.unpack('>l', f.read(4))[0])
def _read_int16(f):
'''Read a signed 16-bit integer'''
return np.int16(struct.unpack('>h', f.read(4)[2:4])[0])
def _read_int32(f):
'''Read a signed 32-bit integer'''
return np.int32(struct.unpack('>i', f.read(4))[0])
def _read_int64(f):
'''Read a signed 64-bit integer'''
return np.int64(struct.unpack('>q', f.read(8))[0])
def _read_uint16(f):
'''Read an unsigned 16-bit integer'''
return np.uint16(struct.unpack('>H', f.read(4)[2:4])[0])
def _read_uint32(f):
'''Read an unsigned 32-bit integer'''
return np.uint32(struct.unpack('>I', f.read(4))[0])
def _read_uint64(f):
'''Read an unsigned 64-bit integer'''
return np.uint64(struct.unpack('>Q', f.read(8))[0])
def _read_float32(f):
'''Read a 32-bit float'''
return np.float32(struct.unpack('>f', f.read(4))[0])
def _read_float64(f):
'''Read a 64-bit float'''
return np.float64(struct.unpack('>d', f.read(8))[0])
class Pointer(object):
'''Class used to define pointers'''
def __init__(self, index):
self.index = index
return
class ObjectPointer(Pointer):
'''Class used to define object pointers'''
pass
def _read_string(f):
'''Read a string'''
length = _read_long(f)
if length > 0:
chars = _read_bytes(f, length)
_align_32(f)
chars = asstr(chars)
else:
chars = ''
return chars
def _read_string_data(f):
'''Read a data string (length is specified twice)'''
length = _read_long(f)
if length > 0:
length = _read_long(f)
string_data = _read_bytes(f, length)
_align_32(f)
else:
string_data = ''
return string_data
def _read_data(f, dtype):
'''Read a variable with a specified data type'''
if dtype == 1:
if _read_int32(f) != 1:
raise Exception("Error occurred while reading byte variable")
return _read_byte(f)
elif dtype == 2:
return _read_int16(f)
elif dtype == 3:
return _read_int32(f)
elif dtype == 4:
return _read_float32(f)
elif dtype == 5:
return _read_float64(f)
elif dtype == 6:
real = _read_float32(f)
imag = _read_float32(f)
return np.complex64(real + imag * 1j)
elif dtype == 7:
return _read_string_data(f)
elif dtype == 8:
raise Exception("Should not be here - please report this")
elif dtype == 9:
real = _read_float64(f)
imag = _read_float64(f)
return np.complex128(real + imag * 1j)
elif dtype == 10:
return Pointer(_read_int32(f))
elif dtype == 11:
return ObjectPointer(_read_int32(f))
elif dtype == 12:
return _read_uint16(f)
elif dtype == 13:
return _read_uint32(f)
elif dtype == 14:
return _read_int64(f)
elif dtype == 15:
return _read_uint64(f)
else:
raise Exception("Unknown IDL type: %i - please report this" % dtype)
def _read_structure(f, array_desc, struct_desc):
'''
Read a structure, with the array and structure descriptors given as
`array_desc` and `structure_desc` respectively.
'''
nrows = array_desc['nelements']
columns = struct_desc['tagtable']
dtype = []
for col in columns:
if col['structure'] or col['array']:
dtype.append(((col['name'].lower(), col['name']), np.object_))
else:
if col['typecode'] in DTYPE_DICT:
dtype.append(((col['name'].lower(), col['name']),
DTYPE_DICT[col['typecode']]))
else:
raise Exception("Variable type %i not implemented" %
col['typecode'])
structure = np.recarray((nrows, ), dtype=dtype)
for i in range(nrows):
for col in columns:
dtype = col['typecode']
if col['structure']:
structure[col['name']][i] = _read_structure(f,
struct_desc['arrtable'][col['name']],
struct_desc['structtable'][col['name']])
elif col['array']:
structure[col['name']][i] = _read_array(f, dtype,
struct_desc['arrtable'][col['name']])
else:
structure[col['name']][i] = _read_data(f, dtype)
# Reshape structure if needed
if array_desc['ndims'] > 1:
dims = array_desc['dims'][:int(array_desc['ndims'])]
dims.reverse()
structure = structure.reshape(dims)
return structure
def _read_array(f, typecode, array_desc):
'''
Read an array of type `typecode`, with the array descriptor given as
`array_desc`.
'''
if typecode in [1, 3, 4, 5, 6, 9, 13, 14, 15]:
if typecode == 1:
nbytes = _read_int32(f)
if nbytes != array_desc['nbytes']:
warnings.warn("Not able to verify number of bytes from header")
# Read bytes as numpy array
array = np.frombuffer(f.read(array_desc['nbytes']),
dtype=DTYPE_DICT[typecode])
elif typecode in [2, 12]:
# These are 2 byte types, need to skip every two as they are not packed
array = np.frombuffer(f.read(array_desc['nbytes']*2),
dtype=DTYPE_DICT[typecode])[1::2]
else:
# Read bytes into list
array = []
for i in range(array_desc['nelements']):
dtype = typecode
data = _read_data(f, dtype)
array.append(data)
array = np.array(array, dtype=np.object_)
# Reshape array if needed
if array_desc['ndims'] > 1:
dims = array_desc['dims'][:int(array_desc['ndims'])]
dims.reverse()
array = array.reshape(dims)
# Go to next alignment position
_align_32(f)
return array
def _read_record(f):
'''Function to read in a full record'''
record = {'rectype': _read_long(f)}
nextrec = _read_uint32(f)
nextrec += _read_uint32(f) * 2**32
_skip_bytes(f, 4)
if not record['rectype'] in RECTYPE_DICT:
raise Exception("Unknown RECTYPE: %i" % record['rectype'])
record['rectype'] = RECTYPE_DICT[record['rectype']]
if record['rectype'] in ["VARIABLE", "HEAP_DATA"]:
if record['rectype'] == "VARIABLE":
record['varname'] = _read_string(f)
else:
record['heap_index'] = _read_long(f)
_skip_bytes(f, 4)
rectypedesc = _read_typedesc(f)
if rectypedesc['typecode'] == 0:
if nextrec == f.tell():
record['data'] = None # Indicates NULL value
else:
raise ValueError("Unexpected type code: 0")
else:
varstart = _read_long(f)
if varstart != 7:
raise Exception("VARSTART is not 7")
if rectypedesc['structure']:
record['data'] = _read_structure(f, rectypedesc['array_desc'],
rectypedesc['struct_desc'])
elif rectypedesc['array']:
record['data'] = _read_array(f, rectypedesc['typecode'],
rectypedesc['array_desc'])
else:
dtype = rectypedesc['typecode']
record['data'] = _read_data(f, dtype)
elif record['rectype'] == "TIMESTAMP":
_skip_bytes(f, 4*256)
record['date'] = _read_string(f)
record['user'] = _read_string(f)
record['host'] = _read_string(f)
elif record['rectype'] == "VERSION":
record['format'] = _read_long(f)
record['arch'] = _read_string(f)
record['os'] = _read_string(f)
record['release'] = _read_string(f)
elif record['rectype'] == "IDENTIFICATON":
record['author'] = _read_string(f)
record['title'] = _read_string(f)
record['idcode'] = _read_string(f)
elif record['rectype'] == "NOTICE":
record['notice'] = _read_string(f)
elif record['rectype'] == "DESCRIPTION":
record['description'] = _read_string_data(f)
elif record['rectype'] == "HEAP_HEADER":
record['nvalues'] = _read_long(f)
record['indices'] = []
for i in range(record['nvalues']):
record['indices'].append(_read_long(f))
elif record['rectype'] == "COMMONBLOCK":
record['nvars'] = _read_long(f)
record['name'] = _read_string(f)
record['varnames'] = []
for i in range(record['nvars']):
record['varnames'].append(_read_string(f))
elif record['rectype'] == "END_MARKER":
record['end'] = True
elif record['rectype'] == "UNKNOWN":
warnings.warn("Skipping UNKNOWN record")
elif record['rectype'] == "SYSTEM_VARIABLE":
warnings.warn("Skipping SYSTEM_VARIABLE record")
else:
raise Exception("record['rectype']=%s not implemented" %
record['rectype'])
f.seek(nextrec)
return record
def _read_typedesc(f):
'''Function to read in a type descriptor'''
typedesc = {'typecode': _read_long(f), 'varflags': _read_long(f)}
if typedesc['varflags'] & 2 == 2:
raise Exception("System variables not implemented")
typedesc['array'] = typedesc['varflags'] & 4 == 4
typedesc['structure'] = typedesc['varflags'] & 32 == 32
if typedesc['structure']:
typedesc['array_desc'] = _read_arraydesc(f)
typedesc['struct_desc'] = _read_structdesc(f)
elif typedesc['array']:
typedesc['array_desc'] = _read_arraydesc(f)
return typedesc
def _read_arraydesc(f):
'''Function to read in an array descriptor'''
arraydesc = {'arrstart': _read_long(f)}
if arraydesc['arrstart'] == 8:
_skip_bytes(f, 4)
arraydesc['nbytes'] = _read_long(f)
arraydesc['nelements'] = _read_long(f)
arraydesc['ndims'] = _read_long(f)
_skip_bytes(f, 8)
arraydesc['nmax'] = _read_long(f)
arraydesc['dims'] = []
for d in range(arraydesc['nmax']):
arraydesc['dims'].append(_read_long(f))
elif arraydesc['arrstart'] == 18:
warnings.warn("Using experimental 64-bit array read")
_skip_bytes(f, 8)
arraydesc['nbytes'] = _read_uint64(f)
arraydesc['nelements'] = _read_uint64(f)
arraydesc['ndims'] = _read_long(f)
_skip_bytes(f, 8)
arraydesc['nmax'] = 8
arraydesc['dims'] = []
for d in range(arraydesc['nmax']):
v = _read_long(f)
if v != 0:
raise Exception("Expected a zero in ARRAY_DESC")
arraydesc['dims'].append(_read_long(f))
else:
raise Exception("Unknown ARRSTART: %i" % arraydesc['arrstart'])
return arraydesc
def _read_structdesc(f):
'''Function to read in a structure descriptor'''
structdesc = {}
structstart = _read_long(f)
if structstart != 9:
raise Exception("STRUCTSTART should be 9")
structdesc['name'] = _read_string(f)
predef = _read_long(f)
structdesc['ntags'] = _read_long(f)
structdesc['nbytes'] = _read_long(f)
structdesc['predef'] = predef & 1
structdesc['inherits'] = predef & 2
structdesc['is_super'] = predef & 4
if not structdesc['predef']:
structdesc['tagtable'] = []
for t in range(structdesc['ntags']):
structdesc['tagtable'].append(_read_tagdesc(f))
for tag in structdesc['tagtable']:
tag['name'] = _read_string(f)
structdesc['arrtable'] = {}
for tag in structdesc['tagtable']:
if tag['array']:
structdesc['arrtable'][tag['name']] = _read_arraydesc(f)
structdesc['structtable'] = {}
for tag in structdesc['tagtable']:
if tag['structure']:
structdesc['structtable'][tag['name']] = _read_structdesc(f)
if structdesc['inherits'] or structdesc['is_super']:
structdesc['classname'] = _read_string(f)
structdesc['nsupclasses'] = _read_long(f)
structdesc['supclassnames'] = []
for s in range(structdesc['nsupclasses']):
structdesc['supclassnames'].append(_read_string(f))
structdesc['supclasstable'] = []
for s in range(structdesc['nsupclasses']):
structdesc['supclasstable'].append(_read_structdesc(f))
STRUCT_DICT[structdesc['name']] = structdesc
else:
if not structdesc['name'] in STRUCT_DICT:
raise Exception("PREDEF=1 but can't find definition")
structdesc = STRUCT_DICT[structdesc['name']]
return structdesc
def _read_tagdesc(f):
'''Function to read in a tag descriptor'''
tagdesc = {'offset': _read_long(f)}
if tagdesc['offset'] == -1:
tagdesc['offset'] = _read_uint64(f)
tagdesc['typecode'] = _read_long(f)
tagflags = _read_long(f)
tagdesc['array'] = tagflags & 4 == 4
tagdesc['structure'] = tagflags & 32 == 32
tagdesc['scalar'] = tagdesc['typecode'] in DTYPE_DICT
# Assume '10'x is scalar
return tagdesc
def _replace_heap(variable, heap):
if isinstance(variable, Pointer):
while isinstance(variable, Pointer):
if variable.index == 0:
variable = None
else:
if variable.index in heap:
variable = heap[variable.index]
else:
warnings.warn("Variable referenced by pointer not found "
"in heap: variable will be set to None")
variable = None
replace, new = _replace_heap(variable, heap)
if replace:
variable = new
return True, variable
elif isinstance(variable, np.core.records.recarray):
# Loop over records
for ir, record in enumerate(variable):
replace, new = _replace_heap(record, heap)
if replace:
variable[ir] = new
return False, variable
elif isinstance(variable, np.core.records.record):
# Loop over values
for iv, value in enumerate(variable):
replace, new = _replace_heap(value, heap)
if replace:
variable[iv] = new
return False, variable
elif isinstance(variable, np.ndarray):
# Loop over values if type is np.object_
if variable.dtype.type is np.object_:
for iv in range(variable.size):
replace, new = _replace_heap(variable.item(iv), heap)
if replace:
variable.itemset(iv, new)
return False, variable
else:
return False, variable
class AttrDict(dict):
'''
A case-insensitive dictionary with access via item, attribute, and call
notations:
>>> d = AttrDict()
>>> d['Variable'] = 123
>>> d['Variable']
123
>>> d.Variable
123
>>> d.variable
123
>>> d('VARIABLE')
123
'''
def __init__(self, init={}):
dict.__init__(self, init)
def __getitem__(self, name):
return super(AttrDict, self).__getitem__(name.lower())
def __setitem__(self, key, value):
return super(AttrDict, self).__setitem__(key.lower(), value)
__getattr__ = __getitem__
__setattr__ = __setitem__
__call__ = __getitem__
def readsav(file_name, idict=None, python_dict=False,
uncompressed_file_name=None, verbose=False):
"""
Read an IDL .sav file.
Parameters
----------
file_name : str
Name of the IDL save file.
idict : dict, optional
Dictionary in which to insert .sav file variables.
python_dict : bool, optional
By default, the object return is not a Python dictionary, but a
case-insensitive dictionary with item, attribute, and call access
to variables. To get a standard Python dictionary, set this option
to True.
uncompressed_file_name : str, optional
This option only has an effect for .sav files written with the
/compress option. If a file name is specified, compressed .sav
files are uncompressed to this file. Otherwise, readsav will use
the `tempfile` module to determine a temporary filename
automatically, and will remove the temporary file upon successfully
reading it in.
verbose : bool, optional
Whether to print out information about the save file, including
the records read, and available variables.
Returns
-------
idl_dict : AttrDict or dict
If `python_dict` is set to False (default), this function returns a
case-insensitive dictionary with item, attribute, and call access
to variables. If `python_dict` is set to True, this function
returns a Python dictionary with all variable names in lowercase.
If `idict` was specified, then variables are written to the
dictionary specified, and the updated dictionary is returned.
"""
# Initialize record and variable holders
records = []
if python_dict or idict:
variables = {}
else:
variables = AttrDict()
# Open the IDL file
f = open(file_name, 'rb')
# Read the signature, which should be 'SR'
signature = _read_bytes(f, 2)
if signature != b'SR':
raise Exception("Invalid SIGNATURE: %s" % signature)
# Next, the record format, which is '\x00\x04' for normal .sav
# files, and '\x00\x06' for compressed .sav files.
recfmt = _read_bytes(f, 2)
if recfmt == b'\x00\x04':
pass
elif recfmt == b'\x00\x06':
if verbose:
print("IDL Save file is compressed")
if uncompressed_file_name:
fout = open(uncompressed_file_name, 'w+b')
else:
fout = tempfile.NamedTemporaryFile(suffix='.sav')
if verbose:
print(" -> expanding to %s" % fout.name)
# Write header
fout.write(b'SR\x00\x04')
# Cycle through records
while True:
# Read record type
rectype = _read_long(f)
fout.write(struct.pack('>l', int(rectype)))
# Read position of next record and return as int
nextrec = _read_uint32(f)
nextrec += _read_uint32(f) * 2**32
# Read the unknown 4 bytes
unknown = f.read(4)
# Check if the end of the file has been reached
if RECTYPE_DICT[rectype] == 'END_MARKER':
fout.write(struct.pack('>I', int(nextrec) % 2**32))
fout.write(struct.pack('>I', int((nextrec - (nextrec % 2**32)) / 2**32)))
fout.write(unknown)
break
# Find current position
pos = f.tell()
# Decompress record
rec_string = zlib.decompress(f.read(nextrec-pos))
# Find new position of next record
nextrec = fout.tell() + len(rec_string) + 12
# Write out record
fout.write(struct.pack('>I', int(nextrec % 2**32)))
fout.write(struct.pack('>I', int((nextrec - (nextrec % 2**32)) / 2**32)))
fout.write(unknown)
fout.write(rec_string)
# Close the original compressed file
f.close()
# Set f to be the decompressed file, and skip the first four bytes
f = fout
f.seek(4)
else:
raise Exception("Invalid RECFMT: %s" % recfmt)
# Loop through records, and add them to the list
while True:
r = _read_record(f)
records.append(r)
if 'end' in r:
if r['end']:
break
# Close the file
f.close()
# Find heap data variables
heap = {}
for r in records:
if r['rectype'] == "HEAP_DATA":
heap[r['heap_index']] = r['data']
# Find all variables
for r in records:
if r['rectype'] == "VARIABLE":
replace, new = _replace_heap(r['data'], heap)
if replace:
r['data'] = new
variables[r['varname'].lower()] = r['data']
if verbose:
# Print out timestamp info about the file
for record in records:
if record['rectype'] == "TIMESTAMP":
print("-"*50)
print("Date: %s" % record['date'])
print("User: %s" % record['user'])
print("Host: %s" % record['host'])
break
# Print out version info about the file
for record in records:
if record['rectype'] == "VERSION":
print("-"*50)
print("Format: %s" % record['format'])
print("Architecture: %s" % record['arch'])
print("Operating System: %s" % record['os'])
print("IDL Version: %s" % record['release'])
break
# Print out identification info about the file
for record in records:
if record['rectype'] == "IDENTIFICATON":
print("-"*50)
print("Author: %s" % record['author'])
print("Title: %s" % record['title'])
print("ID Code: %s" % record['idcode'])
break
# Print out descriptions saved with the file
for record in records:
if record['rectype'] == "DESCRIPTION":
print("-"*50)
print("Description: %s" % record['description'])
break
print("-"*50)
print("Successfully read %i records of which:" %
(len(records)))
# Create convenience list of record types
rectypes = [r['rectype'] for r in records]
for rt in set(rectypes):
if rt != 'END_MARKER':
print(" - %i are of type %s" % (rectypes.count(rt), rt))
print("-"*50)
if 'VARIABLE' in rectypes:
print("Available variables:")
for var in variables:
print(" - %s [%s]" % (var, type(variables[var])))
print("-"*50)
if idict:
for var in variables:
idict[var] = variables[var]
return idict
else:
return variables
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__init__.py
-20
View File
@@ -1,20 +0,0 @@
"""
Utilities for dealing with MATLAB(R) files
Notes
-----
MATLAB(R) is a registered trademark of The MathWorks, Inc., 3 Apple Hill
Drive, Natick, MA 01760-2098, USA.
"""
from __future__ import division, print_function, absolute_import
# Matlab file read and write utilities
from .mio import loadmat, savemat, whosmat
from . import byteordercodes
__all__ = ['loadmat', 'savemat', 'whosmat', 'byteordercodes']
from scipy._lib._testutils import PytestTester
test = PytestTester(__name__)
del PytestTester
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/byteordercodes.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/mio.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/mio4.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/mio5.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/mio5_params.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/miobase.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/__pycache__/setup.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/byteordercodes.py
-70
View File
@@ -1,70 +0,0 @@
''' Byteorder utilities for system - numpy byteorder encoding
Converts a variety of string codes for little endian, big endian,
native byte order and swapped byte order to explicit numpy endian
codes - one of '<' (little endian) or '>' (big endian)
'''
from __future__ import division, print_function, absolute_import
import sys
sys_is_le = sys.byteorder == 'little'
native_code = sys_is_le and '<' or '>'
swapped_code = sys_is_le and '>' or '<'
aliases = {'little': ('little', '<', 'l', 'le'),
'big': ('big', '>', 'b', 'be'),
'native': ('native', '='),
'swapped': ('swapped', 'S')}
def to_numpy_code(code):
"""
Convert various order codings to numpy format.
Parameters
----------
code : str
The code to convert. It is converted to lower case before parsing.
Legal values are:
'little', 'big', 'l', 'b', 'le', 'be', '<', '>', 'native', '=',
'swapped', 's'.
Returns
-------
out_code : {'<', '>'}
Here '<' is the numpy dtype code for little endian,
and '>' is the code for big endian.
Examples
--------
>>> import sys
>>> sys_is_le == (sys.byteorder == 'little')
True
>>> to_numpy_code('big')
'>'
>>> to_numpy_code('little')
'<'
>>> nc = to_numpy_code('native')
>>> nc == '<' if sys_is_le else nc == '>'
True
>>> sc = to_numpy_code('swapped')
>>> sc == '>' if sys_is_le else sc == '<'
True
"""
code = code.lower()
if code is None:
return native_code
if code in aliases['little']:
return '<'
elif code in aliases['big']:
return '>'
elif code in aliases['native']:
return native_code
elif code in aliases['swapped']:
return swapped_code
else:
raise ValueError(
'We cannot handle byte order %s' % code)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/mio.py
-326
View File
@@ -1,326 +0,0 @@
"""
Module for reading and writing matlab (TM) .mat files
"""
# Authors: Travis Oliphant, Matthew Brett
from __future__ import division, print_function, absolute_import
from scipy._lib.six import string_types
from .miobase import get_matfile_version, docfiller
from .mio4 import MatFile4Reader, MatFile4Writer
from .mio5 import MatFile5Reader, MatFile5Writer
__all__ = ['mat_reader_factory', 'loadmat', 'savemat', 'whosmat']
def _open_file(file_like, appendmat):
"""
Open `file_like` and return as file-like object. First, check if object is
already file-like; if so, return it as-is. Otherwise, try to pass it
to open(). If that fails, and `file_like` is a string, and `appendmat` is true,
append '.mat' and try again.
"""
try:
file_like.read(0)
return file_like, False
except AttributeError:
pass
try:
return open(file_like, 'rb'), True
except IOError:
# Probably "not found"
if isinstance(file_like, string_types):
if appendmat and not file_like.endswith('.mat'):
file_like += '.mat'
return open(file_like, 'rb'), True
else:
raise IOError('Reader needs file name or open file-like object')
@docfiller
def mat_reader_factory(file_name, appendmat=True, **kwargs):
"""
Create reader for matlab .mat format files.
Parameters
----------
%(file_arg)s
%(append_arg)s
%(load_args)s
%(struct_arg)s
Returns
-------
matreader : MatFileReader object
Initialized instance of MatFileReader class matching the mat file
type detected in `filename`.
file_opened : bool
Whether the file was opened by this routine.
"""
byte_stream, file_opened = _open_file(file_name, appendmat)
mjv, mnv = get_matfile_version(byte_stream)
if mjv == 0:
return MatFile4Reader(byte_stream, **kwargs), file_opened
elif mjv == 1:
return MatFile5Reader(byte_stream, **kwargs), file_opened
elif mjv == 2:
raise NotImplementedError('Please use HDF reader for matlab v7.3 files')
else:
raise TypeError('Did not recognize version %s' % mjv)
@docfiller
def loadmat(file_name, mdict=None, appendmat=True, **kwargs):
"""
Load MATLAB file.
Parameters
----------
file_name : str
Name of the mat file (do not need .mat extension if
appendmat==True). Can also pass open file-like object.
mdict : dict, optional
Dictionary in which to insert matfile variables.
appendmat : bool, optional
True to append the .mat extension to the end of the given
filename, if not already present.
byte_order : str or None, optional
None by default, implying byte order guessed from mat
file. Otherwise can be one of ('native', '=', 'little', '<',
'BIG', '>').
mat_dtype : bool, optional
If True, return arrays in same dtype as would be loaded into
MATLAB (instead of the dtype with which they are saved).
squeeze_me : bool, optional
Whether to squeeze unit matrix dimensions or not.
chars_as_strings : bool, optional
Whether to convert char arrays to string arrays.
matlab_compatible : bool, optional
Returns matrices as would be loaded by MATLAB (implies
squeeze_me=False, chars_as_strings=False, mat_dtype=True,
struct_as_record=True).
struct_as_record : bool, optional
Whether to load MATLAB structs as numpy record arrays, or as
old-style numpy arrays with dtype=object. Setting this flag to
False replicates the behavior of scipy version 0.7.x (returning
numpy object arrays). The default setting is True, because it
allows easier round-trip load and save of MATLAB files.
verify_compressed_data_integrity : bool, optional
Whether the length of compressed sequences in the MATLAB file
should be checked, to ensure that they are not longer than we expect.
It is advisable to enable this (the default) because overlong
compressed sequences in MATLAB files generally indicate that the
files have experienced some sort of corruption.
variable_names : None or sequence
If None (the default) - read all variables in file. Otherwise
`variable_names` should be a sequence of strings, giving names of the
MATLAB variables to read from the file. The reader will skip any
variable with a name not in this sequence, possibly saving some read
processing.
Returns
-------
mat_dict : dict
dictionary with variable names as keys, and loaded matrices as
values.
Notes
-----
v4 (Level 1.0), v6 and v7 to 7.2 matfiles are supported.
You will need an HDF5 python library to read MATLAB 7.3 format mat
files. Because scipy does not supply one, we do not implement the
HDF5 / 7.3 interface here.
Examples
--------
>>> from os.path import dirname, join as pjoin
>>> import scipy.io as sio
Get the filename for an example .mat file from the tests/data directory.
>>> data_dir = pjoin(dirname(sio.__file__), 'matlab', 'tests', 'data')
>>> mat_fname = pjoin(data_dir, 'testdouble_7.4_GLNX86.mat')
Load the .mat file contents.
>>> mat_contents = sio.loadmat(mat_fname)
The result is a dictionary, one key/value pair for each variable:
>>> sorted(mat_contents.keys())
['__globals__', '__header__', '__version__', 'testdouble']
>>> mat_contents['testdouble']
array([[0. , 0.78539816, 1.57079633, 2.35619449, 3.14159265,
3.92699082, 4.71238898, 5.49778714, 6.28318531]])
By default SciPy reads MATLAB structs as structured NumPy arrays where the
dtype fields are of type `object` and the names correspond to the MATLAB
struct field names. This can be disabled by setting the optional argument
`struct_as_record=False`.
Get the filename for an example .mat file that contains a MATLAB struct
called `teststruct` and load the contents.
>>> matstruct_fname = pjoin(data_dir, 'teststruct_7.4_GLNX86.mat')
>>> matstruct_contents = sio.loadmat(matstruct_fname)
>>> teststruct = matstruct_contents['teststruct']
>>> teststruct.dtype
dtype([('stringfield', 'O'), ('doublefield', 'O'), ('complexfield', 'O')])
The size of the structured array is the size of the MATLAB struct, not the
number of elements in any particular field. The shape defaults to 2-D
unless the optional argument `squeeze_me=True`, in which case all length 1
dimensions are removed.
>>> teststruct.size
1
>>> teststruct.shape
(1, 1)
Get the 'stringfield' of the first element in the MATLAB struct.
>>> teststruct[0, 0]['stringfield']
array(['Rats live on no evil star.'],
dtype='<U26')
Get the first element of the 'doublefield'.
>>> teststruct['doublefield'][0, 0]
array([[ 1.41421356, 2.71828183, 3.14159265]])
Load the MATLAB struct, squeezing out length 1 dimensions, and get the item
from the 'complexfield'.
>>> matstruct_squeezed = sio.loadmat(matstruct_fname, squeeze_me=True)
>>> matstruct_squeezed['teststruct'].shape
()
>>> matstruct_squeezed['teststruct']['complexfield'].shape
()
>>> matstruct_squeezed['teststruct']['complexfield'].item()
array([ 1.41421356+1.41421356j, 2.71828183+2.71828183j,
3.14159265+3.14159265j])
"""
variable_names = kwargs.pop('variable_names', None)
MR, file_opened = mat_reader_factory(file_name, appendmat, **kwargs)
matfile_dict = MR.get_variables(variable_names)
if mdict is not None:
mdict.update(matfile_dict)
else:
mdict = matfile_dict
if file_opened:
MR.mat_stream.close()
return mdict
@docfiller
def savemat(file_name, mdict,
appendmat=True,
format='5',
long_field_names=False,
do_compression=False,
oned_as='row'):
"""
Save a dictionary of names and arrays into a MATLAB-style .mat file.
This saves the array objects in the given dictionary to a MATLAB-
style .mat file.
Parameters
----------
file_name : str or file-like object
Name of the .mat file (.mat extension not needed if ``appendmat ==
True``).
Can also pass open file_like object.
mdict : dict
Dictionary from which to save matfile variables.
appendmat : bool, optional
True (the default) to append the .mat extension to the end of the
given filename, if not already present.
format : {'5', '4'}, string, optional
'5' (the default) for MATLAB 5 and up (to 7.2),
'4' for MATLAB 4 .mat files.
long_field_names : bool, optional
False (the default) - maximum field name length in a structure is
31 characters which is the documented maximum length.
True - maximum field name length in a structure is 63 characters
which works for MATLAB 7.6+.
do_compression : bool, optional
Whether or not to compress matrices on write. Default is False.
oned_as : {'row', 'column'}, optional
If 'column', write 1-D numpy arrays as column vectors.
If 'row', write 1-D numpy arrays as row vectors.
See also
--------
mio4.MatFile4Writer
mio5.MatFile5Writer
"""
file_opened = False
if hasattr(file_name, 'write'):
# File-like object already; use as-is
file_stream = file_name
else:
if isinstance(file_name, string_types):
if appendmat and not file_name.endswith('.mat'):
file_name = file_name + ".mat"
file_stream = open(file_name, 'wb')
file_opened = True
if format == '4':
if long_field_names:
raise ValueError("Long field names are not available for version 4 files")
MW = MatFile4Writer(file_stream, oned_as)
elif format == '5':
MW = MatFile5Writer(file_stream,
do_compression=do_compression,
unicode_strings=True,
long_field_names=long_field_names,
oned_as=oned_as)
else:
raise ValueError("Format should be '4' or '5'")
MW.put_variables(mdict)
if file_opened:
file_stream.close()
@docfiller
def whosmat(file_name, appendmat=True, **kwargs):
"""
List variables inside a MATLAB file.
Parameters
----------
%(file_arg)s
%(append_arg)s
%(load_args)s
%(struct_arg)s
Returns
-------
variables : list of tuples
A list of tuples, where each tuple holds the matrix name (a string),
its shape (tuple of ints), and its data class (a string).
Possible data classes are: int8, uint8, int16, uint16, int32, uint32,
int64, uint64, single, double, cell, struct, object, char, sparse,
function, opaque, logical, unknown.
Notes
-----
v4 (Level 1.0), v6 and v7 to 7.2 matfiles are supported.
You will need an HDF5 python library to read matlab 7.3 format mat
files. Because scipy does not supply one, we do not implement the
HDF5 / 7.3 interface here.
.. versionadded:: 0.12.0
"""
ML, file_opened = mat_reader_factory(file_name, **kwargs)
variables = ML.list_variables()
if file_opened:
ML.mat_stream.close()
return variables
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/mio4.py
-618
View File
@@ -1,618 +0,0 @@
''' Classes for read / write of matlab (TM) 4 files
'''
from __future__ import division, print_function, absolute_import
import sys
import warnings
import numpy as np
from numpy.compat import asbytes, asstr
import scipy.sparse
from scipy._lib.six import string_types
from .miobase import (MatFileReader, docfiller, matdims, read_dtype,
convert_dtypes, arr_to_chars, arr_dtype_number)
from .mio_utils import squeeze_element, chars_to_strings
from functools import reduce
SYS_LITTLE_ENDIAN = sys.byteorder == 'little'
miDOUBLE = 0
miSINGLE = 1
miINT32 = 2
miINT16 = 3
miUINT16 = 4
miUINT8 = 5
mdtypes_template = {
miDOUBLE: 'f8',
miSINGLE: 'f4',
miINT32: 'i4',
miINT16: 'i2',
miUINT16: 'u2',
miUINT8: 'u1',
'header': [('mopt', 'i4'),
('mrows', 'i4'),
('ncols', 'i4'),
('imagf', 'i4'),
('namlen', 'i4')],
'U1': 'U1',
}
np_to_mtypes = {
'f8': miDOUBLE,
'c32': miDOUBLE,
'c24': miDOUBLE,
'c16': miDOUBLE,
'f4': miSINGLE,
'c8': miSINGLE,
'i4': miINT32,
'i2': miINT16,
'u2': miUINT16,
'u1': miUINT8,
'S1': miUINT8,
}
# matrix classes
mxFULL_CLASS = 0
mxCHAR_CLASS = 1
mxSPARSE_CLASS = 2
order_codes = {
0: '<',
1: '>',
2: 'VAX D-float', # !
3: 'VAX G-float',
4: 'Cray', # !!
}
mclass_info = {
mxFULL_CLASS: 'double',
mxCHAR_CLASS: 'char',
mxSPARSE_CLASS: 'sparse',
}
class VarHeader4(object):
# Mat4 variables never logical or global
is_logical = False
is_global = False
def __init__(self,
name,
dtype,
mclass,
dims,
is_complex):
self.name = name
self.dtype = dtype
self.mclass = mclass
self.dims = dims
self.is_complex = is_complex
class VarReader4(object):
''' Class to read matlab 4 variables '''
def __init__(self, file_reader):
self.file_reader = file_reader
self.mat_stream = file_reader.mat_stream
self.dtypes = file_reader.dtypes
self.chars_as_strings = file_reader.chars_as_strings
self.squeeze_me = file_reader.squeeze_me
def read_header(self):
''' Read and return header for variable '''
data = read_dtype(self.mat_stream, self.dtypes['header'])
name = self.mat_stream.read(int(data['namlen'])).strip(b'\x00')
if data['mopt'] < 0 or data['mopt'] > 5000:
raise ValueError('Mat 4 mopt wrong format, byteswapping problem?')
M, rest = divmod(data['mopt'], 1000) # order code
if M not in (0, 1):
warnings.warn("We do not support byte ordering '%s'; returned "
"data may be corrupt" % order_codes[M],
UserWarning)
O, rest = divmod(rest, 100) # unused, should be 0
if O != 0:
raise ValueError('O in MOPT integer should be 0, wrong format?')
P, rest = divmod(rest, 10) # data type code e.g miDOUBLE (see above)
T = rest # matrix type code e.g. mxFULL_CLASS (see above)
dims = (data['mrows'], data['ncols'])
is_complex = data['imagf'] == 1
dtype = self.dtypes[P]
return VarHeader4(
name,
dtype,
T,
dims,
is_complex)
def array_from_header(self, hdr, process=True):
mclass = hdr.mclass
if mclass == mxFULL_CLASS:
arr = self.read_full_array(hdr)
elif mclass == mxCHAR_CLASS:
arr = self.read_char_array(hdr)
if process and self.chars_as_strings:
arr = chars_to_strings(arr)
elif mclass == mxSPARSE_CLASS:
# no current processing (below) makes sense for sparse
return self.read_sparse_array(hdr)
else:
raise TypeError('No reader for class code %s' % mclass)
if process and self.squeeze_me:
return squeeze_element(arr)
return arr
def read_sub_array(self, hdr, copy=True):
''' Mat4 read using header `hdr` dtype and dims
Parameters
----------
hdr : object
object with attributes ``dtype``, ``dims``. dtype is assumed to be
the correct endianness
copy : bool, optional
copies array before return if True (default True)
(buffer is usually read only)
Returns
-------
arr : ndarray
of dtype givem by `hdr` ``dtype`` and shape givem by `hdr` ``dims``
'''
dt = hdr.dtype
dims = hdr.dims
num_bytes = dt.itemsize
for d in dims:
num_bytes *= d
buffer = self.mat_stream.read(int(num_bytes))
if len(buffer) != num_bytes:
raise ValueError("Not enough bytes to read matrix '%s'; is this "
"a badly-formed file? Consider listing matrices "
"with `whosmat` and loading named matrices with "
"`variable_names` kwarg to `loadmat`" % hdr.name)
arr = np.ndarray(shape=dims,
dtype=dt,
buffer=buffer,
order='F')
if copy:
arr = arr.copy()
return arr
def read_full_array(self, hdr):
''' Full (rather than sparse) matrix getter
Read matrix (array) can be real or complex
Parameters
----------
hdr : ``VarHeader4`` instance
Returns
-------
arr : ndarray
complex array if ``hdr.is_complex`` is True, otherwise a real
numeric array
'''
if hdr.is_complex:
# avoid array copy to save memory
res = self.read_sub_array(hdr, copy=False)
res_j = self.read_sub_array(hdr, copy=False)
return res + (res_j * 1j)
return self.read_sub_array(hdr)
def read_char_array(self, hdr):
''' latin-1 text matrix (char matrix) reader
Parameters
----------
hdr : ``VarHeader4`` instance
Returns
-------
arr : ndarray
with dtype 'U1', shape given by `hdr` ``dims``
'''
arr = self.read_sub_array(hdr).astype(np.uint8)
S = arr.tostring().decode('latin-1')
return np.ndarray(shape=hdr.dims,
dtype=np.dtype('U1'),
buffer=np.array(S)).copy()
def read_sparse_array(self, hdr):
''' Read and return sparse matrix type
Parameters
----------
hdr : ``VarHeader4`` instance
Returns
-------
arr : ``scipy.sparse.coo_matrix``
with dtype ``float`` and shape read from the sparse matrix data
Notes
-----
MATLAB 4 real sparse arrays are saved in a N+1 by 3 array format, where
N is the number of non-zero values. Column 1 values [0:N] are the
(1-based) row indices of the each non-zero value, column 2 [0:N] are the
column indices, column 3 [0:N] are the (real) values. The last values
[-1,0:2] of the rows, column indices are shape[0] and shape[1]
respectively of the output matrix. The last value for the values column
is a padding 0. mrows and ncols values from the header give the shape of
the stored matrix, here [N+1, 3]. Complex data is saved as a 4 column
matrix, where the fourth column contains the imaginary component; the
last value is again 0. Complex sparse data do *not* have the header
``imagf`` field set to True; the fact that the data are complex is only
detectable because there are 4 storage columns
'''
res = self.read_sub_array(hdr)
tmp = res[:-1,:]
# All numbers are float64 in Matlab, but Scipy sparse expects int shape
dims = (int(res[-1,0]), int(res[-1,1]))
I = np.ascontiguousarray(tmp[:,0],dtype='intc') # fixes byte order also
J = np.ascontiguousarray(tmp[:,1],dtype='intc')
I -= 1 # for 1-based indexing
J -= 1
if res.shape[1] == 3:
V = np.ascontiguousarray(tmp[:,2],dtype='float')
else:
V = np.ascontiguousarray(tmp[:,2],dtype='complex')
V.imag = tmp[:,3]
return scipy.sparse.coo_matrix((V,(I,J)), dims)
def shape_from_header(self, hdr):
'''Read the shape of the array described by the header.
The file position after this call is unspecified.
'''
mclass = hdr.mclass
if mclass == mxFULL_CLASS:
shape = tuple(map(int, hdr.dims))
elif mclass == mxCHAR_CLASS:
shape = tuple(map(int, hdr.dims))
if self.chars_as_strings:
shape = shape[:-1]
elif mclass == mxSPARSE_CLASS:
dt = hdr.dtype
dims = hdr.dims
if not (len(dims) == 2 and dims[0] >= 1 and dims[1] >= 1):
return ()
# Read only the row and column counts
self.mat_stream.seek(dt.itemsize * (dims[0] - 1), 1)
rows = np.ndarray(shape=(1,), dtype=dt,
buffer=self.mat_stream.read(dt.itemsize))
self.mat_stream.seek(dt.itemsize * (dims[0] - 1), 1)
cols = np.ndarray(shape=(1,), dtype=dt,
buffer=self.mat_stream.read(dt.itemsize))
shape = (int(rows), int(cols))
else:
raise TypeError('No reader for class code %s' % mclass)
if self.squeeze_me:
shape = tuple([x for x in shape if x != 1])
return shape
class MatFile4Reader(MatFileReader):
''' Reader for Mat4 files '''
@docfiller
def __init__(self, mat_stream, *args, **kwargs):
''' Initialize matlab 4 file reader
%(matstream_arg)s
%(load_args)s
'''
super(MatFile4Reader, self).__init__(mat_stream, *args, **kwargs)
self._matrix_reader = None
def guess_byte_order(self):
self.mat_stream.seek(0)
mopt = read_dtype(self.mat_stream, np.dtype('i4'))
self.mat_stream.seek(0)
if mopt == 0:
return '<'
if mopt < 0 or mopt > 5000:
# Number must have been byteswapped
return SYS_LITTLE_ENDIAN and '>' or '<'
# Not byteswapped
return SYS_LITTLE_ENDIAN and '<' or '>'
def initialize_read(self):
''' Run when beginning read of variables
Sets up readers from parameters in `self`
'''
self.dtypes = convert_dtypes(mdtypes_template, self.byte_order)
self._matrix_reader = VarReader4(self)
def read_var_header(self):
''' Read and return header, next position
Parameters
----------
None
Returns
-------
header : object
object that can be passed to self.read_var_array, and that
has attributes ``name`` and ``is_global``
next_position : int
position in stream of next variable
'''
hdr = self._matrix_reader.read_header()
n = reduce(lambda x, y: x*y, hdr.dims, 1) # fast product
remaining_bytes = hdr.dtype.itemsize * n
if hdr.is_complex and not hdr.mclass == mxSPARSE_CLASS:
remaining_bytes *= 2
next_position = self.mat_stream.tell() + remaining_bytes
return hdr, next_position
def read_var_array(self, header, process=True):
''' Read array, given `header`
Parameters
----------
header : header object
object with fields defining variable header
process : {True, False}, optional
If True, apply recursive post-processing during loading of array.
Returns
-------
arr : array
array with post-processing applied or not according to
`process`.
'''
return self._matrix_reader.array_from_header(header, process)
def get_variables(self, variable_names=None):
''' get variables from stream as dictionary
Parameters
----------
variable_names : None or str or sequence of str, optional
variable name, or sequence of variable names to get from Mat file /
file stream. If None, then get all variables in file
'''
if isinstance(variable_names, string_types):
variable_names = [variable_names]
elif variable_names is not None:
variable_names = list(variable_names)
self.mat_stream.seek(0)
# set up variable reader
self.initialize_read()
mdict = {}
while not self.end_of_stream():
hdr, next_position = self.read_var_header()
name = asstr(hdr.name)
if variable_names is not None and name not in variable_names:
self.mat_stream.seek(next_position)
continue
mdict[name] = self.read_var_array(hdr)
self.mat_stream.seek(next_position)
if variable_names is not None:
variable_names.remove(name)
if len(variable_names) == 0:
break
return mdict
def list_variables(self):
''' list variables from stream '''
self.mat_stream.seek(0)
# set up variable reader
self.initialize_read()
vars = []
while not self.end_of_stream():
hdr, next_position = self.read_var_header()
name = asstr(hdr.name)
shape = self._matrix_reader.shape_from_header(hdr)
info = mclass_info.get(hdr.mclass, 'unknown')
vars.append((name, shape, info))
self.mat_stream.seek(next_position)
return vars
def arr_to_2d(arr, oned_as='row'):
''' Make ``arr`` exactly two dimensional
If `arr` has more than 2 dimensions, raise a ValueError
Parameters
----------
arr : array
oned_as : {'row', 'column'}, optional
Whether to reshape 1D vectors as row vectors or column vectors.
See documentation for ``matdims`` for more detail
Returns
-------
arr2d : array
2D version of the array
'''
dims = matdims(arr, oned_as)
if len(dims) > 2:
raise ValueError('Matlab 4 files cannot save arrays with more than '
'2 dimensions')
return arr.reshape(dims)
class VarWriter4(object):
def __init__(self, file_writer):
self.file_stream = file_writer.file_stream
self.oned_as = file_writer.oned_as
def write_bytes(self, arr):
self.file_stream.write(arr.tostring(order='F'))
def write_string(self, s):
self.file_stream.write(s)
def write_header(self, name, shape, P=miDOUBLE, T=mxFULL_CLASS, imagf=0):
''' Write header for given data options
Parameters
----------
name : str
name of variable
shape : sequence
Shape of array as it will be read in matlab
P : int, optional
code for mat4 data type, one of ``miDOUBLE, miSINGLE, miINT32,
miINT16, miUINT16, miUINT8``
T : int, optional
code for mat4 matrix class, one of ``mxFULL_CLASS, mxCHAR_CLASS,
mxSPARSE_CLASS``
imagf : int, optional
flag indicating complex
'''
header = np.empty((), mdtypes_template['header'])
M = not SYS_LITTLE_ENDIAN
O = 0
header['mopt'] = (M * 1000 +
O * 100 +
P * 10 +
T)
header['mrows'] = shape[0]
header['ncols'] = shape[1]
header['imagf'] = imagf
header['namlen'] = len(name) + 1
self.write_bytes(header)
self.write_string(asbytes(name + '\0'))
def write(self, arr, name):
''' Write matrix `arr`, with name `name`
Parameters
----------
arr : array_like
array to write
name : str
name in matlab workspace
'''
# we need to catch sparse first, because np.asarray returns an
# an object array for scipy.sparse
if scipy.sparse.issparse(arr):
self.write_sparse(arr, name)
return
arr = np.asarray(arr)
dt = arr.dtype
if not dt.isnative:
arr = arr.astype(dt.newbyteorder('='))
dtt = dt.type
if dtt is np.object_:
raise TypeError('Cannot save object arrays in Mat4')
elif dtt is np.void:
raise TypeError('Cannot save void type arrays')
elif dtt in (np.unicode_, np.string_):
self.write_char(arr, name)
return
self.write_numeric(arr, name)
def write_numeric(self, arr, name):
arr = arr_to_2d(arr, self.oned_as)
imagf = arr.dtype.kind == 'c'
try:
P = np_to_mtypes[arr.dtype.str[1:]]
except KeyError:
if imagf:
arr = arr.astype('c128')
else:
arr = arr.astype('f8')
P = miDOUBLE
self.write_header(name,
arr.shape,
P=P,
T=mxFULL_CLASS,
imagf=imagf)
if imagf:
self.write_bytes(arr.real)
self.write_bytes(arr.imag)
else:
self.write_bytes(arr)
def write_char(self, arr, name):
arr = arr_to_chars(arr)
arr = arr_to_2d(arr, self.oned_as)
dims = arr.shape
self.write_header(
name,
dims,
P=miUINT8,
T=mxCHAR_CLASS)
if arr.dtype.kind == 'U':
# Recode unicode to latin1
n_chars = np.product(dims)
st_arr = np.ndarray(shape=(),
dtype=arr_dtype_number(arr, n_chars),
buffer=arr)
st = st_arr.item().encode('latin-1')
arr = np.ndarray(shape=dims, dtype='S1', buffer=st)
self.write_bytes(arr)
def write_sparse(self, arr, name):
''' Sparse matrices are 2D
See docstring for VarReader4.read_sparse_array
'''
A = arr.tocoo() # convert to sparse COO format (ijv)
imagf = A.dtype.kind == 'c'
ijv = np.zeros((A.nnz + 1, 3+imagf), dtype='f8')
ijv[:-1,0] = A.row
ijv[:-1,1] = A.col
ijv[:-1,0:2] += 1 # 1 based indexing
if imagf:
ijv[:-1,2] = A.data.real
ijv[:-1,3] = A.data.imag
else:
ijv[:-1,2] = A.data
ijv[-1,0:2] = A.shape
self.write_header(
name,
ijv.shape,
P=miDOUBLE,
T=mxSPARSE_CLASS)
self.write_bytes(ijv)
class MatFile4Writer(object):
''' Class for writing matlab 4 format files '''
def __init__(self, file_stream, oned_as=None):
self.file_stream = file_stream
if oned_as is None:
oned_as = 'row'
self.oned_as = oned_as
self._matrix_writer = None
def put_variables(self, mdict, write_header=None):
''' Write variables in `mdict` to stream
Parameters
----------
mdict : mapping
mapping with method ``items`` return name, contents pairs
where ``name`` which will appeak in the matlab workspace in
file load, and ``contents`` is something writeable to a
matlab file, such as a numpy array.
write_header : {None, True, False}
If True, then write the matlab file header before writing the
variables. If None (the default) then write the file header
if we are at position 0 in the stream. By setting False
here, and setting the stream position to the end of the file,
you can append variables to a matlab file
'''
# there is no header for a matlab 4 mat file, so we ignore the
# ``write_header`` input argument. It's there for compatibility
# with the matlab 5 version of this method
self._matrix_writer = VarWriter4(self)
for name, var in mdict.items():
self._matrix_writer.write(var, name)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/mio5.py
-849
View File
@@ -1,849 +0,0 @@
''' Classes for read / write of matlab (TM) 5 files
The matfile specification last found here:
https://www.mathworks.com/access/helpdesk/help/pdf_doc/matlab/matfile_format.pdf
(as of December 5 2008)
'''
from __future__ import division, print_function, absolute_import
'''
=================================
Note on functions and mat files
=================================
The document above does not give any hints as to the storage of matlab
function handles, or anonymous function handles. I had therefore to
guess the format of matlab arrays of ``mxFUNCTION_CLASS`` and
``mxOPAQUE_CLASS`` by looking at example mat files.
``mxFUNCTION_CLASS`` stores all types of matlab functions. It seems to
contain a struct matrix with a set pattern of fields. For anonymous
functions, a sub-fields of one of these fields seems to contain the
well-named ``mxOPAQUE_CLASS``. This seems to contain:
* array flags as for any matlab matrix
* 3 int8 strings
* a matrix
It seems that, whenever the mat file contains a ``mxOPAQUE_CLASS``
instance, there is also an un-named matrix (name == '') at the end of
the mat file. I'll call this the ``__function_workspace__`` matrix.
When I saved two anonymous functions in a mat file, or appended another
anonymous function to the mat file, there was still only one
``__function_workspace__`` un-named matrix at the end, but larger than
that for a mat file with a single anonymous function, suggesting that
the workspaces for the two functions had been merged.
The ``__function_workspace__`` matrix appears to be of double class
(``mxCLASS_DOUBLE``), but stored as uint8, the memory for which is in
the format of a mini .mat file, without the first 124 bytes of the file
header (the description and the subsystem_offset), but with the version
U2 bytes, and the S2 endian test bytes. There follow 4 zero bytes,
presumably for 8 byte padding, and then a series of ``miMATRIX``
entries, as in a standard mat file. The ``miMATRIX`` entries appear to
be series of un-named (name == '') matrices, and may also contain arrays
of this same mini-mat format.
I guess that:
* saving an anonymous function back to a mat file will need the
associated ``__function_workspace__`` matrix saved as well for the
anonymous function to work correctly.
* appending to a mat file that has a ``__function_workspace__`` would
involve first pulling off this workspace, appending, checking whether
there were any more anonymous functions appended, and then somehow
merging the relevant workspaces, and saving at the end of the mat
file.
The mat files I was playing with are in ``tests/data``:
* sqr.mat
* parabola.mat
* some_functions.mat
See ``tests/test_mio.py:test_mio_funcs.py`` for a debugging
script I was working with.
'''
# Small fragments of current code adapted from matfile.py by Heiko
# Henkelmann
import os
import time
import sys
import zlib
from io import BytesIO
import warnings
import numpy as np
from numpy.compat import asbytes, asstr
import scipy.sparse
from scipy._lib.six import string_types
from .byteordercodes import native_code, swapped_code
from .miobase import (MatFileReader, docfiller, matdims, read_dtype,
arr_to_chars, arr_dtype_number, MatWriteError,
MatReadError, MatReadWarning)
# Reader object for matlab 5 format variables
from .mio5_utils import VarReader5
# Constants and helper objects
from .mio5_params import (MatlabObject, MatlabFunction, MDTYPES, NP_TO_MTYPES,
NP_TO_MXTYPES, miCOMPRESSED, miMATRIX, miINT8,
miUTF8, miUINT32, mxCELL_CLASS, mxSTRUCT_CLASS,
mxOBJECT_CLASS, mxCHAR_CLASS, mxSPARSE_CLASS,
mxDOUBLE_CLASS, mclass_info)
from .streams import ZlibInputStream
class MatFile5Reader(MatFileReader):
''' Reader for Mat 5 mat files
Adds the following attribute to base class
uint16_codec - char codec to use for uint16 char arrays
(defaults to system default codec)
Uses variable reader that has the following stardard interface (see
abstract class in ``miobase``::
__init__(self, file_reader)
read_header(self)
array_from_header(self)
and added interface::
set_stream(self, stream)
read_full_tag(self)
'''
@docfiller
def __init__(self,
mat_stream,
byte_order=None,
mat_dtype=False,
squeeze_me=False,
chars_as_strings=True,
matlab_compatible=False,
struct_as_record=True,
verify_compressed_data_integrity=True,
uint16_codec=None
):
'''Initializer for matlab 5 file format reader
%(matstream_arg)s
%(load_args)s
%(struct_arg)s
uint16_codec : {None, string}
Set codec to use for uint16 char arrays (e.g. 'utf-8').
Use system default codec if None
'''
super(MatFile5Reader, self).__init__(
mat_stream,
byte_order,
mat_dtype,
squeeze_me,
chars_as_strings,
matlab_compatible,
struct_as_record,
verify_compressed_data_integrity
)
# Set uint16 codec
if not uint16_codec:
uint16_codec = sys.getdefaultencoding()
self.uint16_codec = uint16_codec
# placeholders for readers - see initialize_read method
self._file_reader = None
self._matrix_reader = None
def guess_byte_order(self):
''' Guess byte order.
Sets stream pointer to 0 '''
self.mat_stream.seek(126)
mi = self.mat_stream.read(2)
self.mat_stream.seek(0)
return mi == b'IM' and '<' or '>'
def read_file_header(self):
''' Read in mat 5 file header '''
hdict = {}
hdr_dtype = MDTYPES[self.byte_order]['dtypes']['file_header']
hdr = read_dtype(self.mat_stream, hdr_dtype)
hdict['__header__'] = hdr['description'].item().strip(b' \t\n\000')
v_major = hdr['version'] >> 8
v_minor = hdr['version'] & 0xFF
hdict['__version__'] = '%d.%d' % (v_major, v_minor)
return hdict
def initialize_read(self):
''' Run when beginning read of variables
Sets up readers from parameters in `self`
'''
# reader for top level stream. We need this extra top-level
# reader because we use the matrix_reader object to contain
# compressed matrices (so they have their own stream)
self._file_reader = VarReader5(self)
# reader for matrix streams
self._matrix_reader = VarReader5(self)
def read_var_header(self):
''' Read header, return header, next position
Header has to define at least .name and .is_global
Parameters
----------
None
Returns
-------
header : object
object that can be passed to self.read_var_array, and that
has attributes .name and .is_global
next_position : int
position in stream of next variable
'''
mdtype, byte_count = self._file_reader.read_full_tag()
if not byte_count > 0:
raise ValueError("Did not read any bytes")
next_pos = self.mat_stream.tell() + byte_count
if mdtype == miCOMPRESSED:
# Make new stream from compressed data
stream = ZlibInputStream(self.mat_stream, byte_count)
self._matrix_reader.set_stream(stream)
check_stream_limit = self.verify_compressed_data_integrity
mdtype, byte_count = self._matrix_reader.read_full_tag()
else:
check_stream_limit = False
self._matrix_reader.set_stream(self.mat_stream)
if not mdtype == miMATRIX:
raise TypeError('Expecting miMATRIX type here, got %d' % mdtype)
header = self._matrix_reader.read_header(check_stream_limit)
return header, next_pos
def read_var_array(self, header, process=True):
''' Read array, given `header`
Parameters
----------
header : header object
object with fields defining variable header
process : {True, False} bool, optional
If True, apply recursive post-processing during loading of
array.
Returns
-------
arr : array
array with post-processing applied or not according to
`process`.
'''
return self._matrix_reader.array_from_header(header, process)
def get_variables(self, variable_names=None):
''' get variables from stream as dictionary
variable_names - optional list of variable names to get
If variable_names is None, then get all variables in file
'''
if isinstance(variable_names, string_types):
variable_names = [variable_names]
elif variable_names is not None:
variable_names = list(variable_names)
self.mat_stream.seek(0)
# Here we pass all the parameters in self to the reading objects
self.initialize_read()
mdict = self.read_file_header()
mdict['__globals__'] = []
while not self.end_of_stream():
hdr, next_position = self.read_var_header()
name = asstr(hdr.name)
if name in mdict:
warnings.warn('Duplicate variable name "%s" in stream'
' - replacing previous with new\n'
'Consider mio5.varmats_from_mat to split '
'file into single variable files' % name,
MatReadWarning, stacklevel=2)
if name == '':
# can only be a matlab 7 function workspace
name = '__function_workspace__'
# We want to keep this raw because mat_dtype processing
# will break the format (uint8 as mxDOUBLE_CLASS)
process = False
else:
process = True
if variable_names is not None and name not in variable_names:
self.mat_stream.seek(next_position)
continue
try:
res = self.read_var_array(hdr, process)
except MatReadError as err:
warnings.warn(
'Unreadable variable "%s", because "%s"' %
(name, err),
Warning, stacklevel=2)
res = "Read error: %s" % err
self.mat_stream.seek(next_position)
mdict[name] = res
if hdr.is_global:
mdict['__globals__'].append(name)
if variable_names is not None:
variable_names.remove(name)
if len(variable_names) == 0:
break
return mdict
def list_variables(self):
''' list variables from stream '''
self.mat_stream.seek(0)
# Here we pass all the parameters in self to the reading objects
self.initialize_read()
self.read_file_header()
vars = []
while not self.end_of_stream():
hdr, next_position = self.read_var_header()
name = asstr(hdr.name)
if name == '':
# can only be a matlab 7 function workspace
name = '__function_workspace__'
shape = self._matrix_reader.shape_from_header(hdr)
if hdr.is_logical:
info = 'logical'
else:
info = mclass_info.get(hdr.mclass, 'unknown')
vars.append((name, shape, info))
self.mat_stream.seek(next_position)
return vars
def varmats_from_mat(file_obj):
""" Pull variables out of mat 5 file as a sequence of mat file objects
This can be useful with a difficult mat file, containing unreadable
variables. This routine pulls the variables out in raw form and puts them,
unread, back into a file stream for saving or reading. Another use is the
pathological case where there is more than one variable of the same name in
the file; this routine returns the duplicates, whereas the standard reader
will overwrite duplicates in the returned dictionary.
The file pointer in `file_obj` will be undefined. File pointers for the
returned file-like objects are set at 0.
Parameters
----------
file_obj : file-like
file object containing mat file
Returns
-------
named_mats : list
list contains tuples of (name, BytesIO) where BytesIO is a file-like
object containing mat file contents as for a single variable. The
BytesIO contains a string with the original header and a single var. If
``var_file_obj`` is an individual BytesIO instance, then save as a mat
file with something like ``open('test.mat',
'wb').write(var_file_obj.read())``
Examples
--------
>>> import scipy.io
BytesIO is from the ``io`` module in python 3, and is ``cStringIO`` for
python < 3.
>>> mat_fileobj = BytesIO()
>>> scipy.io.savemat(mat_fileobj, {'b': np.arange(10), 'a': 'a string'})
>>> varmats = varmats_from_mat(mat_fileobj)
>>> sorted([name for name, str_obj in varmats])
['a', 'b']
"""
rdr = MatFile5Reader(file_obj)
file_obj.seek(0)
# Raw read of top-level file header
hdr_len = MDTYPES[native_code]['dtypes']['file_header'].itemsize
raw_hdr = file_obj.read(hdr_len)
# Initialize variable reading
file_obj.seek(0)
rdr.initialize_read()
mdict = rdr.read_file_header()
next_position = file_obj.tell()
named_mats = []
while not rdr.end_of_stream():
start_position = next_position
hdr, next_position = rdr.read_var_header()
name = asstr(hdr.name)
# Read raw variable string
file_obj.seek(start_position)
byte_count = next_position - start_position
var_str = file_obj.read(byte_count)
# write to stringio object
out_obj = BytesIO()
out_obj.write(raw_hdr)
out_obj.write(var_str)
out_obj.seek(0)
named_mats.append((name, out_obj))
return named_mats
class EmptyStructMarker(object):
""" Class to indicate presence of empty matlab struct on output """
def to_writeable(source):
''' Convert input object ``source`` to something we can write
Parameters
----------
source : object
Returns
-------
arr : None or ndarray or EmptyStructMarker
If `source` cannot be converted to something we can write to a matfile,
return None. If `source` is equivalent to an empty dictionary, return
``EmptyStructMarker``. Otherwise return `source` converted to an
ndarray with contents for writing to matfile.
'''
if isinstance(source, np.ndarray):
return source
if source is None:
return None
# Objects that implement mappings
is_mapping = (hasattr(source, 'keys') and hasattr(source, 'values') and
hasattr(source, 'items'))
# Objects that don't implement mappings, but do have dicts
if isinstance(source, np.generic):
# Numpy scalars are never mappings (pypy issue workaround)
pass
elif not is_mapping and hasattr(source, '__dict__'):
source = dict((key, value) for key, value in source.__dict__.items()
if not key.startswith('_'))
is_mapping = True
if is_mapping:
dtype = []
values = []
for field, value in source.items():
if (isinstance(field, string_types) and
field[0] not in '_0123456789'):
dtype.append((str(field), object))
values.append(value)
if dtype:
return np.array([tuple(values)], dtype)
else:
return EmptyStructMarker
# Next try and convert to an array
narr = np.asanyarray(source)
if narr.dtype.type in (object, np.object_) and \
narr.shape == () and narr == source:
# No interesting conversion possible
return None
return narr
# Native byte ordered dtypes for convenience for writers
NDT_FILE_HDR = MDTYPES[native_code]['dtypes']['file_header']
NDT_TAG_FULL = MDTYPES[native_code]['dtypes']['tag_full']
NDT_TAG_SMALL = MDTYPES[native_code]['dtypes']['tag_smalldata']
NDT_ARRAY_FLAGS = MDTYPES[native_code]['dtypes']['array_flags']
class VarWriter5(object):
''' Generic matlab matrix writing class '''
mat_tag = np.zeros((), NDT_TAG_FULL)
mat_tag['mdtype'] = miMATRIX
def __init__(self, file_writer):
self.file_stream = file_writer.file_stream
self.unicode_strings = file_writer.unicode_strings
self.long_field_names = file_writer.long_field_names
self.oned_as = file_writer.oned_as
# These are used for top level writes, and unset after
self._var_name = None
self._var_is_global = False
def write_bytes(self, arr):
self.file_stream.write(arr.tostring(order='F'))
def write_string(self, s):
self.file_stream.write(s)
def write_element(self, arr, mdtype=None):
''' write tag and data '''
if mdtype is None:
mdtype = NP_TO_MTYPES[arr.dtype.str[1:]]
# Array needs to be in native byte order
if arr.dtype.byteorder == swapped_code:
arr = arr.byteswap().newbyteorder()
byte_count = arr.size*arr.itemsize
if byte_count <= 4:
self.write_smalldata_element(arr, mdtype, byte_count)
else:
self.write_regular_element(arr, mdtype, byte_count)
def write_smalldata_element(self, arr, mdtype, byte_count):
# write tag with embedded data
tag = np.zeros((), NDT_TAG_SMALL)
tag['byte_count_mdtype'] = (byte_count << 16) + mdtype
# if arr.tostring is < 4, the element will be zero-padded as needed.
tag['data'] = arr.tostring(order='F')
self.write_bytes(tag)
def write_regular_element(self, arr, mdtype, byte_count):
# write tag, data
tag = np.zeros((), NDT_TAG_FULL)
tag['mdtype'] = mdtype
tag['byte_count'] = byte_count
self.write_bytes(tag)
self.write_bytes(arr)
# pad to next 64-bit boundary
bc_mod_8 = byte_count % 8
if bc_mod_8:
self.file_stream.write(b'\x00' * (8-bc_mod_8))
def write_header(self,
shape,
mclass,
is_complex=False,
is_logical=False,
nzmax=0):
''' Write header for given data options
shape : sequence
array shape
mclass - mat5 matrix class
is_complex - True if matrix is complex
is_logical - True if matrix is logical
nzmax - max non zero elements for sparse arrays
We get the name and the global flag from the object, and reset
them to defaults after we've used them
'''
# get name and is_global from one-shot object store
name = self._var_name
is_global = self._var_is_global
# initialize the top-level matrix tag, store position
self._mat_tag_pos = self.file_stream.tell()
self.write_bytes(self.mat_tag)
# write array flags (complex, global, logical, class, nzmax)
af = np.zeros((), NDT_ARRAY_FLAGS)
af['data_type'] = miUINT32
af['byte_count'] = 8
flags = is_complex << 3 | is_global << 2 | is_logical << 1
af['flags_class'] = mclass | flags << 8
af['nzmax'] = nzmax
self.write_bytes(af)
# shape
self.write_element(np.array(shape, dtype='i4'))
# write name
name = np.asarray(name)
if name == '': # empty string zero-terminated
self.write_smalldata_element(name, miINT8, 0)
else:
self.write_element(name, miINT8)
# reset the one-shot store to defaults
self._var_name = ''
self._var_is_global = False
def update_matrix_tag(self, start_pos):
curr_pos = self.file_stream.tell()
self.file_stream.seek(start_pos)
byte_count = curr_pos - start_pos - 8
if byte_count >= 2**32:
raise MatWriteError("Matrix too large to save with Matlab "
"5 format")
self.mat_tag['byte_count'] = byte_count
self.write_bytes(self.mat_tag)
self.file_stream.seek(curr_pos)
def write_top(self, arr, name, is_global):
""" Write variable at top level of mat file
Parameters
----------
arr : array_like
array-like object to create writer for
name : str, optional
name as it will appear in matlab workspace
default is empty string
is_global : {False, True}, optional
whether variable will be global on load into matlab
"""
# these are set before the top-level header write, and unset at
# the end of the same write, because they do not apply for lower levels
self._var_is_global = is_global
self._var_name = name
# write the header and data
self.write(arr)
def write(self, arr):
''' Write `arr` to stream at top and sub levels
Parameters
----------
arr : array_like
array-like object to create writer for
'''
# store position, so we can update the matrix tag
mat_tag_pos = self.file_stream.tell()
# First check if these are sparse
if scipy.sparse.issparse(arr):
self.write_sparse(arr)
self.update_matrix_tag(mat_tag_pos)
return
# Try to convert things that aren't arrays
narr = to_writeable(arr)
if narr is None:
raise TypeError('Could not convert %s (type %s) to array'
% (arr, type(arr)))
if isinstance(narr, MatlabObject):
self.write_object(narr)
elif isinstance(narr, MatlabFunction):
raise MatWriteError('Cannot write matlab functions')
elif narr is EmptyStructMarker: # empty struct array
self.write_empty_struct()
elif narr.dtype.fields: # struct array
self.write_struct(narr)
elif narr.dtype.hasobject: # cell array
self.write_cells(narr)
elif narr.dtype.kind in ('U', 'S'):
if self.unicode_strings:
codec = 'UTF8'
else:
codec = 'ascii'
self.write_char(narr, codec)
else:
self.write_numeric(narr)
self.update_matrix_tag(mat_tag_pos)
def write_numeric(self, arr):
imagf = arr.dtype.kind == 'c'
logif = arr.dtype.kind == 'b'
try:
mclass = NP_TO_MXTYPES[arr.dtype.str[1:]]
except KeyError:
# No matching matlab type, probably complex256 / float128 / float96
# Cast data to complex128 / float64.
if imagf:
arr = arr.astype('c128')
elif logif:
arr = arr.astype('i1') # Should only contain 0/1
else:
arr = arr.astype('f8')
mclass = mxDOUBLE_CLASS
self.write_header(matdims(arr, self.oned_as),
mclass,
is_complex=imagf,
is_logical=logif)
if imagf:
self.write_element(arr.real)
self.write_element(arr.imag)
else:
self.write_element(arr)
def write_char(self, arr, codec='ascii'):
''' Write string array `arr` with given `codec`
'''
if arr.size == 0 or np.all(arr == ''):
# This an empty string array or a string array containing
# only empty strings. Matlab cannot distinguish between a
# string array that is empty, and a string array containing
# only empty strings, because it stores strings as arrays of
# char. There is no way of having an array of char that is
# not empty, but contains an empty string. We have to
# special-case the array-with-empty-strings because even
# empty strings have zero padding, which would otherwise
# appear in matlab as a string with a space.
shape = (0,) * np.max([arr.ndim, 2])
self.write_header(shape, mxCHAR_CLASS)
self.write_smalldata_element(arr, miUTF8, 0)
return
# non-empty string.
#
# Convert to char array
arr = arr_to_chars(arr)
# We have to write the shape directly, because we are going
# recode the characters, and the resulting stream of chars
# may have a different length
shape = arr.shape
self.write_header(shape, mxCHAR_CLASS)
if arr.dtype.kind == 'U' and arr.size:
# Make one long string from all the characters. We need to
# transpose here, because we're flattening the array, before
# we write the bytes. The bytes have to be written in
# Fortran order.
n_chars = np.product(shape)
st_arr = np.ndarray(shape=(),
dtype=arr_dtype_number(arr, n_chars),
buffer=arr.T.copy()) # Fortran order
# Recode with codec to give byte string
st = st_arr.item().encode(codec)
# Reconstruct as one-dimensional byte array
arr = np.ndarray(shape=(len(st),),
dtype='S1',
buffer=st)
self.write_element(arr, mdtype=miUTF8)
def write_sparse(self, arr):
''' Sparse matrices are 2D
'''
A = arr.tocsc() # convert to sparse CSC format
A.sort_indices() # MATLAB expects sorted row indices
is_complex = (A.dtype.kind == 'c')
is_logical = (A.dtype.kind == 'b')
nz = A.nnz
self.write_header(matdims(arr, self.oned_as),
mxSPARSE_CLASS,
is_complex=is_complex,
is_logical=is_logical,
# matlab won't load file with 0 nzmax
nzmax=1 if nz == 0 else nz)
self.write_element(A.indices.astype('i4'))
self.write_element(A.indptr.astype('i4'))
self.write_element(A.data.real)
if is_complex:
self.write_element(A.data.imag)
def write_cells(self, arr):
self.write_header(matdims(arr, self.oned_as),
mxCELL_CLASS)
# loop over data, column major
A = np.atleast_2d(arr).flatten('F')
for el in A:
self.write(el)
def write_empty_struct(self):
self.write_header((1, 1), mxSTRUCT_CLASS)
# max field name length set to 1 in an example matlab struct
self.write_element(np.array(1, dtype=np.int32))
# Field names element is empty
self.write_element(np.array([], dtype=np.int8))
def write_struct(self, arr):
self.write_header(matdims(arr, self.oned_as),
mxSTRUCT_CLASS)
self._write_items(arr)
def _write_items(self, arr):
# write fieldnames
fieldnames = [f[0] for f in arr.dtype.descr]
length = max([len(fieldname) for fieldname in fieldnames])+1
max_length = (self.long_field_names and 64) or 32
if length > max_length:
raise ValueError("Field names are restricted to %d characters" %
(max_length-1))
self.write_element(np.array([length], dtype='i4'))
self.write_element(
np.array(fieldnames, dtype='S%d' % (length)),
mdtype=miINT8)
A = np.atleast_2d(arr).flatten('F')
for el in A:
for f in fieldnames:
self.write(el[f])
def write_object(self, arr):
'''Same as writing structs, except different mx class, and extra
classname element after header
'''
self.write_header(matdims(arr, self.oned_as),
mxOBJECT_CLASS)
self.write_element(np.array(arr.classname, dtype='S'),
mdtype=miINT8)
self._write_items(arr)
class MatFile5Writer(object):
''' Class for writing mat5 files '''
@docfiller
def __init__(self, file_stream,
do_compression=False,
unicode_strings=False,
global_vars=None,
long_field_names=False,
oned_as='row'):
''' Initialize writer for matlab 5 format files
Parameters
----------
%(do_compression)s
%(unicode_strings)s
global_vars : None or sequence of strings, optional
Names of variables to be marked as global for matlab
%(long_fields)s
%(oned_as)s
'''
self.file_stream = file_stream
self.do_compression = do_compression
self.unicode_strings = unicode_strings
if global_vars:
self.global_vars = global_vars
else:
self.global_vars = []
self.long_field_names = long_field_names
self.oned_as = oned_as
self._matrix_writer = None
def write_file_header(self):
# write header
hdr = np.zeros((), NDT_FILE_HDR)
hdr['description'] = 'MATLAB 5.0 MAT-file Platform: %s, Created on: %s' \
% (os.name,time.asctime())
hdr['version'] = 0x0100
hdr['endian_test'] = np.ndarray(shape=(),
dtype='S2',
buffer=np.uint16(0x4d49))
self.file_stream.write(hdr.tostring())
def put_variables(self, mdict, write_header=None):
''' Write variables in `mdict` to stream
Parameters
----------
mdict : mapping
mapping with method ``items`` returns name, contents pairs where
``name`` which will appear in the matlab workspace in file load, and
``contents`` is something writeable to a matlab file, such as a numpy
array.
write_header : {None, True, False}, optional
If True, then write the matlab file header before writing the
variables. If None (the default) then write the file header
if we are at position 0 in the stream. By setting False
here, and setting the stream position to the end of the file,
you can append variables to a matlab file
'''
# write header if requested, or None and start of file
if write_header is None:
write_header = self.file_stream.tell() == 0
if write_header:
self.write_file_header()
self._matrix_writer = VarWriter5(self)
for name, var in mdict.items():
if name[0] == '_':
continue
is_global = name in self.global_vars
if self.do_compression:
stream = BytesIO()
self._matrix_writer.file_stream = stream
self._matrix_writer.write_top(var, asbytes(name), is_global)
out_str = zlib.compress(stream.getvalue())
tag = np.empty((), NDT_TAG_FULL)
tag['mdtype'] = miCOMPRESSED
tag['byte_count'] = len(out_str)
self.file_stream.write(tag.tostring())
self.file_stream.write(out_str)
else: # not compressing
self._matrix_writer.write_top(var, asbytes(name), is_global)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/mio5_params.py
-254
View File
@@ -1,254 +0,0 @@
''' Constants and classes for matlab 5 read and write
See also mio5_utils.pyx where these same constants arise as c enums.
If you make changes in this file, don't forget to change mio5_utils.pyx
'''
from __future__ import division, print_function, absolute_import
import numpy as np
from .miobase import convert_dtypes
miINT8 = 1
miUINT8 = 2
miINT16 = 3
miUINT16 = 4
miINT32 = 5
miUINT32 = 6
miSINGLE = 7
miDOUBLE = 9
miINT64 = 12
miUINT64 = 13
miMATRIX = 14
miCOMPRESSED = 15
miUTF8 = 16
miUTF16 = 17
miUTF32 = 18
mxCELL_CLASS = 1
mxSTRUCT_CLASS = 2
# The March 2008 edition of "Matlab 7 MAT-File Format" says that
# mxOBJECT_CLASS = 3, whereas matrix.h says that mxLOGICAL = 3.
# Matlab 2008a appears to save logicals as type 9, so we assume that
# the document is correct. See type 18, below.
mxOBJECT_CLASS = 3
mxCHAR_CLASS = 4
mxSPARSE_CLASS = 5
mxDOUBLE_CLASS = 6
mxSINGLE_CLASS = 7
mxINT8_CLASS = 8
mxUINT8_CLASS = 9
mxINT16_CLASS = 10
mxUINT16_CLASS = 11
mxINT32_CLASS = 12
mxUINT32_CLASS = 13
# The following are not in the March 2008 edition of "Matlab 7
# MAT-File Format," but were guessed from matrix.h.
mxINT64_CLASS = 14
mxUINT64_CLASS = 15
mxFUNCTION_CLASS = 16
# Not doing anything with these at the moment.
mxOPAQUE_CLASS = 17 # This appears to be a function workspace
# Thread 'saveing/loading symbol table of annymous functions', octave-maintainers, April-May 2007
# https://lists.gnu.org/archive/html/octave-maintainers/2007-04/msg00031.html
# https://lists.gnu.org/archive/html/octave-maintainers/2007-05/msg00032.html
# (Was/Deprecated: https://www-old.cae.wisc.edu/pipermail/octave-maintainers/2007-May/002824.html)
mxOBJECT_CLASS_FROM_MATRIX_H = 18
mdtypes_template = {
miINT8: 'i1',
miUINT8: 'u1',
miINT16: 'i2',
miUINT16: 'u2',
miINT32: 'i4',
miUINT32: 'u4',
miSINGLE: 'f4',
miDOUBLE: 'f8',
miINT64: 'i8',
miUINT64: 'u8',
miUTF8: 'u1',
miUTF16: 'u2',
miUTF32: 'u4',
'file_header': [('description', 'S116'),
('subsystem_offset', 'i8'),
('version', 'u2'),
('endian_test', 'S2')],
'tag_full': [('mdtype', 'u4'), ('byte_count', 'u4')],
'tag_smalldata':[('byte_count_mdtype', 'u4'), ('data', 'S4')],
'array_flags': [('data_type', 'u4'),
('byte_count', 'u4'),
('flags_class','u4'),
('nzmax', 'u4')],
'U1': 'U1',
}
mclass_dtypes_template = {
mxINT8_CLASS: 'i1',
mxUINT8_CLASS: 'u1',
mxINT16_CLASS: 'i2',
mxUINT16_CLASS: 'u2',
mxINT32_CLASS: 'i4',
mxUINT32_CLASS: 'u4',
mxINT64_CLASS: 'i8',
mxUINT64_CLASS: 'u8',
mxSINGLE_CLASS: 'f4',
mxDOUBLE_CLASS: 'f8',
}
mclass_info = {
mxINT8_CLASS: 'int8',
mxUINT8_CLASS: 'uint8',
mxINT16_CLASS: 'int16',
mxUINT16_CLASS: 'uint16',
mxINT32_CLASS: 'int32',
mxUINT32_CLASS: 'uint32',
mxINT64_CLASS: 'int64',
mxUINT64_CLASS: 'uint64',
mxSINGLE_CLASS: 'single',
mxDOUBLE_CLASS: 'double',
mxCELL_CLASS: 'cell',
mxSTRUCT_CLASS: 'struct',
mxOBJECT_CLASS: 'object',
mxCHAR_CLASS: 'char',
mxSPARSE_CLASS: 'sparse',
mxFUNCTION_CLASS: 'function',
mxOPAQUE_CLASS: 'opaque',
}
NP_TO_MTYPES = {
'f8': miDOUBLE,
'c32': miDOUBLE,
'c24': miDOUBLE,
'c16': miDOUBLE,
'f4': miSINGLE,
'c8': miSINGLE,
'i8': miINT64,
'i4': miINT32,
'i2': miINT16,
'i1': miINT8,
'u8': miUINT64,
'u4': miUINT32,
'u2': miUINT16,
'u1': miUINT8,
'S1': miUINT8,
'U1': miUTF16,
'b1': miUINT8, # not standard but seems MATLAB uses this (gh-4022)
}
NP_TO_MXTYPES = {
'f8': mxDOUBLE_CLASS,
'c32': mxDOUBLE_CLASS,
'c24': mxDOUBLE_CLASS,
'c16': mxDOUBLE_CLASS,
'f4': mxSINGLE_CLASS,
'c8': mxSINGLE_CLASS,
'i8': mxINT64_CLASS,
'i4': mxINT32_CLASS,
'i2': mxINT16_CLASS,
'i1': mxINT8_CLASS,
'u8': mxUINT64_CLASS,
'u4': mxUINT32_CLASS,
'u2': mxUINT16_CLASS,
'u1': mxUINT8_CLASS,
'S1': mxUINT8_CLASS,
'b1': mxUINT8_CLASS, # not standard but seems MATLAB uses this
}
''' Before release v7.1 (release 14) matlab (TM) used the system
default character encoding scheme padded out to 16-bits. Release 14
and later use Unicode. When saving character data, R14 checks if it
can be encoded in 7-bit ascii, and saves in that format if so.'''
codecs_template = {
miUTF8: {'codec': 'utf_8', 'width': 1},
miUTF16: {'codec': 'utf_16', 'width': 2},
miUTF32: {'codec': 'utf_32','width': 4},
}
def _convert_codecs(template, byte_order):
''' Convert codec template mapping to byte order
Set codecs not on this system to None
Parameters
----------
template : mapping
key, value are respectively codec name, and root name for codec
(without byte order suffix)
byte_order : {'<', '>'}
code for little or big endian
Returns
-------
codecs : dict
key, value are name, codec (as in .encode(codec))
'''
codecs = {}
postfix = byte_order == '<' and '_le' or '_be'
for k, v in template.items():
codec = v['codec']
try:
" ".encode(codec)
except LookupError:
codecs[k] = None
continue
if v['width'] > 1:
codec += postfix
codecs[k] = codec
return codecs.copy()
MDTYPES = {}
for _bytecode in '<>':
_def = {'dtypes': convert_dtypes(mdtypes_template, _bytecode),
'classes': convert_dtypes(mclass_dtypes_template, _bytecode),
'codecs': _convert_codecs(codecs_template, _bytecode)}
MDTYPES[_bytecode] = _def
class mat_struct(object):
''' Placeholder for holding read data from structs
We use instances of this class when the user passes False as a value to the
``struct_as_record`` parameter of the :func:`scipy.io.matlab.loadmat`
function.
'''
pass
class MatlabObject(np.ndarray):
''' ndarray Subclass to contain matlab object '''
def __new__(cls, input_array, classname=None):
# Input array is an already formed ndarray instance
# We first cast to be our class type
obj = np.asarray(input_array).view(cls)
# add the new attribute to the created instance
obj.classname = classname
# Finally, we must return the newly created object:
return obj
def __array_finalize__(self,obj):
# reset the attribute from passed original object
self.classname = getattr(obj, 'classname', None)
# We do not need to return anything
class MatlabFunction(np.ndarray):
''' Subclass to signal this is a matlab function '''
def __new__(cls, input_array):
obj = np.asarray(input_array).view(cls)
return obj
class MatlabOpaque(np.ndarray):
''' Subclass to signal this is a matlab opaque matrix '''
def __new__(cls, input_array):
obj = np.asarray(input_array).view(cls)
return obj
OPAQUE_DTYPE = np.dtype(
[('s0', 'O'), ('s1', 'O'), ('s2', 'O'), ('arr', 'O')])
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/mio5_utils.cpython-36m-x86_64-linux-gnu.so
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/mio_utils.cpython-36m-x86_64-linux-gnu.so
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/miobase.py
-415
View File
@@ -1,415 +0,0 @@
# Authors: Travis Oliphant, Matthew Brett
"""
Base classes for MATLAB file stream reading.
MATLAB is a registered trademark of the Mathworks inc.
"""
from __future__ import division, print_function, absolute_import
import sys
import operator
from scipy._lib.six import reduce
import numpy as np
if sys.version_info[0] >= 3:
byteord = int
else:
byteord = ord
from scipy.misc import doccer
from . import byteordercodes as boc
class MatReadError(Exception):
pass
class MatWriteError(Exception):
pass
class MatReadWarning(UserWarning):
pass
doc_dict = \
{'file_arg':
'''file_name : str
Name of the mat file (do not need .mat extension if
appendmat==True) Can also pass open file-like object.''',
'append_arg':
'''appendmat : bool, optional
True to append the .mat extension to the end of the given
filename, if not already present.''',
'load_args':
'''byte_order : str or None, optional
None by default, implying byte order guessed from mat
file. Otherwise can be one of ('native', '=', 'little', '<',
'BIG', '>').
mat_dtype : bool, optional
If True, return arrays in same dtype as would be loaded into
MATLAB (instead of the dtype with which they are saved).
squeeze_me : bool, optional
Whether to squeeze unit matrix dimensions or not.
chars_as_strings : bool, optional
Whether to convert char arrays to string arrays.
matlab_compatible : bool, optional
Returns matrices as would be loaded by MATLAB (implies
squeeze_me=False, chars_as_strings=False, mat_dtype=True,
struct_as_record=True).''',
'struct_arg':
'''struct_as_record : bool, optional
Whether to load MATLAB structs as numpy record arrays, or as
old-style numpy arrays with dtype=object. Setting this flag to
False replicates the behavior of scipy version 0.7.x (returning
numpy object arrays). The default setting is True, because it
allows easier round-trip load and save of MATLAB files.''',
'matstream_arg':
'''mat_stream : file-like
Object with file API, open for reading.''',
'long_fields':
'''long_field_names : bool, optional
* False - maximum field name length in a structure is 31 characters
which is the documented maximum length. This is the default.
* True - maximum field name length in a structure is 63 characters
which works for MATLAB 7.6''',
'do_compression':
'''do_compression : bool, optional
Whether to compress matrices on write. Default is False.''',
'oned_as':
'''oned_as : {'row', 'column'}, optional
If 'column', write 1-D numpy arrays as column vectors.
If 'row', write 1D numpy arrays as row vectors.''',
'unicode_strings':
'''unicode_strings : bool, optional
If True, write strings as Unicode, else MATLAB usual encoding.'''}
docfiller = doccer.filldoc(doc_dict)
'''
Note on architecture
======================
There are three sets of parameters relevant for reading files. The
first are *file read parameters* - containing options that are common
for reading the whole file, and therefore every variable within that
file. At the moment these are:
* mat_stream
* dtypes (derived from byte code)
* byte_order
* chars_as_strings
* squeeze_me
* struct_as_record (MATLAB 5 files)
* class_dtypes (derived from order code, MATLAB 5 files)
* codecs (MATLAB 5 files)
* uint16_codec (MATLAB 5 files)
Another set of parameters are those that apply only to the current
variable being read - the *header*:
* header related variables (different for v4 and v5 mat files)
* is_complex
* mclass
* var_stream
With the header, we need ``next_position`` to tell us where the next
variable in the stream is.
Then, for each element in a matrix, there can be *element read
parameters*. An element is, for example, one element in a MATLAB cell
array. At the moment these are:
* mat_dtype
The file-reading object contains the *file read parameters*. The
*header* is passed around as a data object, or may be read and discarded
in a single function. The *element read parameters* - the mat_dtype in
this instance, is passed into a general post-processing function - see
``mio_utils`` for details.
'''
def convert_dtypes(dtype_template, order_code):
''' Convert dtypes in mapping to given order
Parameters
----------
dtype_template : mapping
mapping with values returning numpy dtype from ``np.dtype(val)``
order_code : str
an order code suitable for using in ``dtype.newbyteorder()``
Returns
-------
dtypes : mapping
mapping where values have been replaced by
``np.dtype(val).newbyteorder(order_code)``
'''
dtypes = dtype_template.copy()
for k in dtypes:
dtypes[k] = np.dtype(dtypes[k]).newbyteorder(order_code)
return dtypes
def read_dtype(mat_stream, a_dtype):
"""
Generic get of byte stream data of known type
Parameters
----------
mat_stream : file_like object
MATLAB (tm) mat file stream
a_dtype : dtype
dtype of array to read. `a_dtype` is assumed to be correct
endianness.
Returns
-------
arr : ndarray
Array of dtype `a_dtype` read from stream.
"""
num_bytes = a_dtype.itemsize
arr = np.ndarray(shape=(),
dtype=a_dtype,
buffer=mat_stream.read(num_bytes),
order='F')
return arr
def get_matfile_version(fileobj):
"""
Return major, minor tuple depending on apparent mat file type
Where:
#. 0,x -> version 4 format mat files
#. 1,x -> version 5 format mat files
#. 2,x -> version 7.3 format mat files (HDF format)
Parameters
----------
fileobj : file_like
object implementing seek() and read()
Returns
-------
major_version : {0, 1, 2}
major MATLAB File format version
minor_version : int
minor MATLAB file format version
Raises
------
MatReadError
If the file is empty.
ValueError
The matfile version is unknown.
Notes
-----
Has the side effect of setting the file read pointer to 0
"""
# Mat4 files have a zero somewhere in first 4 bytes
fileobj.seek(0)
mopt_bytes = fileobj.read(4)
if len(mopt_bytes) == 0:
raise MatReadError("Mat file appears to be empty")
mopt_ints = np.ndarray(shape=(4,), dtype=np.uint8, buffer=mopt_bytes)
if 0 in mopt_ints:
fileobj.seek(0)
return (0,0)
# For 5 format or 7.3 format we need to read an integer in the
# header. Bytes 124 through 128 contain a version integer and an
# endian test string
fileobj.seek(124)
tst_str = fileobj.read(4)
fileobj.seek(0)
maj_ind = int(tst_str[2] == b'I'[0])
maj_val = byteord(tst_str[maj_ind])
min_val = byteord(tst_str[1-maj_ind])
ret = (maj_val, min_val)
if maj_val in (1, 2):
return ret
raise ValueError('Unknown mat file type, version %s, %s' % ret)
def matdims(arr, oned_as='column'):
"""
Determine equivalent MATLAB dimensions for given array
Parameters
----------
arr : ndarray
Input array
oned_as : {'column', 'row'}, optional
Whether 1-D arrays are returned as MATLAB row or column matrices.
Default is 'column'.
Returns
-------
dims : tuple
Shape tuple, in the form MATLAB expects it.
Notes
-----
We had to decide what shape a 1 dimensional array would be by
default. ``np.atleast_2d`` thinks it is a row vector. The
default for a vector in MATLAB (e.g. ``>> 1:12``) is a row vector.
Versions of scipy up to and including 0.11 resulted (accidentally)
in 1-D arrays being read as column vectors. For the moment, we
maintain the same tradition here.
Examples
--------
>>> matdims(np.array(1)) # numpy scalar
(1, 1)
>>> matdims(np.array([1])) # 1d array, 1 element
(1, 1)
>>> matdims(np.array([1,2])) # 1d array, 2 elements
(2, 1)
>>> matdims(np.array([[2],[3]])) # 2d array, column vector
(2, 1)
>>> matdims(np.array([[2,3]])) # 2d array, row vector
(1, 2)
>>> matdims(np.array([[[2,3]]])) # 3d array, rowish vector
(1, 1, 2)
>>> matdims(np.array([])) # empty 1d array
(0, 0)
>>> matdims(np.array([[]])) # empty 2d
(0, 0)
>>> matdims(np.array([[[]]])) # empty 3d
(0, 0, 0)
Optional argument flips 1-D shape behavior.
>>> matdims(np.array([1,2]), 'row') # 1d array, 2 elements
(1, 2)
The argument has to make sense though
>>> matdims(np.array([1,2]), 'bizarre')
Traceback (most recent call last):
...
ValueError: 1D option "bizarre" is strange
"""
shape = arr.shape
if shape == (): # scalar
return (1,1)
if reduce(operator.mul, shape) == 0: # zero elememts
return (0,) * np.max([arr.ndim, 2])
if len(shape) == 1: # 1D
if oned_as == 'column':
return shape + (1,)
elif oned_as == 'row':
return (1,) + shape
else:
raise ValueError('1D option "%s" is strange'
% oned_as)
return shape
class MatVarReader(object):
''' Abstract class defining required interface for var readers'''
def __init__(self, file_reader):
pass
def read_header(self):
''' Returns header '''
pass
def array_from_header(self, header):
''' Reads array given header '''
pass
class MatFileReader(object):
""" Base object for reading mat files
To make this class functional, you will need to override the
following methods:
matrix_getter_factory - gives object to fetch next matrix from stream
guess_byte_order - guesses file byte order from file
"""
@docfiller
def __init__(self, mat_stream,
byte_order=None,
mat_dtype=False,
squeeze_me=False,
chars_as_strings=True,
matlab_compatible=False,
struct_as_record=True,
verify_compressed_data_integrity=True
):
'''
Initializer for mat file reader
mat_stream : file-like
object with file API, open for reading
%(load_args)s
'''
# Initialize stream
self.mat_stream = mat_stream
self.dtypes = {}
if not byte_order:
byte_order = self.guess_byte_order()
else:
byte_order = boc.to_numpy_code(byte_order)
self.byte_order = byte_order
self.struct_as_record = struct_as_record
if matlab_compatible:
self.set_matlab_compatible()
else:
self.squeeze_me = squeeze_me
self.chars_as_strings = chars_as_strings
self.mat_dtype = mat_dtype
self.verify_compressed_data_integrity = verify_compressed_data_integrity
def set_matlab_compatible(self):
''' Sets options to return arrays as MATLAB loads them '''
self.mat_dtype = True
self.squeeze_me = False
self.chars_as_strings = False
def guess_byte_order(self):
''' As we do not know what file type we have, assume native '''
return boc.native_code
def end_of_stream(self):
b = self.mat_stream.read(1)
curpos = self.mat_stream.tell()
self.mat_stream.seek(curpos-1)
return len(b) == 0
def arr_dtype_number(arr, num):
''' Return dtype for given number of items per element'''
return np.dtype(arr.dtype.str[:2] + str(num))
def arr_to_chars(arr):
''' Convert string array to char array '''
dims = list(arr.shape)
if not dims:
dims = [1]
dims.append(int(arr.dtype.str[2:]))
arr = np.ndarray(shape=dims,
dtype=arr_dtype_number(arr, 1),
buffer=arr)
empties = [arr == '']
if not np.any(empties):
return arr
arr = arr.copy()
arr[tuple(empties)] = ' '
return arr
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/setup.py
-16
View File
@@ -1,16 +0,0 @@
from __future__ import division, print_function, absolute_import
def configuration(parent_package='io',top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('matlab', parent_package, top_path)
config.add_extension('streams', sources=['streams.c'])
config.add_extension('mio_utils', sources=['mio_utils.c'])
config.add_extension('mio5_utils', sources=['mio5_utils.c'])
config.add_data_dir('tests')
return config
if __name__ == '__main__':
from numpy.distutils.core import setup
setup(**configuration(top_path='').todict())
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/streams.cpython-36m-x86_64-linux-gnu.so
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__init__.py
View File
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_byteordercodes.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_mio.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_mio5_utils.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_mio_funcs.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_mio_utils.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_miobase.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_pathological.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/__pycache__/test_streams.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/afunc.m
-4
View File
@@ -1,4 +0,0 @@
function [a, b] = afunc(c, d)
% A function
a = c + 1;
b = d + 10;
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/bad_miuint32.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/bad_miutf8_array_name.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/big_endian.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/broken_utf8.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/corrupted_zlib_checksum.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/corrupted_zlib_data.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/japanese_utf8.txt
-5
View File
@@ -1,5 +0,0 @@
Japanese:
すべての人間は、生まれながらにして自由であり、
かつ、尊厳と権利と について平等である。
人間は、理性と良心とを授けられており、
互いに同胞の精神をもって行動しなければならない。
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/little_endian.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/logical_sparse.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/malformed1.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/miuint32_for_miint32.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/miutf8_array_name.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/nasty_duplicate_fieldnames.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/one_by_zero_char.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/parabola.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/single_empty_string.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/some_functions.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/sqr.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/test3dmatrix_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/test3dmatrix_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/test3dmatrix_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/test3dmatrix_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/test_empty_struct.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/test_mat4_le_floats.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/test_skip_variable.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testbool_8_WIN64.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcell_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcell_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcell_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcell_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcellnest_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcellnest_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcellnest_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcellnest_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcomplex_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcomplex_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcomplex_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcomplex_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testcomplex_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testdouble_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testdouble_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testdouble_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testdouble_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testdouble_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testemptycell_5.3_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testemptycell_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testemptycell_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testemptycell_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testfunc_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testhdf5_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmatrix_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmatrix_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmatrix_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmatrix_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmatrix_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testminus_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testminus_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testminus_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testminus_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testminus_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmulti_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmulti_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testmulti_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testobject_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testobject_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testobject_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testobject_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testonechar_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testonechar_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testonechar_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testonechar_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testonechar_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testscalarcell_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparse_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparse_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparse_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparse_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparse_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparsecomplex_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparsecomplex_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparsecomplex_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparsecomplex_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparsecomplex_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testsparsefloat_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststring_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststring_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststring_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststring_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststring_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststringarray_4.2c_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststringarray_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststringarray_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststringarray_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststringarray_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststruct_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststruct_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststruct_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststruct_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructarr_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructarr_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructarr_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructarr_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructnest_6.1_SOL2.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructnest_6.5.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructnest_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/teststructnest_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testunicode_7.1_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testunicode_7.4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/data/testvec_4_GLNX86.mat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/gen_mat4files.m
-50
View File
@@ -1,50 +0,0 @@
% Generates mat files for loadmat unit tests
% Uses save_matfile.m function
% This is the version for matlab 4
% work out matlab version and file suffix for test files
global FILEPREFIX FILESUFFIX
sepchar = '/';
if strcmp(computer, 'PCWIN'), sepchar = '\'; end
FILEPREFIX = [pwd sepchar 'data' sepchar];
mlv = version;
FILESUFFIX = ['_' mlv '_' computer '.mat'];
% basic double array
theta = 0:pi/4:2*pi;
save_matfile('testdouble', theta);
% string
save_matfile('teststring', '"Do nine men interpret?" "Nine men," I nod.')
% complex
save_matfile('testcomplex', cos(theta) + 1j*sin(theta));
% asymmetric array to check indexing
a = zeros(3, 5);
a(:,1) = [1:3]';
a(1,:) = 1:5;
% 2D matrix
save_matfile('testmatrix', a);
% minus number - tests signed int
save_matfile('testminus', -1);
% single character
save_matfile('testonechar', 'r');
% string array
save_matfile('teststringarray', ['one '; 'two '; 'three']);
% sparse array
save_matfile('testsparse', sparse(a));
% sparse complex array
b = sparse(a);
b(1,1) = b(1,1) + j;
save_matfile('testsparsecomplex', b);
% Two variables in same file
save([FILEPREFIX 'testmulti' FILESUFFIX], 'a', 'theta')
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/gen_mat5files.m
-100
View File
@@ -1,100 +0,0 @@
% Generates mat files for loadmat unit tests
% This is the version for matlab 5 and higher
% Uses save_matfile.m function
% work out matlab version and file suffix for test files
global FILEPREFIX FILESUFFIX
FILEPREFIX = [fullfile(pwd, 'data') filesep];
temp = ver('MATLAB');
mlv = temp.Version;
FILESUFFIX = ['_' mlv '_' computer '.mat'];
% basic double array
theta = 0:pi/4:2*pi;
save_matfile('testdouble', theta);
% string
save_matfile('teststring', '"Do nine men interpret?" "Nine men," I nod.')
% complex
save_matfile('testcomplex', cos(theta) + 1j*sin(theta));
% asymmetric array to check indexing
a = zeros(3, 5);
a(:,1) = [1:3]';
a(1,:) = 1:5;
% 2D matrix
save_matfile('testmatrix', a);
% minus number - tests signed int
save_matfile('testminus', -1);
% single character
save_matfile('testonechar', 'r');
% string array
save_matfile('teststringarray', ['one '; 'two '; 'three']);
% sparse array
save_matfile('testsparse', sparse(a));
% sparse complex array
b = sparse(a);
b(1,1) = b(1,1) + j;
save_matfile('testsparsecomplex', b);
% Two variables in same file
save([FILEPREFIX 'testmulti' FILESUFFIX], 'a', 'theta')
% struct
save_matfile('teststruct', ...
struct('stringfield','Rats live on no evil star.',...
'doublefield',[sqrt(2) exp(1) pi],...
'complexfield',(1+1j)*[sqrt(2) exp(1) pi]));
% cell
save_matfile('testcell', ...
{['This cell contains this string and 3 arrays of increasing' ...
' length'], 1., 1.:2., 1.:3.});
% scalar cell
save_matfile('testscalarcell', {1})
% Empty cells in two cell matrices
save_matfile('testemptycell', {1, 2, [], [], 3});
% 3D matrix
save_matfile('test3dmatrix', reshape(1:24,[2 3 4]))
% nested cell array
save_matfile('testcellnest', {1, {2, 3, {4, 5}}});
% nested struct
save_matfile('teststructnest', struct('one', 1, 'two', ...
struct('three', 'number 3')));
% array of struct
save_matfile('teststructarr', [struct('one', 1, 'two', 2) ...
struct('one', 'number 1', 'two', 'number 2')]);
% matlab object
save_matfile('testobject', inline('x'))
% array of matlab objects
%save_matfile('testobjarr', [inline('x') inline('x')])
% unicode test
if str2num(mlv) > 7 % function added 7.0.1
fid = fopen([FILEPREFIX 'japanese_utf8.txt']);
from_japan = fread(fid, 'uint8')';
fclose(fid);
save_matfile('testunicode', native2unicode(from_japan, 'utf-8'));
end
% func
if str2num(mlv) > 7 % function pointers added recently
func = @afunc;
save_matfile('testfunc', func);
end
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/save_matfile.m
-6
View File
@@ -1,6 +0,0 @@
function save_matfile(test_name, v)
% saves variable passed in m with filename from prefix
global FILEPREFIX FILESUFFIX
eval([test_name ' = v;']);
save([FILEPREFIX test_name FILESUFFIX], test_name)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_byteordercodes.py
-31
View File
@@ -1,31 +0,0 @@
''' Tests for byteorder module '''
from __future__ import division, print_function, absolute_import
import sys
from numpy.testing import assert_
from pytest import raises as assert_raises
import scipy.io.matlab.byteordercodes as sibc
def test_native():
native_is_le = sys.byteorder == 'little'
assert_(sibc.sys_is_le == native_is_le)
def test_to_numpy():
if sys.byteorder == 'little':
assert_(sibc.to_numpy_code('native') == '<')
assert_(sibc.to_numpy_code('swapped') == '>')
else:
assert_(sibc.to_numpy_code('native') == '>')
assert_(sibc.to_numpy_code('swapped') == '<')
assert_(sibc.to_numpy_code('native') == sibc.to_numpy_code('='))
assert_(sibc.to_numpy_code('big') == '>')
for code in ('little', '<', 'l', 'L', 'le'):
assert_(sibc.to_numpy_code(code) == '<')
for code in ('big', '>', 'b', 'B', 'be'):
assert_(sibc.to_numpy_code(code) == '>')
assert_raises(ValueError, sibc.to_numpy_code, 'silly string')
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_mio.py
-1238
View File
File diff suppressed because it is too large Load Diff
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_mio5_utils.py
-185
View File
@@ -1,185 +0,0 @@
""" Testing mio5_utils Cython module
"""
from __future__ import division, print_function, absolute_import
import sys
from io import BytesIO
cStringIO = BytesIO
import numpy as np
from numpy.testing import assert_array_equal, assert_equal, assert_
from pytest import raises as assert_raises
from scipy._lib.six import u
import scipy.io.matlab.byteordercodes as boc
import scipy.io.matlab.streams as streams
import scipy.io.matlab.mio5_params as mio5p
import scipy.io.matlab.mio5_utils as m5u
def test_byteswap():
for val in (
1,
0x100,
0x10000):
a = np.array(val, dtype=np.uint32)
b = a.byteswap()
c = m5u.byteswap_u4(a)
assert_equal(b.item(), c)
d = m5u.byteswap_u4(c)
assert_equal(a.item(), d)
def _make_tag(base_dt, val, mdtype, sde=False):
''' Makes a simple matlab tag, full or sde '''
base_dt = np.dtype(base_dt)
bo = boc.to_numpy_code(base_dt.byteorder)
byte_count = base_dt.itemsize
if not sde:
udt = bo + 'u4'
padding = 8 - (byte_count % 8)
all_dt = [('mdtype', udt),
('byte_count', udt),
('val', base_dt)]
if padding:
all_dt.append(('padding', 'u1', padding))
else: # is sde
udt = bo + 'u2'
padding = 4-byte_count
if bo == '<': # little endian
all_dt = [('mdtype', udt),
('byte_count', udt),
('val', base_dt)]
else: # big endian
all_dt = [('byte_count', udt),
('mdtype', udt),
('val', base_dt)]
if padding:
all_dt.append(('padding', 'u1', padding))
tag = np.zeros((1,), dtype=all_dt)
tag['mdtype'] = mdtype
tag['byte_count'] = byte_count
tag['val'] = val
return tag
def _write_stream(stream, *strings):
stream.truncate(0)
stream.seek(0)
for s in strings:
stream.write(s)
stream.seek(0)
def _make_readerlike(stream, byte_order=boc.native_code):
class R(object):
pass
r = R()
r.mat_stream = stream
r.byte_order = byte_order
r.struct_as_record = True
r.uint16_codec = sys.getdefaultencoding()
r.chars_as_strings = False
r.mat_dtype = False
r.squeeze_me = False
return r
def test_read_tag():
# mainly to test errors
# make reader-like thing
str_io = BytesIO()
r = _make_readerlike(str_io)
c_reader = m5u.VarReader5(r)
# This works for StringIO but _not_ cStringIO
assert_raises(IOError, c_reader.read_tag)
# bad SDE
tag = _make_tag('i4', 1, mio5p.miINT32, sde=True)
tag['byte_count'] = 5
_write_stream(str_io, tag.tostring())
assert_raises(ValueError, c_reader.read_tag)
def test_read_stream():
tag = _make_tag('i4', 1, mio5p.miINT32, sde=True)
tag_str = tag.tostring()
str_io = cStringIO(tag_str)
st = streams.make_stream(str_io)
s = streams._read_into(st, tag.itemsize)
assert_equal(s, tag.tostring())
def test_read_numeric():
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike(str_io)
# check simplest of tags
for base_dt, val, mdtype in (('u2', 30, mio5p.miUINT16),
('i4', 1, mio5p.miINT32),
('i2', -1, mio5p.miINT16)):
for byte_code in ('<', '>'):
r.byte_order = byte_code
c_reader = m5u.VarReader5(r)
assert_equal(c_reader.little_endian, byte_code == '<')
assert_equal(c_reader.is_swapped, byte_code != boc.native_code)
for sde_f in (False, True):
dt = np.dtype(base_dt).newbyteorder(byte_code)
a = _make_tag(dt, val, mdtype, sde_f)
a_str = a.tostring()
_write_stream(str_io, a_str)
el = c_reader.read_numeric()
assert_equal(el, val)
# two sequential reads
_write_stream(str_io, a_str, a_str)
el = c_reader.read_numeric()
assert_equal(el, val)
el = c_reader.read_numeric()
assert_equal(el, val)
def test_read_numeric_writeable():
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike(str_io, '<')
c_reader = m5u.VarReader5(r)
dt = np.dtype('<u2')
a = _make_tag(dt, 30, mio5p.miUINT16, 0)
a_str = a.tostring()
_write_stream(str_io, a_str)
el = c_reader.read_numeric()
assert_(el.flags.writeable is True)
def test_zero_byte_string():
# Tests hack to allow chars of non-zero length, but 0 bytes
# make reader-like thing
str_io = cStringIO()
r = _make_readerlike(str_io, boc.native_code)
c_reader = m5u.VarReader5(r)
tag_dt = np.dtype([('mdtype', 'u4'), ('byte_count', 'u4')])
tag = np.zeros((1,), dtype=tag_dt)
tag['mdtype'] = mio5p.miINT8
tag['byte_count'] = 1
hdr = m5u.VarHeader5()
# Try when string is 1 length
hdr.set_dims([1,])
_write_stream(str_io, tag.tostring() + b' ')
str_io.seek(0)
val = c_reader.read_char(hdr)
assert_equal(val, u(' '))
# Now when string has 0 bytes 1 length
tag['byte_count'] = 0
_write_stream(str_io, tag.tostring())
str_io.seek(0)
val = c_reader.read_char(hdr)
assert_equal(val, u(' '))
# Now when string has 0 bytes 4 length
str_io.seek(0)
hdr.set_dims([4,])
val = c_reader.read_char(hdr)
assert_array_equal(val, [u(' ')] * 4)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_mio_funcs.py
-57
View File
@@ -1,57 +0,0 @@
''' Jottings to work out format for __function_workspace__ matrix at end
of mat file.
'''
from __future__ import division, print_function, absolute_import
import os.path
import sys
import io
from numpy.compat import asstr
from scipy.io.matlab.mio5 import (MatlabObject, MatFile5Writer,
MatFile5Reader, MatlabFunction)
test_data_path = os.path.join(os.path.dirname(__file__), 'data')
def read_minimat_vars(rdr):
rdr.initialize_read()
mdict = {'__globals__': []}
i = 0
while not rdr.end_of_stream():
hdr, next_position = rdr.read_var_header()
name = asstr(hdr.name)
if name == '':
name = 'var_%d' % i
i += 1
res = rdr.read_var_array(hdr, process=False)
rdr.mat_stream.seek(next_position)
mdict[name] = res
if hdr.is_global:
mdict['__globals__'].append(name)
return mdict
def read_workspace_vars(fname):
fp = open(fname, 'rb')
rdr = MatFile5Reader(fp, struct_as_record=True)
vars = rdr.get_variables()
fws = vars['__function_workspace__']
ws_bs = io.BytesIO(fws.tostring())
ws_bs.seek(2)
rdr.mat_stream = ws_bs
# Guess byte order.
mi = rdr.mat_stream.read(2)
rdr.byte_order = mi == b'IM' and '<' or '>'
rdr.mat_stream.read(4) # presumably byte padding
mdict = read_minimat_vars(rdr)
fp.close()
return mdict
def test_jottings():
# example
fname = os.path.join(test_data_path, 'parabola.mat')
ws_vars = read_workspace_vars(fname)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_mio_utils.py
-46
View File
@@ -1,46 +0,0 @@
""" Testing
"""
from __future__ import division, print_function, absolute_import
import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal, \
assert_
from scipy.io.matlab.mio_utils import squeeze_element, chars_to_strings
def test_squeeze_element():
a = np.zeros((1,3))
assert_array_equal(np.squeeze(a), squeeze_element(a))
# 0d output from squeeze gives scalar
sq_int = squeeze_element(np.zeros((1,1), dtype=float))
assert_(isinstance(sq_int, float))
# Unless it's a structured array
sq_sa = squeeze_element(np.zeros((1,1),dtype=[('f1', 'f')]))
assert_(isinstance(sq_sa, np.ndarray))
def test_chars_strings():
# chars as strings
strings = ['learn ', 'python', 'fast ', 'here ']
str_arr = np.array(strings, dtype='U6') # shape (4,)
chars = [list(s) for s in strings]
char_arr = np.array(chars, dtype='U1') # shape (4,6)
assert_array_equal(chars_to_strings(char_arr), str_arr)
ca2d = char_arr.reshape((2,2,6))
sa2d = str_arr.reshape((2,2))
assert_array_equal(chars_to_strings(ca2d), sa2d)
ca3d = char_arr.reshape((1,2,2,6))
sa3d = str_arr.reshape((1,2,2))
assert_array_equal(chars_to_strings(ca3d), sa3d)
# Fortran ordered arrays
char_arrf = np.array(chars, dtype='U1', order='F') # shape (4,6)
assert_array_equal(chars_to_strings(char_arrf), str_arr)
# empty array
arr = np.array([['']], dtype='U1')
out_arr = np.array([''], dtype='U1')
assert_array_equal(chars_to_strings(arr), out_arr)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_miobase.py
-31
View File
@@ -1,31 +0,0 @@
""" Testing miobase module
"""
import numpy as np
from numpy.testing import assert_equal
from pytest import raises as assert_raises
from scipy.io.matlab.miobase import matdims
def test_matdims():
# Test matdims dimension finder
assert_equal(matdims(np.array(1)), (1, 1)) # numpy scalar
assert_equal(matdims(np.array([1])), (1, 1)) # 1d array, 1 element
assert_equal(matdims(np.array([1,2])), (2, 1)) # 1d array, 2 elements
assert_equal(matdims(np.array([[2],[3]])), (2, 1)) # 2d array, column vector
assert_equal(matdims(np.array([[2,3]])), (1, 2)) # 2d array, row vector
# 3d array, rowish vector
assert_equal(matdims(np.array([[[2,3]]])), (1, 1, 2))
assert_equal(matdims(np.array([])), (0, 0)) # empty 1d array
assert_equal(matdims(np.array([[]])), (0, 0)) # empty 2d
assert_equal(matdims(np.array([[[]]])), (0, 0, 0)) # empty 3d
# Optional argument flips 1-D shape behavior.
assert_equal(matdims(np.array([1,2]), 'row'), (1, 2)) # 1d array, 2 elements
# The argument has to make sense though
assert_raises(ValueError, matdims, np.array([1,2]), 'bizarre')
# Check empty sparse matrices get their own shape
from scipy.sparse import csr_matrix, csc_matrix
assert_equal(matdims(csr_matrix(np.zeros((3, 3)))), (3, 3))
assert_equal(matdims(csc_matrix(np.zeros((2, 2)))), (2, 2))
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_pathological.py
-35
View File
@@ -1,35 +0,0 @@
""" Test reading of files not conforming to matlab specification
We try and read any file that matlab reads, these files included
"""
from __future__ import division, print_function, absolute_import
from os.path import dirname, join as pjoin
from numpy.testing import assert_
from pytest import raises as assert_raises
from scipy.io.matlab.mio import loadmat
TEST_DATA_PATH = pjoin(dirname(__file__), 'data')
def test_multiple_fieldnames():
# Example provided by Dharhas Pothina
# Extracted using mio5.varmats_from_mat
multi_fname = pjoin(TEST_DATA_PATH, 'nasty_duplicate_fieldnames.mat')
vars = loadmat(multi_fname)
funny_names = vars['Summary'].dtype.names
assert_(set(['_1_Station_Q', '_2_Station_Q',
'_3_Station_Q']).issubset(funny_names))
def test_malformed1():
# Example from gh-6072
# Contains malformed header data, which previously resulted into a
# buffer overflow.
#
# Should raise an exception, not segfault
fname = pjoin(TEST_DATA_PATH, 'malformed1.mat')
with open(fname, 'rb') as f:
assert_raises(ValueError, loadmat, f)
utils/venv/lib/python3.6/site-packages/scipy/io/matlab/tests/test_streams.py
-184
View File
@@ -1,184 +0,0 @@
""" Testing
"""
from __future__ import division, print_function, absolute_import
import os
import sys
import zlib
from io import BytesIO
if sys.version_info[0] >= 3:
cStringIO = BytesIO
else:
from cStringIO import StringIO as cStringIO
from tempfile import mkstemp
from contextlib import contextmanager
import numpy as np
from numpy.testing import assert_, assert_equal
from pytest import raises as assert_raises
from scipy.io.matlab.streams import (make_stream,
GenericStream, cStringStream, FileStream, ZlibInputStream,
_read_into, _read_string)
IS_PYPY = ('__pypy__' in sys.modules)
@contextmanager
def setup_test_file():
val = b'a\x00string'
fd, fname = mkstemp()
with os.fdopen(fd, 'wb') as fs:
fs.write(val)
with open(fname, 'rb') as fs:
gs = BytesIO(val)
cs = cStringIO(val)
yield fs, gs, cs
os.unlink(fname)
def test_make_stream():
with setup_test_file() as (fs, gs, cs):
# test stream initialization
assert_(isinstance(make_stream(gs), GenericStream))
if sys.version_info[0] < 3 and not IS_PYPY:
assert_(isinstance(make_stream(cs), cStringStream))
assert_(isinstance(make_stream(fs), FileStream))
def test_tell_seek():
with setup_test_file() as (fs, gs, cs):
for s in (fs, gs, cs):
st = make_stream(s)
res = st.seek(0)
assert_equal(res, 0)
assert_equal(st.tell(), 0)
res = st.seek(5)
assert_equal(res, 0)
assert_equal(st.tell(), 5)
res = st.seek(2, 1)
assert_equal(res, 0)
assert_equal(st.tell(), 7)
res = st.seek(-2, 2)
assert_equal(res, 0)
assert_equal(st.tell(), 6)
def test_read():
with setup_test_file() as (fs, gs, cs):
for s in (fs, gs, cs):
st = make_stream(s)
st.seek(0)
res = st.read(-1)
assert_equal(res, b'a\x00string')
st.seek(0)
res = st.read(4)
assert_equal(res, b'a\x00st')
# read into
st.seek(0)
res = _read_into(st, 4)
assert_equal(res, b'a\x00st')
res = _read_into(st, 4)
assert_equal(res, b'ring')
assert_raises(IOError, _read_into, st, 2)
# read alloc
st.seek(0)
res = _read_string(st, 4)
assert_equal(res, b'a\x00st')
res = _read_string(st, 4)
assert_equal(res, b'ring')
assert_raises(IOError, _read_string, st, 2)
class TestZlibInputStream(object):
def _get_data(self, size):
data = np.random.randint(0, 256, size).astype(np.uint8).tostring()
compressed_data = zlib.compress(data)
stream = BytesIO(compressed_data)
return stream, len(compressed_data), data
def test_read(self):
block_size = 131072
SIZES = [0, 1, 10, block_size//2, block_size-1,
block_size, block_size+1, 2*block_size-1]
READ_SIZES = [block_size//2, block_size-1,
block_size, block_size+1]
def check(size, read_size):
compressed_stream, compressed_data_len, data = self._get_data(size)
stream = ZlibInputStream(compressed_stream, compressed_data_len)
data2 = b''
so_far = 0
while True:
block = stream.read(min(read_size,
size - so_far))
if not block:
break
so_far += len(block)
data2 += block
assert_equal(data, data2)
for size in SIZES:
for read_size in READ_SIZES:
check(size, read_size)
def test_read_max_length(self):
size = 1234
data = np.random.randint(0, 256, size).astype(np.uint8).tostring()
compressed_data = zlib.compress(data)
compressed_stream = BytesIO(compressed_data + b"abbacaca")
stream = ZlibInputStream(compressed_stream, len(compressed_data))
stream.read(len(data))
assert_equal(compressed_stream.tell(), len(compressed_data))
assert_raises(IOError, stream.read, 1)
def test_seek(self):
compressed_stream, compressed_data_len, data = self._get_data(1024)
stream = ZlibInputStream(compressed_stream, compressed_data_len)
stream.seek(123)
p = 123
assert_equal(stream.tell(), p)
d1 = stream.read(11)
assert_equal(d1, data[p:p+11])
stream.seek(321, 1)
p = 123+11+321
assert_equal(stream.tell(), p)
d2 = stream.read(21)
assert_equal(d2, data[p:p+21])
stream.seek(641, 0)
p = 641
assert_equal(stream.tell(), p)
d3 = stream.read(11)
assert_equal(d3, data[p:p+11])
assert_raises(IOError, stream.seek, 10, 2)
assert_raises(IOError, stream.seek, -1, 1)
assert_raises(ValueError, stream.seek, 1, 123)
stream.seek(10000, 1)
assert_raises(IOError, stream.read, 12)
def test_all_data_read(self):
compressed_stream, compressed_data_len, data = self._get_data(1024)
stream = ZlibInputStream(compressed_stream, compressed_data_len)
assert_(not stream.all_data_read())
stream.seek(512)
assert_(not stream.all_data_read())
stream.seek(1024)
assert_(stream.all_data_read())
utils/venv/lib/python3.6/site-packages/scipy/io/mmio.py
-835
View File
@@ -1,835 +0,0 @@
"""
Matrix Market I/O in Python.
See http://math.nist.gov/MatrixMarket/formats.html
for information about the Matrix Market format.
"""
#
# Author: Pearu Peterson <pearu@cens.ioc.ee>
# Created: October, 2004
#
# References:
# http://math.nist.gov/MatrixMarket/
#
from __future__ import division, print_function, absolute_import
import os
import sys
from numpy import (asarray, real, imag, conj, zeros, ndarray, concatenate,
ones, can_cast)
from numpy.compat import asbytes, asstr
from scipy._lib.six import string_types
from scipy.sparse import coo_matrix, isspmatrix
__all__ = ['mminfo', 'mmread', 'mmwrite', 'MMFile']
# -----------------------------------------------------------------------------
def mminfo(source):
"""
Return size and storage parameters from Matrix Market file-like 'source'.
Parameters
----------
source : str or file-like
Matrix Market filename (extension .mtx) or open file-like object
Returns
-------
rows : int
Number of matrix rows.
cols : int
Number of matrix columns.
entries : int
Number of non-zero entries of a sparse matrix
or rows*cols for a dense matrix.
format : str
Either 'coordinate' or 'array'.
field : str
Either 'real', 'complex', 'pattern', or 'integer'.
symmetry : str
Either 'general', 'symmetric', 'skew-symmetric', or 'hermitian'.
"""
return MMFile.info(source)
# -----------------------------------------------------------------------------
def mmread(source):
"""
Reads the contents of a Matrix Market file-like 'source' into a matrix.
Parameters
----------
source : str or file-like
Matrix Market filename (extensions .mtx, .mtz.gz)
or open file-like object.
Returns
-------
a : ndarray or coo_matrix
Dense or sparse matrix depending on the matrix format in the
Matrix Market file.
"""
return MMFile().read(source)
# -----------------------------------------------------------------------------
def mmwrite(target, a, comment='', field=None, precision=None, symmetry=None):
"""
Writes the sparse or dense array `a` to Matrix Market file-like `target`.
Parameters
----------
target : str or file-like
Matrix Market filename (extension .mtx) or open file-like object.
a : array like
Sparse or dense 2D array.
comment : str, optional
Comments to be prepended to the Matrix Market file.
field : None or str, optional
Either 'real', 'complex', 'pattern', or 'integer'.
precision : None or int, optional
Number of digits to display for real or complex values.
symmetry : None or str, optional
Either 'general', 'symmetric', 'skew-symmetric', or 'hermitian'.
If symmetry is None the symmetry type of 'a' is determined by its
values.
"""
MMFile().write(target, a, comment, field, precision, symmetry)
###############################################################################
class MMFile (object):
__slots__ = ('_rows',
'_cols',
'_entries',
'_format',
'_field',
'_symmetry')
@property
def rows(self):
return self._rows
@property
def cols(self):
return self._cols
@property
def entries(self):
return self._entries
@property
def format(self):
return self._format
@property
def field(self):
return self._field
@property
def symmetry(self):
return self._symmetry
@property
def has_symmetry(self):
return self._symmetry in (self.SYMMETRY_SYMMETRIC,
self.SYMMETRY_SKEW_SYMMETRIC,
self.SYMMETRY_HERMITIAN)
# format values
FORMAT_COORDINATE = 'coordinate'
FORMAT_ARRAY = 'array'
FORMAT_VALUES = (FORMAT_COORDINATE, FORMAT_ARRAY)
@classmethod
def _validate_format(self, format):
if format not in self.FORMAT_VALUES:
raise ValueError('unknown format type %s, must be one of %s' %
(format, self.FORMAT_VALUES))
# field values
FIELD_INTEGER = 'integer'
FIELD_UNSIGNED = 'unsigned-integer'
FIELD_REAL = 'real'
FIELD_COMPLEX = 'complex'
FIELD_PATTERN = 'pattern'
FIELD_VALUES = (FIELD_INTEGER, FIELD_UNSIGNED, FIELD_REAL, FIELD_COMPLEX, FIELD_PATTERN)
@classmethod
def _validate_field(self, field):
if field not in self.FIELD_VALUES:
raise ValueError('unknown field type %s, must be one of %s' %
(field, self.FIELD_VALUES))
# symmetry values
SYMMETRY_GENERAL = 'general'
SYMMETRY_SYMMETRIC = 'symmetric'
SYMMETRY_SKEW_SYMMETRIC = 'skew-symmetric'
SYMMETRY_HERMITIAN = 'hermitian'
SYMMETRY_VALUES = (SYMMETRY_GENERAL, SYMMETRY_SYMMETRIC,
SYMMETRY_SKEW_SYMMETRIC, SYMMETRY_HERMITIAN)
@classmethod
def _validate_symmetry(self, symmetry):
if symmetry not in self.SYMMETRY_VALUES:
raise ValueError('unknown symmetry type %s, must be one of %s' %
(symmetry, self.SYMMETRY_VALUES))
DTYPES_BY_FIELD = {FIELD_INTEGER: 'intp',
FIELD_UNSIGNED: 'uint64',
FIELD_REAL: 'd',
FIELD_COMPLEX: 'D',
FIELD_PATTERN: 'd'}
# -------------------------------------------------------------------------
@staticmethod
def reader():
pass
# -------------------------------------------------------------------------
@staticmethod
def writer():
pass
# -------------------------------------------------------------------------
@classmethod
def info(self, source):
"""
Return size, storage parameters from Matrix Market file-like 'source'.
Parameters
----------
source : str or file-like
Matrix Market filename (extension .mtx) or open file-like object
Returns
-------
rows : int
Number of matrix rows.
cols : int
Number of matrix columns.
entries : int
Number of non-zero entries of a sparse matrix
or rows*cols for a dense matrix.
format : str
Either 'coordinate' or 'array'.
field : str
Either 'real', 'complex', 'pattern', or 'integer'.
symmetry : str
Either 'general', 'symmetric', 'skew-symmetric', or 'hermitian'.
"""
stream, close_it = self._open(source)
try:
# read and validate header line
line = stream.readline()
mmid, matrix, format, field, symmetry = \
[asstr(part.strip()) for part in line.split()]
if not mmid.startswith('%%MatrixMarket'):
raise ValueError('source is not in Matrix Market format')
if not matrix.lower() == 'matrix':
raise ValueError("Problem reading file header: " + line)
# http://math.nist.gov/MatrixMarket/formats.html
if format.lower() == 'array':
format = self.FORMAT_ARRAY
elif format.lower() == 'coordinate':
format = self.FORMAT_COORDINATE
# skip comments
while line.startswith(b'%'):
line = stream.readline()
line = line.split()
if format == self.FORMAT_ARRAY:
if not len(line) == 2:
raise ValueError("Header line not of length 2: " + line)
rows, cols = map(int, line)
entries = rows * cols
else:
if not len(line) == 3:
raise ValueError("Header line not of length 3: " + line)
rows, cols, entries = map(int, line)
return (rows, cols, entries, format, field.lower(),
symmetry.lower())
finally:
if close_it:
stream.close()
# -------------------------------------------------------------------------
@staticmethod
def _open(filespec, mode='rb'):
""" Return an open file stream for reading based on source.
If source is a file name, open it (after trying to find it with mtx and
gzipped mtx extensions). Otherwise, just return source.
Parameters
----------
filespec : str or file-like
String giving file name or file-like object
mode : str, optional
Mode with which to open file, if `filespec` is a file name.
Returns
-------
fobj : file-like
Open file-like object.
close_it : bool
True if the calling function should close this file when done,
false otherwise.
"""
close_it = False
if isinstance(filespec, string_types):
close_it = True
# open for reading
if mode[0] == 'r':
# determine filename plus extension
if not os.path.isfile(filespec):
if os.path.isfile(filespec+'.mtx'):
filespec = filespec + '.mtx'
elif os.path.isfile(filespec+'.mtx.gz'):
filespec = filespec + '.mtx.gz'
elif os.path.isfile(filespec+'.mtx.bz2'):
filespec = filespec + '.mtx.bz2'
# open filename
if filespec.endswith('.gz'):
import gzip
stream = gzip.open(filespec, mode)
elif filespec.endswith('.bz2'):
import bz2
stream = bz2.BZ2File(filespec, 'rb')
else:
stream = open(filespec, mode)
# open for writing
else:
if filespec[-4:] != '.mtx':
filespec = filespec + '.mtx'
stream = open(filespec, mode)
else:
stream = filespec
return stream, close_it
# -------------------------------------------------------------------------
@staticmethod
def _get_symmetry(a):
m, n = a.shape
if m != n:
return MMFile.SYMMETRY_GENERAL
issymm = True
isskew = True
isherm = a.dtype.char in 'FD'
# sparse input
if isspmatrix(a):
# check if number of nonzero entries of lower and upper triangle
# matrix are equal
a = a.tocoo()
(row, col) = a.nonzero()
if (row < col).sum() != (row > col).sum():
return MMFile.SYMMETRY_GENERAL
# define iterator over symmetric pair entries
a = a.todok()
def symm_iterator():
for ((i, j), aij) in a.items():
if i > j:
aji = a[j, i]
yield (aij, aji)
# non-sparse input
else:
# define iterator over symmetric pair entries
def symm_iterator():
for j in range(n):
for i in range(j+1, n):
aij, aji = a[i][j], a[j][i]
yield (aij, aji)
# check for symmetry
for (aij, aji) in symm_iterator():
if issymm and aij != aji:
issymm = False
if isskew and aij != -aji:
isskew = False
if isherm and aij != conj(aji):
isherm = False
if not (issymm or isskew or isherm):
break
# return symmetry value
if issymm:
return MMFile.SYMMETRY_SYMMETRIC
if isskew:
return MMFile.SYMMETRY_SKEW_SYMMETRIC
if isherm:
return MMFile.SYMMETRY_HERMITIAN
return MMFile.SYMMETRY_GENERAL
# -------------------------------------------------------------------------
@staticmethod
def _field_template(field, precision):
return {MMFile.FIELD_REAL: '%%.%ie\n' % precision,
MMFile.FIELD_INTEGER: '%i\n',
MMFile.FIELD_UNSIGNED: '%u\n',
MMFile.FIELD_COMPLEX: '%%.%ie %%.%ie\n' %
(precision, precision)
}.get(field, None)
# -------------------------------------------------------------------------
def __init__(self, **kwargs):
self._init_attrs(**kwargs)
# -------------------------------------------------------------------------
def read(self, source):
"""
Reads the contents of a Matrix Market file-like 'source' into a matrix.
Parameters
----------
source : str or file-like
Matrix Market filename (extensions .mtx, .mtz.gz)
or open file object.
Returns
-------
a : ndarray or coo_matrix
Dense or sparse matrix depending on the matrix format in the
Matrix Market file.
"""
stream, close_it = self._open(source)
try:
self._parse_header(stream)
return self._parse_body(stream)
finally:
if close_it:
stream.close()
# -------------------------------------------------------------------------
def write(self, target, a, comment='', field=None, precision=None,
symmetry=None):
"""
Writes sparse or dense array `a` to Matrix Market file-like `target`.
Parameters
----------
target : str or file-like
Matrix Market filename (extension .mtx) or open file-like object.
a : array like
Sparse or dense 2D array.
comment : str, optional
Comments to be prepended to the Matrix Market file.
field : None or str, optional
Either 'real', 'complex', 'pattern', or 'integer'.
precision : None or int, optional
Number of digits to display for real or complex values.
symmetry : None or str, optional
Either 'general', 'symmetric', 'skew-symmetric', or 'hermitian'.
If symmetry is None the symmetry type of 'a' is determined by its
values.
"""
stream, close_it = self._open(target, 'wb')
try:
self._write(stream, a, comment, field, precision, symmetry)
finally:
if close_it:
stream.close()
else:
stream.flush()
# -------------------------------------------------------------------------
def _init_attrs(self, **kwargs):
"""
Initialize each attributes with the corresponding keyword arg value
or a default of None
"""
attrs = self.__class__.__slots__
public_attrs = [attr[1:] for attr in attrs]
invalid_keys = set(kwargs.keys()) - set(public_attrs)
if invalid_keys:
raise ValueError('''found %s invalid keyword arguments, please only
use %s''' % (tuple(invalid_keys),
public_attrs))
for attr in attrs:
setattr(self, attr, kwargs.get(attr[1:], None))
# -------------------------------------------------------------------------
def _parse_header(self, stream):
rows, cols, entries, format, field, symmetry = \
self.__class__.info(stream)
self._init_attrs(rows=rows, cols=cols, entries=entries, format=format,
field=field, symmetry=symmetry)
# -------------------------------------------------------------------------
def _parse_body(self, stream):
rows, cols, entries, format, field, symm = (self.rows, self.cols,
self.entries, self.format,
self.field, self.symmetry)
try:
from scipy.sparse import coo_matrix
except ImportError:
coo_matrix = None
dtype = self.DTYPES_BY_FIELD.get(field, None)
has_symmetry = self.has_symmetry
is_integer = field == self.FIELD_INTEGER
is_unsigned_integer = field == self.FIELD_UNSIGNED
is_complex = field == self.FIELD_COMPLEX
is_skew = symm == self.SYMMETRY_SKEW_SYMMETRIC
is_herm = symm == self.SYMMETRY_HERMITIAN
is_pattern = field == self.FIELD_PATTERN
if format == self.FORMAT_ARRAY:
a = zeros((rows, cols), dtype=dtype)
line = 1
i, j = 0, 0
if is_skew:
a[i, j] = 0
if i < rows - 1:
i += 1
while line:
line = stream.readline()
if not line or line.startswith(b'%'):
continue
if is_integer:
aij = int(line)
elif is_unsigned_integer:
aij = int(line)
elif is_complex:
aij = complex(*map(float, line.split()))
else:
aij = float(line)
a[i, j] = aij
if has_symmetry and i != j:
if is_skew:
a[j, i] = -aij
elif is_herm:
a[j, i] = conj(aij)
else:
a[j, i] = aij
if i < rows-1:
i = i + 1
else:
j = j + 1
if not has_symmetry:
i = 0
else:
i = j
if is_skew:
a[i, j] = 0
if i < rows-1:
i += 1
if is_skew:
if not (i in [0, j] and j == cols - 1):
raise ValueError("Parse error, did not read all lines.")
else:
if not (i in [0, j] and j == cols):
raise ValueError("Parse error, did not read all lines.")
elif format == self.FORMAT_COORDINATE and coo_matrix is None:
# Read sparse matrix to dense when coo_matrix is not available.
a = zeros((rows, cols), dtype=dtype)
line = 1
k = 0
while line:
line = stream.readline()
if not line or line.startswith(b'%'):
continue
l = line.split()
i, j = map(int, l[:2])
i, j = i-1, j-1
if is_integer:
aij = int(l[2])
elif is_unsigned_integer:
aij = int(l[2])
elif is_complex:
aij = complex(*map(float, l[2:]))
else:
aij = float(l[2])
a[i, j] = aij
if has_symmetry and i != j:
if is_skew:
a[j, i] = -aij
elif is_herm:
a[j, i] = conj(aij)
else:
a[j, i] = aij
k = k + 1
if not k == entries:
ValueError("Did not read all entries")
elif format == self.FORMAT_COORDINATE:
# Read sparse COOrdinate format
if entries == 0:
# empty matrix
return coo_matrix((rows, cols), dtype=dtype)
I = zeros(entries, dtype='intc')
J = zeros(entries, dtype='intc')
if is_pattern:
V = ones(entries, dtype='int8')
elif is_integer:
V = zeros(entries, dtype='intp')
elif is_unsigned_integer:
V = zeros(entries, dtype='uint64')
elif is_complex:
V = zeros(entries, dtype='complex')
else:
V = zeros(entries, dtype='float')
entry_number = 0
for line in stream:
if not line or line.startswith(b'%'):
continue
if entry_number+1 > entries:
raise ValueError("'entries' in header is smaller than "
"number of entries")
l = line.split()
I[entry_number], J[entry_number] = map(int, l[:2])
if not is_pattern:
if is_integer:
V[entry_number] = int(l[2])
elif is_unsigned_integer:
V[entry_number] = int(l[2])
elif is_complex:
V[entry_number] = complex(*map(float, l[2:]))
else:
V[entry_number] = float(l[2])
entry_number += 1
if entry_number < entries:
raise ValueError("'entries' in header is larger than "
"number of entries")
I -= 1 # adjust indices (base 1 -> base 0)
J -= 1
if has_symmetry:
mask = (I != J) # off diagonal mask
od_I = I[mask]
od_J = J[mask]
od_V = V[mask]
I = concatenate((I, od_J))
J = concatenate((J, od_I))
if is_skew:
od_V *= -1
elif is_herm:
od_V = od_V.conjugate()
V = concatenate((V, od_V))
a = coo_matrix((V, (I, J)), shape=(rows, cols), dtype=dtype)
else:
raise NotImplementedError(format)
return a
# ------------------------------------------------------------------------
def _write(self, stream, a, comment='', field=None, precision=None,
symmetry=None):
if isinstance(a, list) or isinstance(a, ndarray) or \
isinstance(a, tuple) or hasattr(a, '__array__'):
rep = self.FORMAT_ARRAY
a = asarray(a)
if len(a.shape) != 2:
raise ValueError('Expected 2 dimensional array')
rows, cols = a.shape
if field is not None:
if field == self.FIELD_INTEGER:
if not can_cast(a.dtype, 'intp'):
raise OverflowError("mmwrite does not support integer "
"dtypes larger than native 'intp'.")
a = a.astype('intp')
elif field == self.FIELD_REAL:
if a.dtype.char not in 'fd':
a = a.astype('d')
elif field == self.FIELD_COMPLEX:
if a.dtype.char not in 'FD':
a = a.astype('D')
else:
if not isspmatrix(a):
raise ValueError('unknown matrix type: %s' % type(a))
rep = 'coordinate'
rows, cols = a.shape
typecode = a.dtype.char
if precision is None:
if typecode in 'fF':
precision = 8
else:
precision = 16
if field is None:
kind = a.dtype.kind
if kind == 'i':
if not can_cast(a.dtype, 'intp'):
raise OverflowError("mmwrite does not support integer "
"dtypes larger than native 'intp'.")
field = 'integer'
elif kind == 'f':
field = 'real'
elif kind == 'c':
field = 'complex'
elif kind == 'u':
field = 'unsigned-integer'
else:
raise TypeError('unexpected dtype kind ' + kind)
if symmetry is None:
symmetry = self._get_symmetry(a)
# validate rep, field, and symmetry
self.__class__._validate_format(rep)
self.__class__._validate_field(field)
self.__class__._validate_symmetry(symmetry)
# write initial header line
stream.write(asbytes('%%MatrixMarket matrix {0} {1} {2}\n'.format(rep,
field, symmetry)))
# write comments
for line in comment.split('\n'):
stream.write(asbytes('%%%s\n' % (line)))
template = self._field_template(field, precision)
# write dense format
if rep == self.FORMAT_ARRAY:
# write shape spec
stream.write(asbytes('%i %i\n' % (rows, cols)))
if field in (self.FIELD_INTEGER, self.FIELD_REAL, self.FIELD_UNSIGNED):
if symmetry == self.SYMMETRY_GENERAL:
for j in range(cols):
for i in range(rows):
stream.write(asbytes(template % a[i, j]))
elif symmetry == self.SYMMETRY_SKEW_SYMMETRIC:
for j in range(cols):
for i in range(j + 1, rows):
stream.write(asbytes(template % a[i, j]))
else:
for j in range(cols):
for i in range(j, rows):
stream.write(asbytes(template % a[i, j]))
elif field == self.FIELD_COMPLEX:
if symmetry == self.SYMMETRY_GENERAL:
for j in range(cols):
for i in range(rows):
aij = a[i, j]
stream.write(asbytes(template % (real(aij),
imag(aij))))
else:
for j in range(cols):
for i in range(j, rows):
aij = a[i, j]
stream.write(asbytes(template % (real(aij),
imag(aij))))
elif field == self.FIELD_PATTERN:
raise ValueError('pattern type inconsisted with dense format')
else:
raise TypeError('Unknown field type %s' % field)
# write sparse format
else:
coo = a.tocoo() # convert to COOrdinate format
# if symmetry format used, remove values above main diagonal
if symmetry != self.SYMMETRY_GENERAL:
lower_triangle_mask = coo.row >= coo.col
coo = coo_matrix((coo.data[lower_triangle_mask],
(coo.row[lower_triangle_mask],
coo.col[lower_triangle_mask])),
shape=coo.shape)
# write shape spec
stream.write(asbytes('%i %i %i\n' % (rows, cols, coo.nnz)))
template = self._field_template(field, precision-1)
if field == self.FIELD_PATTERN:
for r, c in zip(coo.row+1, coo.col+1):
stream.write(asbytes("%i %i\n" % (r, c)))
elif field in (self.FIELD_INTEGER, self.FIELD_REAL, self.FIELD_UNSIGNED):
for r, c, d in zip(coo.row+1, coo.col+1, coo.data):
stream.write(asbytes(("%i %i " % (r, c)) +
(template % d)))
elif field == self.FIELD_COMPLEX:
for r, c, d in zip(coo.row+1, coo.col+1, coo.data):
stream.write(asbytes(("%i %i " % (r, c)) +
(template % (d.real, d.imag))))
else:
raise TypeError('Unknown field type %s' % field)
def _is_fromfile_compatible(stream):
"""
Check whether `stream` is compatible with numpy.fromfile.
Passing a gzipped file object to ``fromfile/fromstring`` doesn't work with
Python3.
"""
if sys.version_info[0] < 3:
return True
bad_cls = []
try:
import gzip
bad_cls.append(gzip.GzipFile)
except ImportError:
pass
try:
import bz2
bad_cls.append(bz2.BZ2File)
except ImportError:
pass
bad_cls = tuple(bad_cls)
return not isinstance(stream, bad_cls)
# -----------------------------------------------------------------------------
if __name__ == '__main__':
import time
for filename in sys.argv[1:]:
print('Reading', filename, '...', end=' ')
sys.stdout.flush()
t = time.time()
mmread(filename)
print('took %s seconds' % (time.time() - t))
utils/venv/lib/python3.6/site-packages/scipy/io/netcdf.py
-1098
View File
File diff suppressed because it is too large Load Diff
utils/venv/lib/python3.6/site-packages/scipy/io/setup.py
-20
View File
@@ -1,20 +0,0 @@
from __future__ import division, print_function, absolute_import
def configuration(parent_package='',top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('io', parent_package, top_path)
config.add_extension('_test_fortran',
sources=['_test_fortran.pyf', '_test_fortran.f'])
config.add_data_dir('tests')
config.add_subpackage('matlab')
config.add_subpackage('arff')
config.add_subpackage('harwell_boeing')
return config
if __name__ == '__main__':
from numpy.distutils.core import setup
setup(**configuration(top_path='').todict())
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__init__.py
View File
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__pycache__/__init__.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__pycache__/test_fortran.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__pycache__/test_idl.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__pycache__/test_mmio.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__pycache__/test_netcdf.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__pycache__/test_paths.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/__pycache__/test_wavfile.cpython-36.pyc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_1d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_2d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_3d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_4d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_5d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_6d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_7d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_8d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_1d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_2d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_3d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_4d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_5d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_6d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_7d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/array_float32_pointer_8d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/example_1.nc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/example_2.nc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/example_3_maskedvals.nc
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-3x3d-2i.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-mixed.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-sf8-11x1x10.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-sf8-15x10x22.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-sf8-1x1x1.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-sf8-1x1x5.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-sf8-1x1x7.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-sf8-1x3x5.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-si4-11x1x10.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-si4-15x10x22.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-si4-1x1x1.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-si4-1x1x5.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-si4-1x1x7.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/fortran-si4-1x3x5.dat
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/invalid_pointer.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/null_pointer.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_byte.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_byte_descr.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_complex32.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_complex64.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_float32.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_float64.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_heap_pointer.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_int16.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_int32.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_int64.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_string.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_uint16.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_uint32.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/scalar_uint64.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_arrays.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_arrays_byte_idl80.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_arrays_replicated.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_arrays_replicated_3d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_inherit.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_pointer_arrays.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_pointer_arrays_replicated.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_pointer_arrays_replicated_3d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_pointers.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_pointers_replicated.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_pointers_replicated_3d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_scalars.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_scalars_replicated.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/struct_scalars_replicated_3d.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/test-44100Hz-2ch-32bit-float-be.wav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/test-44100Hz-2ch-32bit-float-le.wav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/test-44100Hz-le-1ch-4bytes-early-eof.wav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/test-44100Hz-le-1ch-4bytes-incomplete-chunk.wav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/test-44100Hz-le-1ch-4bytes.wav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/test-48000Hz-2ch-64bit-float-le-wavex.wav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/test-8000Hz-le-2ch-1byteu.wav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/data/various_compressed.sav
BIN
View File
Binary file not shown.
utils/venv/lib/python3.6/site-packages/scipy/io/tests/test_fortran.py
-159
View File
@@ -1,159 +0,0 @@
''' Tests for fortran sequential files '''
import tempfile
import shutil
from os import path, unlink
from glob import iglob
import re
from numpy.testing import assert_equal, assert_allclose
import numpy as np
from scipy.io import FortranFile, _test_fortran
DATA_PATH = path.join(path.dirname(__file__), 'data')
def test_fortranfiles_read():
for filename in iglob(path.join(DATA_PATH, "fortran-*-*x*x*.dat")):
m = re.search(r'fortran-([^-]+)-(\d+)x(\d+)x(\d+).dat', filename, re.I)
if not m:
raise RuntimeError("Couldn't match %s filename to regex" % filename)
dims = (int(m.group(2)), int(m.group(3)), int(m.group(4)))
dtype = m.group(1).replace('s', '<')
f = FortranFile(filename, 'r', '<u4')
data = f.read_record(dtype=dtype).reshape(dims, order='F')
f.close()
expected = np.arange(np.prod(dims)).reshape(dims).astype(dtype)
assert_equal(data, expected)
def test_fortranfiles_mixed_record():
filename = path.join(DATA_PATH, "fortran-mixed.dat")
with FortranFile(filename, 'r', '<u4') as f:
record = f.read_record('<i4,<f4,<i8,(2)<f8')
assert_equal(record['f0'][0], 1)
assert_allclose(record['f1'][0], 2.3)
assert_equal(record['f2'][0], 4)
assert_allclose(record['f3'][0], [5.6, 7.8])
def test_fortranfiles_write():
for filename in iglob(path.join(DATA_PATH, "fortran-*-*x*x*.dat")):
m = re.search(r'fortran-([^-]+)-(\d+)x(\d+)x(\d+).dat', filename, re.I)
if not m:
raise RuntimeError("Couldn't match %s filename to regex" % filename)
dims = (int(m.group(2)), int(m.group(3)), int(m.group(4)))
dtype = m.group(1).replace('s', '<')
data = np.arange(np.prod(dims)).reshape(dims).astype(dtype)
tmpdir = tempfile.mkdtemp()
try:
testFile = path.join(tmpdir,path.basename(filename))
f = FortranFile(testFile, 'w','<u4')
f.write_record(data.T)
f.close()
originalfile = open(filename, 'rb')
newfile = open(testFile, 'rb')
assert_equal(originalfile.read(), newfile.read(),
err_msg=filename)
originalfile.close()
newfile.close()
finally:
shutil.rmtree(tmpdir)
def test_fortranfile_read_mixed_record():
# The data file fortran-3x3d-2i.dat contains the program that
# produced it at the end.
#
# double precision :: a(3,3)
# integer :: b(2)
# ...
# open(1, file='fortran-3x3d-2i.dat', form='unformatted')
# write(1) a, b
# close(1)
#
filename = path.join(DATA_PATH, "fortran-3x3d-2i.dat")
with FortranFile(filename, 'r', '<u4') as f:
record = f.read_record('(3,3)f8', '2i4')
ax = np.arange(3*3).reshape(3, 3).astype(np.double)
bx = np.array([-1, -2], dtype=np.int32)
assert_equal(record[0], ax.T)
assert_equal(record[1], bx.T)
def test_fortranfile_write_mixed_record(tmpdir):
tf = path.join(str(tmpdir), 'test.dat')
records = [
(('f4', 'f4', 'i4'), (np.float32(2), np.float32(3), np.int32(100))),
(('4f4', '(3,3)f4', '8i4'), (np.random.randint(255, size=[4]).astype(np.float32),
np.random.randint(255, size=[3, 3]).astype(np.float32),
np.random.randint(255, size=[8]).astype(np.int32)))
]
for dtype, a in records:
with FortranFile(tf, 'w') as f:
f.write_record(*a)
with FortranFile(tf, 'r') as f:
b = f.read_record(*dtype)
assert_equal(len(a), len(b))
for aa, bb in zip(a, b):
assert_equal(bb, aa)
def test_fortran_roundtrip(tmpdir):
filename = path.join(str(tmpdir), 'test.dat')
np.random.seed(1)
# double precision
m, n, k = 5, 3, 2
a = np.random.randn(m, n, k)
with FortranFile(filename, 'w') as f:
f.write_record(a.T)
a2 = _test_fortran.read_unformatted_double(m, n, k, filename)
with FortranFile(filename, 'r') as f:
a3 = f.read_record('(2,3,5)f8').T
assert_equal(a2, a)
assert_equal(a3, a)
# integer
m, n, k = 5, 3, 2
a = np.random.randn(m, n, k).astype(np.int32)
with FortranFile(filename, 'w') as f:
f.write_record(a.T)
a2 = _test_fortran.read_unformatted_int(m, n, k, filename)
with FortranFile(filename, 'r') as f:
a3 = f.read_record('(2,3,5)i4').T
assert_equal(a2, a)
assert_equal(a3, a)
# mixed
m, n, k = 5, 3, 2
a = np.random.randn(m, n)
b = np.random.randn(k).astype(np.intc)
with FortranFile(filename, 'w') as f:
f.write_record(a.T, b.T)
a2, b2 = _test_fortran.read_unformatted_mixed(m, n, k, filename)
with FortranFile(filename, 'r') as f:
a3, b3 = f.read_record('(3,5)f8', '2i4')
a3 = a3.T
assert_equal(a2, a)
assert_equal(a3, a)
assert_equal(b2, b)
assert_equal(b3, b)
utils/venv/lib/python3.6/site-packages/scipy/io/tests/test_idl.py
-442
View File
@@ -1,442 +0,0 @@
from __future__ import division, print_function, absolute_import
from os import path
import warnings
DATA_PATH = path.join(path.dirname(__file__), 'data')
import numpy as np
from numpy.testing import (assert_equal, assert_array_equal,
assert_)
from scipy._lib._numpy_compat import suppress_warnings
from scipy.io.idl import readsav
def object_array(*args):
"""Constructs a numpy array of objects"""
array = np.empty(len(args), dtype=object)
for i in range(len(args)):
array[i] = args[i]
return array
def assert_identical(a, b):
"""Assert whether value AND type are the same"""
assert_equal(a, b)
if type(b) is str:
assert_equal(type(a), type(b))
else:
assert_equal(np.asarray(a).dtype.type, np.asarray(b).dtype.type)
def assert_array_identical(a, b):
"""Assert whether values AND type are the same"""
assert_array_equal(a, b)
assert_equal(a.dtype.type, b.dtype.type)
# Define vectorized ID function for pointer arrays
vect_id = np.vectorize(id)
class TestIdict:
def test_idict(self):
custom_dict = {'a': np.int16(999)}
original_id = id(custom_dict)
s = readsav(path.join(DATA_PATH, 'scalar_byte.sav'), idict=custom_dict, verbose=False)
assert_equal(original_id, id(s))
assert_('a' in s)
assert_identical(s['a'], np.int16(999))
assert_identical(s['i8u'], np.uint8(234))
class TestScalars:
# Test that scalar values are read in with the correct value and type
def test_byte(self):
s = readsav(path.join(DATA_PATH, 'scalar_byte.sav'), verbose=False)
assert_identical(s.i8u, np.uint8(234))
def test_int16(self):
s = readsav(path.join(DATA_PATH, 'scalar_int16.sav'), verbose=False)
assert_identical(s.i16s, np.int16(-23456))
def test_int32(self):
s = readsav(path.join(DATA_PATH, 'scalar_int32.sav'), verbose=False)
assert_identical(s.i32s, np.int32(-1234567890))
def test_float32(self):
s = readsav(path.join(DATA_PATH, 'scalar_float32.sav'), verbose=False)
assert_identical(s.f32, np.float32(-3.1234567e+37))
def test_float64(self):
s = readsav(path.join(DATA_PATH, 'scalar_float64.sav'), verbose=False)
assert_identical(s.f64, np.float64(-1.1976931348623157e+307))
def test_complex32(self):
s = readsav(path.join(DATA_PATH, 'scalar_complex32.sav'), verbose=False)
assert_identical(s.c32, np.complex64(3.124442e13-2.312442e31j))
def test_bytes(self):
s = readsav(path.join(DATA_PATH, 'scalar_string.sav'), verbose=False)
assert_identical(s.s, np.bytes_("The quick brown fox jumps over the lazy python"))
def test_structure(self):
pass
def test_complex64(self):
s = readsav(path.join(DATA_PATH, 'scalar_complex64.sav'), verbose=False)
assert_identical(s.c64, np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j))
def test_heap_pointer(self):
pass
def test_object_reference(self):
pass
def test_uint16(self):
s = readsav(path.join(DATA_PATH, 'scalar_uint16.sav'), verbose=False)
assert_identical(s.i16u, np.uint16(65511))
def test_uint32(self):
s = readsav(path.join(DATA_PATH, 'scalar_uint32.sav'), verbose=False)
assert_identical(s.i32u, np.uint32(4294967233))
def test_int64(self):
s = readsav(path.join(DATA_PATH, 'scalar_int64.sav'), verbose=False)
assert_identical(s.i64s, np.int64(-9223372036854774567))
def test_uint64(self):
s = readsav(path.join(DATA_PATH, 'scalar_uint64.sav'), verbose=False)
assert_identical(s.i64u, np.uint64(18446744073709529285))
class TestCompressed(TestScalars):
# Test that compressed .sav files can be read in
def test_compressed(self):
s = readsav(path.join(DATA_PATH, 'various_compressed.sav'), verbose=False)
assert_identical(s.i8u, np.uint8(234))
assert_identical(s.f32, np.float32(-3.1234567e+37))
assert_identical(s.c64, np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j))
assert_equal(s.array5d.shape, (4, 3, 4, 6, 5))
assert_identical(s.arrays.a[0], np.array([1, 2, 3], dtype=np.int16))
assert_identical(s.arrays.b[0], np.array([4., 5., 6., 7.], dtype=np.float32))
assert_identical(s.arrays.c[0], np.array([np.complex64(1+2j), np.complex64(7+8j)]))
assert_identical(s.arrays.d[0], np.array([b"cheese", b"bacon", b"spam"], dtype=object))
class TestArrayDimensions:
# Test that multi-dimensional arrays are read in with the correct dimensions
def test_1d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_1d.sav'), verbose=False)
assert_equal(s.array1d.shape, (123, ))
def test_2d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_2d.sav'), verbose=False)
assert_equal(s.array2d.shape, (22, 12))
def test_3d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_3d.sav'), verbose=False)
assert_equal(s.array3d.shape, (11, 22, 12))
def test_4d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_4d.sav'), verbose=False)
assert_equal(s.array4d.shape, (4, 5, 8, 7))
def test_5d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_5d.sav'), verbose=False)
assert_equal(s.array5d.shape, (4, 3, 4, 6, 5))
def test_6d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_6d.sav'), verbose=False)
assert_equal(s.array6d.shape, (3, 6, 4, 5, 3, 4))
def test_7d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_7d.sav'), verbose=False)
assert_equal(s.array7d.shape, (2, 1, 2, 3, 4, 3, 2))
def test_8d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_8d.sav'), verbose=False)
assert_equal(s.array8d.shape, (4, 3, 2, 1, 2, 3, 5, 4))
class TestStructures:
def test_scalars(self):
s = readsav(path.join(DATA_PATH, 'struct_scalars.sav'), verbose=False)
assert_identical(s.scalars.a, np.array(np.int16(1)))
assert_identical(s.scalars.b, np.array(np.int32(2)))
assert_identical(s.scalars.c, np.array(np.float32(3.)))
assert_identical(s.scalars.d, np.array(np.float64(4.)))
assert_identical(s.scalars.e, np.array([b"spam"], dtype=object))
assert_identical(s.scalars.f, np.array(np.complex64(-1.+3j)))
def test_scalars_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_scalars_replicated.sav'), verbose=False)
assert_identical(s.scalars_rep.a, np.repeat(np.int16(1), 5))
assert_identical(s.scalars_rep.b, np.repeat(np.int32(2), 5))
assert_identical(s.scalars_rep.c, np.repeat(np.float32(3.), 5))
assert_identical(s.scalars_rep.d, np.repeat(np.float64(4.), 5))
assert_identical(s.scalars_rep.e, np.repeat(b"spam", 5).astype(object))
assert_identical(s.scalars_rep.f, np.repeat(np.complex64(-1.+3j), 5))
def test_scalars_replicated_3d(self):
s = readsav(path.join(DATA_PATH, 'struct_scalars_replicated_3d.sav'), verbose=False)
assert_identical(s.scalars_rep.a, np.repeat(np.int16(1), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.b, np.repeat(np.int32(2), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.c, np.repeat(np.float32(3.), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.d, np.repeat(np.float64(4.), 24).reshape(4, 3, 2))
assert_identical(s.scalars_rep.e, np.repeat(b"spam", 24).reshape(4, 3, 2).astype(object))
assert_identical(s.scalars_rep.f, np.repeat(np.complex64(-1.+3j), 24).reshape(4, 3, 2))
def test_arrays(self):
s = readsav(path.join(DATA_PATH, 'struct_arrays.sav'), verbose=False)
assert_array_identical(s.arrays.a[0], np.array([1, 2, 3], dtype=np.int16))
assert_array_identical(s.arrays.b[0], np.array([4., 5., 6., 7.], dtype=np.float32))
assert_array_identical(s.arrays.c[0], np.array([np.complex64(1+2j), np.complex64(7+8j)]))
assert_array_identical(s.arrays.d[0], np.array([b"cheese", b"bacon", b"spam"], dtype=object))
def test_arrays_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_arrays_replicated.sav'), verbose=False)
# Check column types
assert_(s.arrays_rep.a.dtype.type is np.object_)
assert_(s.arrays_rep.b.dtype.type is np.object_)
assert_(s.arrays_rep.c.dtype.type is np.object_)
assert_(s.arrays_rep.d.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.a.shape, (5, ))
assert_equal(s.arrays_rep.b.shape, (5, ))
assert_equal(s.arrays_rep.c.shape, (5, ))
assert_equal(s.arrays_rep.d.shape, (5, ))
# Check values
for i in range(5):
assert_array_identical(s.arrays_rep.a[i],
np.array([1, 2, 3], dtype=np.int16))
assert_array_identical(s.arrays_rep.b[i],
np.array([4., 5., 6., 7.], dtype=np.float32))
assert_array_identical(s.arrays_rep.c[i],
np.array([np.complex64(1+2j),
np.complex64(7+8j)]))
assert_array_identical(s.arrays_rep.d[i],
np.array([b"cheese", b"bacon", b"spam"],
dtype=object))
def test_arrays_replicated_3d(self):
s = readsav(path.join(DATA_PATH, 'struct_arrays_replicated_3d.sav'), verbose=False)
# Check column types
assert_(s.arrays_rep.a.dtype.type is np.object_)
assert_(s.arrays_rep.b.dtype.type is np.object_)
assert_(s.arrays_rep.c.dtype.type is np.object_)
assert_(s.arrays_rep.d.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.a.shape, (4, 3, 2))
assert_equal(s.arrays_rep.b.shape, (4, 3, 2))
assert_equal(s.arrays_rep.c.shape, (4, 3, 2))
assert_equal(s.arrays_rep.d.shape, (4, 3, 2))
# Check values
for i in range(4):
for j in range(3):
for k in range(2):
assert_array_identical(s.arrays_rep.a[i, j, k],
np.array([1, 2, 3], dtype=np.int16))
assert_array_identical(s.arrays_rep.b[i, j, k],
np.array([4., 5., 6., 7.],
dtype=np.float32))
assert_array_identical(s.arrays_rep.c[i, j, k],
np.array([np.complex64(1+2j),
np.complex64(7+8j)]))
assert_array_identical(s.arrays_rep.d[i, j, k],
np.array([b"cheese", b"bacon", b"spam"],
dtype=object))
def test_inheritance(self):
s = readsav(path.join(DATA_PATH, 'struct_inherit.sav'), verbose=False)
assert_identical(s.fc.x, np.array([0], dtype=np.int16))
assert_identical(s.fc.y, np.array([0], dtype=np.int16))
assert_identical(s.fc.r, np.array([0], dtype=np.int16))
assert_identical(s.fc.c, np.array([4], dtype=np.int16))
def test_arrays_corrupt_idl80(self):
# test byte arrays with missing nbyte information from IDL 8.0 .sav file
with suppress_warnings() as sup:
sup.filter(UserWarning, "Not able to verify number of bytes from header")
s = readsav(path.join(DATA_PATH,'struct_arrays_byte_idl80.sav'),
verbose=False)
assert_identical(s.y.x[0], np.array([55,66], dtype=np.uint8))
class TestPointers:
# Check that pointers in .sav files produce references to the same object in Python
def test_pointers(self):
s = readsav(path.join(DATA_PATH, 'scalar_heap_pointer.sav'), verbose=False)
assert_identical(s.c64_pointer1, np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j))
assert_identical(s.c64_pointer2, np.complex128(1.1987253647623157e+112-5.1987258887729157e+307j))
assert_(s.c64_pointer1 is s.c64_pointer2)
class TestPointerArray:
# Test that pointers in arrays are correctly read in
def test_1d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_1d.sav'), verbose=False)
assert_equal(s.array1d.shape, (123, ))
assert_(np.all(s.array1d == np.float32(4.)))
assert_(np.all(vect_id(s.array1d) == id(s.array1d[0])))
def test_2d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_2d.sav'), verbose=False)
assert_equal(s.array2d.shape, (22, 12))
assert_(np.all(s.array2d == np.float32(4.)))
assert_(np.all(vect_id(s.array2d) == id(s.array2d[0,0])))
def test_3d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_3d.sav'), verbose=False)
assert_equal(s.array3d.shape, (11, 22, 12))
assert_(np.all(s.array3d == np.float32(4.)))
assert_(np.all(vect_id(s.array3d) == id(s.array3d[0,0,0])))
def test_4d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_4d.sav'), verbose=False)
assert_equal(s.array4d.shape, (4, 5, 8, 7))
assert_(np.all(s.array4d == np.float32(4.)))
assert_(np.all(vect_id(s.array4d) == id(s.array4d[0,0,0,0])))
def test_5d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_5d.sav'), verbose=False)
assert_equal(s.array5d.shape, (4, 3, 4, 6, 5))
assert_(np.all(s.array5d == np.float32(4.)))
assert_(np.all(vect_id(s.array5d) == id(s.array5d[0,0,0,0,0])))
def test_6d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_6d.sav'), verbose=False)
assert_equal(s.array6d.shape, (3, 6, 4, 5, 3, 4))
assert_(np.all(s.array6d == np.float32(4.)))
assert_(np.all(vect_id(s.array6d) == id(s.array6d[0,0,0,0,0,0])))
def test_7d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_7d.sav'), verbose=False)
assert_equal(s.array7d.shape, (2, 1, 2, 3, 4, 3, 2))
assert_(np.all(s.array7d == np.float32(4.)))
assert_(np.all(vect_id(s.array7d) == id(s.array7d[0,0,0,0,0,0,0])))
def test_8d(self):
s = readsav(path.join(DATA_PATH, 'array_float32_pointer_8d.sav'), verbose=False)
assert_equal(s.array8d.shape, (4, 3, 2, 1, 2, 3, 5, 4))
assert_(np.all(s.array8d == np.float32(4.)))
assert_(np.all(vect_id(s.array8d) == id(s.array8d[0,0,0,0,0,0,0,0])))
class TestPointerStructures:
# Test that structures are correctly read in
def test_scalars(self):
s = readsav(path.join(DATA_PATH, 'struct_pointers.sav'), verbose=False)
assert_identical(s.pointers.g, np.array(np.float32(4.), dtype=np.object_))
assert_identical(s.pointers.h, np.array(np.float32(4.), dtype=np.object_))
assert_(id(s.pointers.g[0]) == id(s.pointers.h[0]))
def test_pointers_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_pointers_replicated.sav'), verbose=False)
assert_identical(s.pointers_rep.g, np.repeat(np.float32(4.), 5).astype(np.object_))
assert_identical(s.pointers_rep.h, np.repeat(np.float32(4.), 5).astype(np.object_))
assert_(np.all(vect_id(s.pointers_rep.g) == vect_id(s.pointers_rep.h)))
def test_pointers_replicated_3d(self):
s = readsav(path.join(DATA_PATH, 'struct_pointers_replicated_3d.sav'), verbose=False)
s_expect = np.repeat(np.float32(4.), 24).reshape(4, 3, 2).astype(np.object_)
assert_identical(s.pointers_rep.g, s_expect)
assert_identical(s.pointers_rep.h, s_expect)
assert_(np.all(vect_id(s.pointers_rep.g) == vect_id(s.pointers_rep.h)))
def test_arrays(self):
s = readsav(path.join(DATA_PATH, 'struct_pointer_arrays.sav'), verbose=False)
assert_array_identical(s.arrays.g[0], np.repeat(np.float32(4.), 2).astype(np.object_))
assert_array_identical(s.arrays.h[0], np.repeat(np.float32(4.), 3).astype(np.object_))
assert_(np.all(vect_id(s.arrays.g[0]) == id(s.arrays.g[0][0])))
assert_(np.all(vect_id(s.arrays.h[0]) == id(s.arrays.h[0][0])))
assert_(id(s.arrays.g[0][0]) == id(s.arrays.h[0][0]))
def test_arrays_replicated(self):
s = readsav(path.join(DATA_PATH, 'struct_pointer_arrays_replicated.sav'), verbose=False)
# Check column types
assert_(s.arrays_rep.g.dtype.type is np.object_)
assert_(s.arrays_rep.h.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.g.shape, (5, ))
assert_equal(s.arrays_rep.h.shape, (5, ))
# Check values
for i in range(5):
assert_array_identical(s.arrays_rep.g[i], np.repeat(np.float32(4.), 2).astype(np.object_))
assert_array_identical(s.arrays_rep.h[i], np.repeat(np.float32(4.), 3).astype(np.object_))
assert_(np.all(vect_id(s.arrays_rep.g[i]) == id(s.arrays_rep.g[0][0])))
assert_(np.all(vect_id(s.arrays_rep.h[i]) == id(s.arrays_rep.h[0][0])))
def test_arrays_replicated_3d(self):
pth = path.join(DATA_PATH, 'struct_pointer_arrays_replicated_3d.sav')
s = readsav(pth, verbose=False)
# Check column types
assert_(s.arrays_rep.g.dtype.type is np.object_)
assert_(s.arrays_rep.h.dtype.type is np.object_)
# Check column shapes
assert_equal(s.arrays_rep.g.shape, (4, 3, 2))
assert_equal(s.arrays_rep.h.shape, (4, 3, 2))
# Check values
for i in range(4):
for j in range(3):
for k in range(2):
assert_array_identical(s.arrays_rep.g[i, j, k],
np.repeat(np.float32(4.), 2).astype(np.object_))
assert_array_identical(s.arrays_rep.h[i, j, k],
np.repeat(np.float32(4.), 3).astype(np.object_))
assert_(np.all(vect_id(s.arrays_rep.g[i, j, k]) == id(s.arrays_rep.g[0, 0, 0][0])))
assert_(np.all(vect_id(s.arrays_rep.h[i, j, k]) == id(s.arrays_rep.h[0, 0, 0][0])))
class TestTags:
'''Test that sav files with description tag read at all'''
def test_description(self):
s = readsav(path.join(DATA_PATH, 'scalar_byte_descr.sav'), verbose=False)
assert_identical(s.i8u, np.uint8(234))
def test_null_pointer():
# Regression test for null pointers.
s = readsav(path.join(DATA_PATH, 'null_pointer.sav'), verbose=False)
assert_identical(s.point, None)
assert_identical(s.check, np.int16(5))
def test_invalid_pointer():
# Regression test for invalid pointers (gh-4613).
# In some files in the wild, pointers can sometimes refer to a heap
# variable that does not exist. In that case, we now gracefully fail for
# that variable and replace the variable with None and emit a warning.
# Since it's difficult to artificially produce such files, the file used
# here has been edited to force the pointer reference to be invalid.
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
s = readsav(path.join(DATA_PATH, 'invalid_pointer.sav'), verbose=False)
assert_(len(w) == 1)
assert_(str(w[0].message) == ("Variable referenced by pointer not found in "
"heap: variable will be set to None"))
assert_identical(s['a'], np.array([None, None]))
utils/venv/lib/python3.6/site-packages/scipy/io/tests/test_mmio.py
-673
View File
@@ -1,673 +0,0 @@
from __future__ import division, print_function, absolute_import
from tempfile import mkdtemp, mktemp
import os
import shutil
import numpy as np
from numpy import array, transpose, pi
from numpy.testing import (assert_equal,
assert_array_equal, assert_array_almost_equal)
import pytest
from pytest import raises as assert_raises
import scipy.sparse
from scipy.io.mmio import mminfo, mmread, mmwrite
parametrize_args = [('integer', 'int'),
('unsigned-integer', 'uint')]
class TestMMIOArray(object):
def setup_method(self):
self.tmpdir = mkdtemp()
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn)
assert_array_almost_equal(a, b)
def check_exact(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn)
assert_equal(a, b)
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_integer(self, typeval, dtype):
self.check_exact(array([[1, 2], [3, 4]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_32bit_integer(self, typeval, dtype):
a = array([[2**31-1, 2**31-2], [2**31-3, 2**31-4]], dtype=dtype)
self.check_exact(a, (2, 2, 4, 'array', typeval, 'general'))
def test_64bit_integer(self):
a = array([[2**31, 2**32], [2**63-2, 2**63-1]], dtype=np.int64)
if (np.intp(0).itemsize < 8):
assert_raises(OverflowError, mmwrite, self.fn, a)
else:
self.check_exact(a, (2, 2, 4, 'array', 'integer', 'general'))
def test_64bit_unsigned_integer(self):
a = array([[2**31, 2**32], [2**64-2, 2**64-1]], dtype=np.uint64)
self.check_exact(a, (2, 2, 4, 'array', 'unsigned-integer', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_upper_triangle_integer(self, typeval, dtype):
self.check_exact(array([[0, 1], [0, 0]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_lower_triangle_integer(self, typeval, dtype):
self.check_exact(array([[0, 0], [1, 0]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_rectangular_integer(self, typeval, dtype):
self.check_exact(array([[1, 2, 3], [4, 5, 6]], dtype=dtype),
(2, 3, 6, 'array', typeval, 'general'))
def test_simple_rectangular_float(self):
self.check([[1, 2], [3.5, 4], [5, 6]],
(3, 2, 6, 'array', 'real', 'general'))
def test_simple_float(self):
self.check([[1, 2], [3, 4.0]],
(2, 2, 4, 'array', 'real', 'general'))
def test_simple_complex(self):
self.check([[1, 2], [3, 4j]],
(2, 2, 4, 'array', 'complex', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_symmetric_integer(self, typeval, dtype):
self.check_exact(array([[1, 2], [2, 4]], dtype=dtype),
(2, 2, 4, 'array', typeval, 'symmetric'))
def test_simple_skew_symmetric_integer(self):
self.check_exact([[0, 2], [-2, 0]],
(2, 2, 4, 'array', 'integer', 'skew-symmetric'))
def test_simple_skew_symmetric_float(self):
self.check(array([[0, 2], [-2.0, 0.0]], 'f'),
(2, 2, 4, 'array', 'real', 'skew-symmetric'))
def test_simple_hermitian_complex(self):
self.check([[1, 2+3j], [2-3j, 4]],
(2, 2, 4, 'array', 'complex', 'hermitian'))
def test_random_symmetric_float(self):
sz = (20, 20)
a = np.random.random(sz)
a = a + transpose(a)
self.check(a, (20, 20, 400, 'array', 'real', 'symmetric'))
def test_random_rectangular_float(self):
sz = (20, 15)
a = np.random.random(sz)
self.check(a, (20, 15, 300, 'array', 'real', 'general'))
class TestMMIOSparseCSR(TestMMIOArray):
def setup_method(self):
self.tmpdir = mkdtemp()
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn)
assert_array_almost_equal(a.todense(), b.todense())
def check_exact(self, a, info):
mmwrite(self.fn, a)
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn)
assert_equal(a.todense(), b.todense())
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_matrix([[1, 2], [3, 4]], dtype=dtype),
(2, 2, 4, 'coordinate', typeval, 'general'))
def test_32bit_integer(self):
a = scipy.sparse.csr_matrix(array([[2**31-1, -2**31+2],
[2**31-3, 2**31-4]],
dtype=np.int32))
self.check_exact(a, (2, 2, 4, 'coordinate', 'integer', 'general'))
def test_64bit_integer(self):
a = scipy.sparse.csr_matrix(array([[2**32+1, 2**32+1],
[-2**63+2, 2**63-2]],
dtype=np.int64))
if (np.intp(0).itemsize < 8):
assert_raises(OverflowError, mmwrite, self.fn, a)
else:
self.check_exact(a, (2, 2, 4, 'coordinate', 'integer', 'general'))
def test_32bit_unsigned_integer(self):
a = scipy.sparse.csr_matrix(array([[2**31-1, 2**31-2],
[2**31-3, 2**31-4]],
dtype=np.uint32))
self.check_exact(a, (2, 2, 4, 'coordinate', 'unsigned-integer', 'general'))
def test_64bit_unsigned_integer(self):
a = scipy.sparse.csr_matrix(array([[2**32+1, 2**32+1],
[2**64-2, 2**64-1]],
dtype=np.uint64))
self.check_exact(a, (2, 2, 4, 'coordinate', 'unsigned-integer', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_upper_triangle_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_matrix([[0, 1], [0, 0]], dtype=dtype),
(2, 2, 1, 'coordinate', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_lower_triangle_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_matrix([[0, 0], [1, 0]], dtype=dtype),
(2, 2, 1, 'coordinate', typeval, 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_rectangular_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_matrix([[1, 2, 3], [4, 5, 6]], dtype=dtype),
(2, 3, 6, 'coordinate', typeval, 'general'))
def test_simple_rectangular_float(self):
self.check(scipy.sparse.csr_matrix([[1, 2], [3.5, 4], [5, 6]]),
(3, 2, 6, 'coordinate', 'real', 'general'))
def test_simple_float(self):
self.check(scipy.sparse.csr_matrix([[1, 2], [3, 4.0]]),
(2, 2, 4, 'coordinate', 'real', 'general'))
def test_simple_complex(self):
self.check(scipy.sparse.csr_matrix([[1, 2], [3, 4j]]),
(2, 2, 4, 'coordinate', 'complex', 'general'))
@pytest.mark.parametrize('typeval, dtype', parametrize_args)
def test_simple_symmetric_integer(self, typeval, dtype):
self.check_exact(scipy.sparse.csr_matrix([[1, 2], [2, 4]], dtype=dtype),
(2, 2, 3, 'coordinate', typeval, 'symmetric'))
def test_simple_skew_symmetric_integer(self):
self.check_exact(scipy.sparse.csr_matrix([[1, 2], [-2, 4]]),
(2, 2, 3, 'coordinate', 'integer', 'skew-symmetric'))
def test_simple_skew_symmetric_float(self):
self.check(scipy.sparse.csr_matrix(array([[1, 2], [-2.0, 4]], 'f')),
(2, 2, 3, 'coordinate', 'real', 'skew-symmetric'))
def test_simple_hermitian_complex(self):
self.check(scipy.sparse.csr_matrix([[1, 2+3j], [2-3j, 4]]),
(2, 2, 3, 'coordinate', 'complex', 'hermitian'))
def test_random_symmetric_float(self):
sz = (20, 20)
a = np.random.random(sz)
a = a + transpose(a)
a = scipy.sparse.csr_matrix(a)
self.check(a, (20, 20, 210, 'coordinate', 'real', 'symmetric'))
def test_random_rectangular_float(self):
sz = (20, 15)
a = np.random.random(sz)
a = scipy.sparse.csr_matrix(a)
self.check(a, (20, 15, 300, 'coordinate', 'real', 'general'))
def test_simple_pattern(self):
a = scipy.sparse.csr_matrix([[0, 1.5], [3.0, 2.5]])
p = np.zeros_like(a.todense())
p[a.todense() > 0] = 1
info = (2, 2, 3, 'coordinate', 'pattern', 'general')
mmwrite(self.fn, a, field='pattern')
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn)
assert_array_almost_equal(p, b.todense())
_32bit_integer_dense_example = '''\
%%MatrixMarket matrix array integer general
2 2
2147483647
2147483646
2147483647
2147483646
'''
_32bit_integer_sparse_example = '''\
%%MatrixMarket matrix coordinate integer symmetric
2 2 2
1 1 2147483647
2 2 2147483646
'''
_64bit_integer_dense_example = '''\
%%MatrixMarket matrix array integer general
2 2
2147483648
-9223372036854775806
-2147483648
9223372036854775807
'''
_64bit_integer_sparse_general_example = '''\
%%MatrixMarket matrix coordinate integer general
2 2 3
1 1 2147483648
1 2 9223372036854775807
2 2 9223372036854775807
'''
_64bit_integer_sparse_symmetric_example = '''\
%%MatrixMarket matrix coordinate integer symmetric
2 2 3
1 1 2147483648
1 2 -9223372036854775807
2 2 9223372036854775807
'''
_64bit_integer_sparse_skew_example = '''\
%%MatrixMarket matrix coordinate integer skew-symmetric
2 2 3
1 1 2147483648
1 2 -9223372036854775807
2 2 9223372036854775807
'''
_over64bit_integer_dense_example = '''\
%%MatrixMarket matrix array integer general
2 2
2147483648
9223372036854775807
2147483648
9223372036854775808
'''
_over64bit_integer_sparse_example = '''\
%%MatrixMarket matrix coordinate integer symmetric
2 2 2
1 1 2147483648
2 2 19223372036854775808
'''
class TestMMIOReadLargeIntegers(object):
def setup_method(self):
self.tmpdir = mkdtemp()
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check_read(self, example, a, info, dense, over32, over64):
with open(self.fn, 'w') as f:
f.write(example)
assert_equal(mminfo(self.fn), info)
if (over32 and (np.intp(0).itemsize < 8)) or over64:
assert_raises(OverflowError, mmread, self.fn)
else:
b = mmread(self.fn)
if not dense:
b = b.todense()
assert_equal(a, b)
def test_read_32bit_integer_dense(self):
a = array([[2**31-1, 2**31-1],
[2**31-2, 2**31-2]], dtype=np.int64)
self.check_read(_32bit_integer_dense_example,
a,
(2, 2, 4, 'array', 'integer', 'general'),
dense=True,
over32=False,
over64=False)
def test_read_32bit_integer_sparse(self):
a = array([[2**31-1, 0],
[0, 2**31-2]], dtype=np.int64)
self.check_read(_32bit_integer_sparse_example,
a,
(2, 2, 2, 'coordinate', 'integer', 'symmetric'),
dense=False,
over32=False,
over64=False)
def test_read_64bit_integer_dense(self):
a = array([[2**31, -2**31],
[-2**63+2, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_dense_example,
a,
(2, 2, 4, 'array', 'integer', 'general'),
dense=True,
over32=True,
over64=False)
def test_read_64bit_integer_sparse_general(self):
a = array([[2**31, 2**63-1],
[0, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_sparse_general_example,
a,
(2, 2, 3, 'coordinate', 'integer', 'general'),
dense=False,
over32=True,
over64=False)
def test_read_64bit_integer_sparse_symmetric(self):
a = array([[2**31, -2**63+1],
[-2**63+1, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_sparse_symmetric_example,
a,
(2, 2, 3, 'coordinate', 'integer', 'symmetric'),
dense=False,
over32=True,
over64=False)
def test_read_64bit_integer_sparse_skew(self):
a = array([[2**31, -2**63+1],
[2**63-1, 2**63-1]], dtype=np.int64)
self.check_read(_64bit_integer_sparse_skew_example,
a,
(2, 2, 3, 'coordinate', 'integer', 'skew-symmetric'),
dense=False,
over32=True,
over64=False)
def test_read_over64bit_integer_dense(self):
self.check_read(_over64bit_integer_dense_example,
None,
(2, 2, 4, 'array', 'integer', 'general'),
dense=True,
over32=True,
over64=True)
def test_read_over64bit_integer_sparse(self):
self.check_read(_over64bit_integer_sparse_example,
None,
(2, 2, 2, 'coordinate', 'integer', 'symmetric'),
dense=False,
over32=True,
over64=True)
_general_example = '''\
%%MatrixMarket matrix coordinate real general
%=================================================================================
%
% This ASCII file represents a sparse MxN matrix with L
% nonzeros in the following Matrix Market format:
%
% +----------------------------------------------+
% |%%MatrixMarket matrix coordinate real general | <--- header line
% |% | <--+
% |% comments | |-- 0 or more comment lines
% |% | <--+
% | M N L | <--- rows, columns, entries
% | I1 J1 A(I1, J1) | <--+
% | I2 J2 A(I2, J2) | |
% | I3 J3 A(I3, J3) | |-- L lines
% | . . . | |
% | IL JL A(IL, JL) | <--+
% +----------------------------------------------+
%
% Indices are 1-based, i.e. A(1,1) is the first element.
%
%=================================================================================
5 5 8
1 1 1.000e+00
2 2 1.050e+01
3 3 1.500e-02
1 4 6.000e+00
4 2 2.505e+02
4 4 -2.800e+02
4 5 3.332e+01
5 5 1.200e+01
'''
_hermitian_example = '''\
%%MatrixMarket matrix coordinate complex hermitian
5 5 7
1 1 1.0 0
2 2 10.5 0
4 2 250.5 22.22
3 3 1.5e-2 0
4 4 -2.8e2 0
5 5 12. 0
5 4 0 33.32
'''
_skew_example = '''\
%%MatrixMarket matrix coordinate real skew-symmetric
5 5 7
1 1 1.0
2 2 10.5
4 2 250.5
3 3 1.5e-2
4 4 -2.8e2
5 5 12.
5 4 0
'''
_symmetric_example = '''\
%%MatrixMarket matrix coordinate real symmetric
5 5 7
1 1 1.0
2 2 10.5
4 2 250.5
3 3 1.5e-2
4 4 -2.8e2
5 5 12.
5 4 8
'''
_symmetric_pattern_example = '''\
%%MatrixMarket matrix coordinate pattern symmetric
5 5 7
1 1
2 2
4 2
3 3
4 4
5 5
5 4
'''
class TestMMIOCoordinate(object):
def setup_method(self):
self.tmpdir = mkdtemp()
self.fn = os.path.join(self.tmpdir, 'testfile.mtx')
def teardown_method(self):
shutil.rmtree(self.tmpdir)
def check_read(self, example, a, info):
f = open(self.fn, 'w')
f.write(example)
f.close()
assert_equal(mminfo(self.fn), info)
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_read_general(self):
a = [[1, 0, 0, 6, 0],
[0, 10.5, 0, 0, 0],
[0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 33.32],
[0, 0, 0, 0, 12]]
self.check_read(_general_example, a,
(5, 5, 8, 'coordinate', 'real', 'general'))
def test_read_hermitian(self):
a = [[1, 0, 0, 0, 0],
[0, 10.5, 0, 250.5 - 22.22j, 0],
[0, 0, .015, 0, 0],
[0, 250.5 + 22.22j, 0, -280, -33.32j],
[0, 0, 0, 33.32j, 12]]
self.check_read(_hermitian_example, a,
(5, 5, 7, 'coordinate', 'complex', 'hermitian'))
def test_read_skew(self):
a = [[1, 0, 0, 0, 0],
[0, 10.5, 0, -250.5, 0],
[0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 0],
[0, 0, 0, 0, 12]]
self.check_read(_skew_example, a,
(5, 5, 7, 'coordinate', 'real', 'skew-symmetric'))
def test_read_symmetric(self):
a = [[1, 0, 0, 0, 0],
[0, 10.5, 0, 250.5, 0],
[0, 0, .015, 0, 0],
[0, 250.5, 0, -280, 8],
[0, 0, 0, 8, 12]]
self.check_read(_symmetric_example, a,
(5, 5, 7, 'coordinate', 'real', 'symmetric'))
def test_read_symmetric_pattern(self):
a = [[1, 0, 0, 0, 0],
[0, 1, 0, 1, 0],
[0, 0, 1, 0, 0],
[0, 1, 0, 1, 1],
[0, 0, 0, 1, 1]]
self.check_read(_symmetric_pattern_example, a,
(5, 5, 7, 'coordinate', 'pattern', 'symmetric'))
def test_empty_write_read(self):
# https://github.com/scipy/scipy/issues/1410 (Trac #883)
b = scipy.sparse.coo_matrix((10, 10))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(10, 10, 0, 'coordinate', 'real', 'symmetric'))
a = b.todense()
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_bzip2_py3(self):
# test if fix for #2152 works
try:
# bz2 module isn't always built when building Python.
import bz2
except ImportError:
return
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
fn_bzip2 = "%s.bz2" % self.fn
with open(self.fn, 'rb') as f_in:
f_out = bz2.BZ2File(fn_bzip2, 'wb')
f_out.write(f_in.read())
f_out.close()
a = mmread(fn_bzip2).todense()
assert_array_almost_equal(a, b.todense())
def test_gzip_py3(self):
# test if fix for #2152 works
try:
# gzip module can be missing from Python installation
import gzip
except ImportError:
return
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
fn_gzip = "%s.gz" % self.fn
with open(self.fn, 'rb') as f_in:
f_out = gzip.open(fn_gzip, 'wb')
f_out.write(f_in.read())
f_out.close()
a = mmread(fn_gzip).todense()
assert_array_almost_equal(a, b.todense())
def test_real_write_read(self):
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(5, 5, 8, 'coordinate', 'real', 'general'))
a = b.todense()
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_complex_write_read(self):
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0 + 3j, 6.0 + 2j, 10.50 + 0.9j, 0.015 + -4.4j,
250.5 + 0j, -280.0 + 5j, 33.32 + 6.4j, 12.00 + 0.8j])
b = scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5))
mmwrite(self.fn, b)
assert_equal(mminfo(self.fn),
(5, 5, 8, 'coordinate', 'complex', 'general'))
a = b.todense()
b = mmread(self.fn).todense()
assert_array_almost_equal(a, b)
def test_sparse_formats(self):
mats = []
I = array([0, 0, 1, 2, 3, 3, 3, 4])
J = array([0, 3, 1, 2, 1, 3, 4, 4])
V = array([1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0])
mats.append(scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5)))
V = array([1.0 + 3j, 6.0 + 2j, 10.50 + 0.9j, 0.015 + -4.4j,
250.5 + 0j, -280.0 + 5j, 33.32 + 6.4j, 12.00 + 0.8j])
mats.append(scipy.sparse.coo_matrix((V, (I, J)), shape=(5, 5)))
for mat in mats:
expected = mat.todense()
for fmt in ['csr', 'csc', 'coo']:
fn = mktemp(dir=self.tmpdir) # safe, we own tmpdir
mmwrite(fn, mat.asformat(fmt))
result = mmread(fn).todense()
assert_array_almost_equal(result, expected)
def test_precision(self):
test_values = [pi] + [10**(i) for i in range(0, -10, -1)]
test_precisions = range(1, 10)
for value in test_values:
for precision in test_precisions:
# construct sparse matrix with test value at last main diagonal
n = 10**precision + 1
A = scipy.sparse.dok_matrix((n, n))
A[n-1, n-1] = value
# write matrix with test precision and read again
mmwrite(self.fn, A, precision=precision)
A = scipy.io.mmread(self.fn)
# check for right entries in matrix
assert_array_equal(A.row, [n-1])
assert_array_equal(A.col, [n-1])
assert_array_almost_equal(A.data,
[float('%%.%dg' % precision % value)])
utils/venv/lib/python3.6/site-packages/scipy/io/tests/test_netcdf.py
-544
View File
@@ -1,544 +0,0 @@
''' Tests for netcdf '''
from __future__ import division, print_function, absolute_import
import os
from os.path import join as pjoin, dirname
import shutil
import tempfile
import warnings
from io import BytesIO
from glob import glob
from contextlib import contextmanager
import numpy as np
from numpy.testing import assert_, assert_allclose, assert_equal
from pytest import raises as assert_raises
from scipy.io.netcdf import netcdf_file, IS_PYPY
from scipy._lib._numpy_compat import suppress_warnings
from scipy._lib._tmpdirs import in_tempdir
TEST_DATA_PATH = pjoin(dirname(__file__), 'data')
N_EG_ELS = 11 # number of elements for example variable
VARTYPE_EG = 'b' # var type for example variable
@contextmanager
def make_simple(*args, **kwargs):
f = netcdf_file(*args, **kwargs)
f.history = 'Created for a test'
f.createDimension('time', N_EG_ELS)
time = f.createVariable('time', VARTYPE_EG, ('time',))
time[:] = np.arange(N_EG_ELS)
time.units = 'days since 2008-01-01'
f.flush()
yield f
f.close()
def check_simple(ncfileobj):
'''Example fileobj tests '''
assert_equal(ncfileobj.history, b'Created for a test')
time = ncfileobj.variables['time']
assert_equal(time.units, b'days since 2008-01-01')
assert_equal(time.shape, (N_EG_ELS,))
assert_equal(time[-1], N_EG_ELS-1)
def assert_mask_matches(arr, expected_mask):
'''
Asserts that the mask of arr is effectively the same as expected_mask.
In contrast to numpy.ma.testutils.assert_mask_equal, this function allows
testing the 'mask' of a standard numpy array (the mask in this case is treated
as all False).
Parameters
----------
arr: ndarray or MaskedArray
Array to test.
expected_mask: array_like of booleans
A list giving the expected mask.
'''
mask = np.ma.getmaskarray(arr)
assert_equal(mask, expected_mask)
def test_read_write_files():
# test round trip for example file
cwd = os.getcwd()
try:
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
with make_simple('simple.nc', 'w') as f:
pass
# read the file we just created in 'a' mode
with netcdf_file('simple.nc', 'a') as f:
check_simple(f)
# add something
f._attributes['appendRan'] = 1
# To read the NetCDF file we just created::
with netcdf_file('simple.nc') as f:
# Using mmap is the default (but not on pypy)
assert_equal(f.use_mmap, not IS_PYPY)
check_simple(f)
assert_equal(f._attributes['appendRan'], 1)
# Read it in append (and check mmap is off)
with netcdf_file('simple.nc', 'a') as f:
assert_(not f.use_mmap)
check_simple(f)
assert_equal(f._attributes['appendRan'], 1)
# Now without mmap
with netcdf_file('simple.nc', mmap=False) as f:
# Using mmap is the default
assert_(not f.use_mmap)
check_simple(f)
# To read the NetCDF file we just created, as file object, no
# mmap. When n * n_bytes(var_type) is not divisible by 4, this
# raised an error in pupynere 1.0.12 and scipy rev 5893, because
# calculated vsize was rounding up in units of 4 - see
# https://www.unidata.ucar.edu/software/netcdf/docs/user_guide.html
with open('simple.nc', 'rb') as fobj:
with netcdf_file(fobj) as f:
# by default, don't use mmap for file-like
assert_(not f.use_mmap)
check_simple(f)
# Read file from fileobj, with mmap
with suppress_warnings() as sup:
if IS_PYPY:
sup.filter(RuntimeWarning,
"Cannot close a netcdf_file opened with mmap=True.*")
with open('simple.nc', 'rb') as fobj:
with netcdf_file(fobj, mmap=True) as f:
assert_(f.use_mmap)
check_simple(f)
# Again read it in append mode (adding another att)
with open('simple.nc', 'r+b') as fobj:
with netcdf_file(fobj, 'a') as f:
assert_(not f.use_mmap)
check_simple(f)
f.createDimension('app_dim', 1)
var = f.createVariable('app_var', 'i', ('app_dim',))
var[:] = 42
# And... check that app_var made it in...
with netcdf_file('simple.nc') as f:
check_simple(f)
assert_equal(f.variables['app_var'][:], 42)
except: # noqa: E722
os.chdir(cwd)
shutil.rmtree(tmpdir)
raise
os.chdir(cwd)
shutil.rmtree(tmpdir)
def test_read_write_sio():
eg_sio1 = BytesIO()
with make_simple(eg_sio1, 'w') as f1:
str_val = eg_sio1.getvalue()
eg_sio2 = BytesIO(str_val)
with netcdf_file(eg_sio2) as f2:
check_simple(f2)
# Test that error is raised if attempting mmap for sio
eg_sio3 = BytesIO(str_val)
assert_raises(ValueError, netcdf_file, eg_sio3, 'r', True)
# Test 64-bit offset write / read
eg_sio_64 = BytesIO()
with make_simple(eg_sio_64, 'w', version=2) as f_64:
str_val = eg_sio_64.getvalue()
eg_sio_64 = BytesIO(str_val)
with netcdf_file(eg_sio_64) as f_64:
check_simple(f_64)
assert_equal(f_64.version_byte, 2)
# also when version 2 explicitly specified
eg_sio_64 = BytesIO(str_val)
with netcdf_file(eg_sio_64, version=2) as f_64:
check_simple(f_64)
assert_equal(f_64.version_byte, 2)
def test_bytes():
raw_file = BytesIO()
f = netcdf_file(raw_file, mode='w')
# Dataset only has a single variable, dimension and attribute to avoid
# any ambiguity related to order.
f.a = 'b'
f.createDimension('dim', 1)
var = f.createVariable('var', np.int16, ('dim',))
var[0] = -9999
var.c = 'd'
f.sync()
actual = raw_file.getvalue()
expected = (b'CDF\x01'
b'\x00\x00\x00\x00'
b'\x00\x00\x00\x0a'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x03'
b'dim\x00'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x0c'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x01'
b'a\x00\x00\x00'
b'\x00\x00\x00\x02'
b'\x00\x00\x00\x01'
b'b\x00\x00\x00'
b'\x00\x00\x00\x0b'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x03'
b'var\x00'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x00'
b'\x00\x00\x00\x0c'
b'\x00\x00\x00\x01'
b'\x00\x00\x00\x01'
b'c\x00\x00\x00'
b'\x00\x00\x00\x02'
b'\x00\x00\x00\x01'
b'd\x00\x00\x00'
b'\x00\x00\x00\x03'
b'\x00\x00\x00\x04'
b'\x00\x00\x00\x78'
b'\xd8\xf1\x80\x01')
assert_equal(actual, expected)
def test_encoded_fill_value():
with netcdf_file(BytesIO(), mode='w') as f:
f.createDimension('x', 1)
var = f.createVariable('var', 'S1', ('x',))
assert_equal(var._get_encoded_fill_value(), b'\x00')
var._FillValue = b'\x01'
assert_equal(var._get_encoded_fill_value(), b'\x01')
var._FillValue = b'\x00\x00' # invalid, wrong size
assert_equal(var._get_encoded_fill_value(), b'\x00')
def test_read_example_data():
# read any example data files
for fname in glob(pjoin(TEST_DATA_PATH, '*.nc')):
with netcdf_file(fname, 'r') as f:
pass
with netcdf_file(fname, 'r', mmap=False) as f:
pass
def test_itemset_no_segfault_on_readonly():
# Regression test for ticket #1202.
# Open the test file in read-only mode.
filename = pjoin(TEST_DATA_PATH, 'example_1.nc')
with suppress_warnings() as sup:
sup.filter(RuntimeWarning,
"Cannot close a netcdf_file opened with mmap=True, when netcdf_variables or arrays referring to its data still exist")
with netcdf_file(filename, 'r', mmap=True) as f:
time_var = f.variables['time']
# time_var.assignValue(42) should raise a RuntimeError--not seg. fault!
assert_raises(RuntimeError, time_var.assignValue, 42)
def test_appending_issue_gh_8625():
stream = BytesIO()
with make_simple(stream, mode='w') as f:
f.createDimension('x', 2)
f.createVariable('x', float, ('x',))
f.variables['x'][...] = 1
f.flush()
contents = stream.getvalue()
stream = BytesIO(contents)
with netcdf_file(stream, mode='a') as f:
f.variables['x'][...] = 2
def test_write_invalid_dtype():
dtypes = ['int64', 'uint64']
if np.dtype('int').itemsize == 8: # 64-bit machines
dtypes.append('int')
if np.dtype('uint').itemsize == 8: # 64-bit machines
dtypes.append('uint')
with netcdf_file(BytesIO(), 'w') as f:
f.createDimension('time', N_EG_ELS)
for dt in dtypes:
assert_raises(ValueError, f.createVariable, 'time', dt, ('time',))
def test_flush_rewind():
stream = BytesIO()
with make_simple(stream, mode='w') as f:
x = f.createDimension('x',4)
v = f.createVariable('v', 'i2', ['x'])
v[:] = 1
f.flush()
len_single = len(stream.getvalue())
f.flush()
len_double = len(stream.getvalue())
assert_(len_single == len_double)
def test_dtype_specifiers():
# Numpy 1.7.0-dev had a bug where 'i2' wouldn't work.
# Specifying np.int16 or similar only works from the same commit as this
# comment was made.
with make_simple(BytesIO(), mode='w') as f:
f.createDimension('x',4)
f.createVariable('v1', 'i2', ['x'])
f.createVariable('v2', np.int16, ['x'])
f.createVariable('v3', np.dtype(np.int16), ['x'])
def test_ticket_1720():
io = BytesIO()
items = [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
with netcdf_file(io, 'w') as f:
f.history = 'Created for a test'
f.createDimension('float_var', 10)
float_var = f.createVariable('float_var', 'f', ('float_var',))
float_var[:] = items
float_var.units = 'metres'
f.flush()
contents = io.getvalue()
io = BytesIO(contents)
with netcdf_file(io, 'r') as f:
assert_equal(f.history, b'Created for a test')
float_var = f.variables['float_var']
assert_equal(float_var.units, b'metres')
assert_equal(float_var.shape, (10,))
assert_allclose(float_var[:], items)
def test_mmaps_segfault():
filename = pjoin(TEST_DATA_PATH, 'example_1.nc')
if not IS_PYPY:
with warnings.catch_warnings():
warnings.simplefilter("error")
with netcdf_file(filename, mmap=True) as f:
x = f.variables['lat'][:]
# should not raise warnings
del x
def doit():
with netcdf_file(filename, mmap=True) as f:
return f.variables['lat'][:]
# should not crash
with suppress_warnings() as sup:
sup.filter(RuntimeWarning,
"Cannot close a netcdf_file opened with mmap=True, when netcdf_variables or arrays referring to its data still exist")
x = doit()
x.sum()
def test_zero_dimensional_var():
io = BytesIO()
with make_simple(io, 'w') as f:
v = f.createVariable('zerodim', 'i2', [])
# This is checking that .isrec returns a boolean - don't simplify it
# to 'assert not ...'
assert v.isrec is False, v.isrec
f.flush()
def test_byte_gatts():
# Check that global "string" atts work like they did before py3k
# unicode and general bytes confusion
with in_tempdir():
filename = 'g_byte_atts.nc'
f = netcdf_file(filename, 'w')
f._attributes['holy'] = b'grail'
f._attributes['witch'] = 'floats'
f.close()
f = netcdf_file(filename, 'r')
assert_equal(f._attributes['holy'], b'grail')
assert_equal(f._attributes['witch'], b'floats')
f.close()
def test_open_append():
# open 'w' put one attr
with in_tempdir():
filename = 'append_dat.nc'
f = netcdf_file(filename, 'w')
f._attributes['Kilroy'] = 'was here'
f.close()
# open again in 'a', read the att and and a new one
f = netcdf_file(filename, 'a')
assert_equal(f._attributes['Kilroy'], b'was here')
f._attributes['naughty'] = b'Zoot'
f.close()
# open yet again in 'r' and check both atts
f = netcdf_file(filename, 'r')
assert_equal(f._attributes['Kilroy'], b'was here')
assert_equal(f._attributes['naughty'], b'Zoot')
f.close()
def test_append_recordDimension():
dataSize = 100
with in_tempdir():
# Create file with record time dimension
with netcdf_file('withRecordDimension.nc', 'w') as f:
f.createDimension('time', None)
f.createVariable('time', 'd', ('time',))
f.createDimension('x', dataSize)
x = f.createVariable('x', 'd', ('x',))
x[:] = np.array(range(dataSize))
f.createDimension('y', dataSize)
y = f.createVariable('y', 'd', ('y',))
y[:] = np.array(range(dataSize))
f.createVariable('testData', 'i', ('time', 'x', 'y'))
f.flush()
f.close()
for i in range(2):
# Open the file in append mode and add data
with netcdf_file('withRecordDimension.nc', 'a') as f:
f.variables['time'].data = np.append(f.variables["time"].data, i)
f.variables['testData'][i, :, :] = np.ones((dataSize, dataSize))*i
f.flush()
# Read the file and check that append worked
with netcdf_file('withRecordDimension.nc') as f:
assert_equal(f.variables['time'][-1], i)
assert_equal(f.variables['testData'][-1, :, :].copy(), np.ones((dataSize, dataSize))*i)
assert_equal(f.variables['time'].data.shape[0], i+1)
assert_equal(f.variables['testData'].data.shape[0], i+1)
# Read the file and check that 'data' was not saved as user defined
# attribute of testData variable during append operation
with netcdf_file('withRecordDimension.nc') as f:
with assert_raises(KeyError) as ar:
f.variables['testData']._attributes['data']
ex = ar.value
assert_equal(ex.args[0], 'data')
def test_maskandscale():
t = np.linspace(20, 30, 15)
t[3] = 100
tm = np.ma.masked_greater(t, 99)
fname = pjoin(TEST_DATA_PATH, 'example_2.nc')
with netcdf_file(fname, maskandscale=True) as f:
Temp = f.variables['Temperature']
assert_equal(Temp.missing_value, 9999)
assert_equal(Temp.add_offset, 20)
assert_equal(Temp.scale_factor, np.float32(0.01))
found = Temp[:].compressed()
del Temp # Remove ref to mmap, so file can be closed.
expected = np.round(tm.compressed(), 2)
assert_allclose(found, expected)
with in_tempdir():
newfname = 'ms.nc'
f = netcdf_file(newfname, 'w', maskandscale=True)
f.createDimension('Temperature', len(tm))
temp = f.createVariable('Temperature', 'i', ('Temperature',))
temp.missing_value = 9999
temp.scale_factor = 0.01
temp.add_offset = 20
temp[:] = tm
f.close()
with netcdf_file(newfname, maskandscale=True) as f:
Temp = f.variables['Temperature']
assert_equal(Temp.missing_value, 9999)
assert_equal(Temp.add_offset, 20)
assert_equal(Temp.scale_factor, np.float32(0.01))
expected = np.round(tm.compressed(), 2)
found = Temp[:].compressed()
del Temp
assert_allclose(found, expected)
# ------------------------------------------------------------------------
# Test reading with masked values (_FillValue / missing_value)
# ------------------------------------------------------------------------
def test_read_withValuesNearFillValue():
# Regression test for ticket #5626
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var1_fillval0'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_withNoFillValue():
# For a variable with no fill value, reading data with maskandscale=True
# should return unmasked data
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var2_noFillval'][:]
assert_mask_matches(vardata, [False, False, False])
assert_equal(vardata, [1,2,3])
def test_read_withFillValueAndMissingValue():
# For a variable with both _FillValue and missing_value, the _FillValue
# should be used
IRRELEVANT_VALUE = 9999
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var3_fillvalAndMissingValue'][:]
assert_mask_matches(vardata, [True, False, False])
assert_equal(vardata, [IRRELEVANT_VALUE, 2, 3])
def test_read_withMissingValue():
# For a variable with missing_value but not _FillValue, the missing_value
# should be used
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var4_missingValue'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_withFillValNaN():
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var5_fillvalNaN'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_withChar():
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var6_char'][:]
assert_mask_matches(vardata, [False, True, False])
def test_read_with2dVar():
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
with netcdf_file(fname, maskandscale=True) as f:
vardata = f.variables['var7_2d'][:]
assert_mask_matches(vardata, [[True, False], [False, False], [False, True]])
def test_read_withMaskAndScaleFalse():
# If a variable has a _FillValue (or missing_value) attribute, but is read
# with maskandscale set to False, the result should be unmasked
fname = pjoin(TEST_DATA_PATH, 'example_3_maskedvals.nc')
# Open file with mmap=False to avoid problems with closing a mmap'ed file
# when arrays referring to its data still exist:
with netcdf_file(fname, maskandscale=False, mmap=False) as f:
vardata = f.variables['var3_fillvalAndMissingValue'][:]
assert_mask_matches(vardata, [False, False, False])
assert_equal(vardata, [1, 2, 3])
utils/venv/lib/python3.6/site-packages/scipy/io/tests/test_paths.py
-88
View File
@@ -1,88 +0,0 @@
"""
Ensure that we can use pathlib.Path objects in all relevant IO functions.
"""
import sys
try:
from pathlib import Path
except ImportError:
# Not available. No fallback import, since we'll skip the entire
# test suite for Python < 3.6.
pass
import numpy as np
from numpy.testing import assert_
import pytest
import scipy.io
import scipy.io.wavfile
from scipy._lib._tmpdirs import tempdir
import scipy.sparse
@pytest.mark.skipif(sys.version_info < (3, 6),
reason='Passing path-like objects to IO functions requires Python >= 3.6')
class TestPaths(object):
data = np.arange(5).astype(np.int64)
def test_savemat(self):
with tempdir() as temp_dir:
path = Path(temp_dir) / 'data.mat'
scipy.io.savemat(path, {'data': self.data})
assert_(path.is_file())
def test_loadmat(self):
# Save data with string path, load with pathlib.Path
with tempdir() as temp_dir:
path = Path(temp_dir) / 'data.mat'
scipy.io.savemat(str(path), {'data': self.data})
mat_contents = scipy.io.loadmat(path)
assert_((mat_contents['data'] == self.data).all())
def test_whosmat(self):
# Save data with string path, load with pathlib.Path
with tempdir() as temp_dir:
path = Path(temp_dir) / 'data.mat'
scipy.io.savemat(str(path), {'data': self.data})
contents = scipy.io.whosmat(path)
assert_(contents[0] == ('data', (1, 5), 'int64'))
def test_readsav(self):
path = Path(__file__).parent / 'data/scalar_string.sav'
scipy.io.readsav(path)
def test_hb_read(self):
# Save data with string path, load with pathlib.Path
with tempdir() as temp_dir:
data = scipy.sparse.csr_matrix(scipy.sparse.eye(3))
path = Path(temp_dir) / 'data.hb'
scipy.io.harwell_boeing.hb_write(str(path), data)
data_new = scipy.io.harwell_boeing.hb_read(path)
assert_((data_new != data).nnz == 0)
def test_hb_write(self):
with tempdir() as temp_dir:
data = scipy.sparse.csr_matrix(scipy.sparse.eye(3))
path = Path(temp_dir) / 'data.hb'
scipy.io.harwell_boeing.hb_write(path, data)
assert_(path.is_file())
def test_netcdf_file(self):
path = Path(__file__).parent / 'data/example_1.nc'
scipy.io.netcdf.netcdf_file(path)
def test_wavfile_read(self):
path = Path(__file__).parent / 'data/test-8000Hz-le-2ch-1byteu.wav'
scipy.io.wavfile.read(path)
def test_wavfile_write(self):
# Read from str path, write to Path
input_path = Path(__file__).parent / 'data/test-8000Hz-le-2ch-1byteu.wav'
rate, data = scipy.io.wavfile.read(str(input_path))
with tempdir() as temp_dir:
output_path = Path(temp_dir) / input_path.name
scipy.io.wavfile.write(output_path, rate, data)
utils/venv/lib/python3.6/site-packages/scipy/io/tests/test_wavfile.py
-159
View File
@@ -1,159 +0,0 @@
from __future__ import division, print_function, absolute_import
import os
import sys
import tempfile
from io import BytesIO
import numpy as np
from numpy.testing import assert_equal, assert_, assert_array_equal
from pytest import raises as assert_raises
from scipy._lib._numpy_compat import suppress_warnings
from scipy.io import wavfile
def datafile(fn):
return os.path.join(os.path.dirname(__file__), 'data', fn)
def test_read_1():
for mmap in [False, True]:
rate, data = wavfile.read(datafile('test-44100Hz-le-1ch-4bytes.wav'),
mmap=mmap)
assert_equal(rate, 44100)
assert_(np.issubdtype(data.dtype, np.int32))
assert_equal(data.shape, (4410,))
del data
def test_read_2():
for mmap in [False, True]:
rate, data = wavfile.read(datafile('test-8000Hz-le-2ch-1byteu.wav'),
mmap=mmap)
assert_equal(rate, 8000)
assert_(np.issubdtype(data.dtype, np.uint8))
assert_equal(data.shape, (800, 2))
del data
def test_read_3():
for mmap in [False, True]:
rate, data = wavfile.read(datafile('test-44100Hz-2ch-32bit-float-le.wav'),
mmap=mmap)
assert_equal(rate, 44100)
assert_(np.issubdtype(data.dtype, np.float32))
assert_equal(data.shape, (441, 2))
del data
def test_read_4():
for mmap in [False, True]:
with suppress_warnings() as sup:
sup.filter(wavfile.WavFileWarning,
"Chunk .non-data. not understood, skipping it")
rate, data = wavfile.read(datafile('test-48000Hz-2ch-64bit-float-le-wavex.wav'),
mmap=mmap)
assert_equal(rate, 48000)
assert_(np.issubdtype(data.dtype, np.float64))
assert_equal(data.shape, (480, 2))
del data
def test_read_5():
for mmap in [False, True]:
rate, data = wavfile.read(datafile('test-44100Hz-2ch-32bit-float-be.wav'),
mmap=mmap)
assert_equal(rate, 44100)
assert_(np.issubdtype(data.dtype, np.float32))
assert_(data.dtype.byteorder == '>' or (sys.byteorder == 'big' and
data.dtype.byteorder == '='))
assert_equal(data.shape, (441, 2))
del data
def test_read_fail():
for mmap in [False, True]:
fp = open(datafile('example_1.nc'), 'rb')
assert_raises(ValueError, wavfile.read, fp, mmap=mmap)
fp.close()
def test_read_early_eof():
for mmap in [False, True]:
fp = open(datafile('test-44100Hz-le-1ch-4bytes-early-eof.wav'), 'rb')
assert_raises(ValueError, wavfile.read, fp, mmap=mmap)
fp.close()
def test_read_incomplete_chunk():
for mmap in [False, True]:
fp = open(datafile('test-44100Hz-le-1ch-4bytes-incomplete-chunk.wav'), 'rb')
assert_raises(ValueError, wavfile.read, fp, mmap=mmap)
fp.close()
def _check_roundtrip(realfile, rate, dtype, channels):
if realfile:
fd, tmpfile = tempfile.mkstemp(suffix='.wav')
os.close(fd)
else:
tmpfile = BytesIO()
try:
data = np.random.rand(100, channels)
if channels == 1:
data = data[:,0]
if dtype.kind == 'f':
# The range of the float type should be in [-1, 1]
data = data.astype(dtype)
else:
data = (data*128).astype(dtype)
wavfile.write(tmpfile, rate, data)
for mmap in [False, True]:
rate2, data2 = wavfile.read(tmpfile, mmap=mmap)
assert_equal(rate, rate2)
assert_(data2.dtype.byteorder in ('<', '=', '|'), msg=data2.dtype)
assert_array_equal(data, data2)
del data2
finally:
if realfile:
os.unlink(tmpfile)
def test_write_roundtrip():
for realfile in (False, True):
for dtypechar in ('i', 'u', 'f', 'g', 'q'):
for size in (1, 2, 4, 8):
if size == 1 and dtypechar == 'i':
# signed 8-bit integer PCM is not allowed
continue
if size > 1 and dtypechar == 'u':
# unsigned > 8-bit integer PCM is not allowed
continue
if (size == 1 or size == 2) and dtypechar == 'f':
# 8- or 16-bit float PCM is not expected
continue
if dtypechar in 'gq':
# no size allowed for these types
if size == 1:
size = ''
else:
continue
for endianness in ('>', '<'):
if size == 1 and endianness == '<':
continue
for rate in (8000, 32000):
for channels in (1, 2, 5):
dt = np.dtype('%s%s%s' % (endianness, dtypechar, size))
_check_roundtrip(realfile, rate, dt, channels)
utils/venv/lib/python3.6/site-packages/scipy/io/wavfile.py
-405
View File
@@ -1,405 +0,0 @@
"""
Module to read / write wav files using numpy arrays
Functions
---------
`read`: Return the sample rate (in samples/sec) and data from a WAV file.
`write`: Write a numpy array as a WAV file.
"""
from __future__ import division, print_function, absolute_import
import sys
import numpy
import struct
import warnings
__all__ = [
'WavFileWarning',
'read',
'write'
]
class WavFileWarning(UserWarning):
pass
WAVE_FORMAT_PCM = 0x0001
WAVE_FORMAT_IEEE_FLOAT = 0x0003
WAVE_FORMAT_EXTENSIBLE = 0xfffe
KNOWN_WAVE_FORMATS = (WAVE_FORMAT_PCM, WAVE_FORMAT_IEEE_FLOAT)
# assumes file pointer is immediately
# after the 'fmt ' id
def _read_fmt_chunk(fid, is_big_endian):
"""
Returns
-------
size : int
size of format subchunk in bytes (minus 8 for "fmt " and itself)
format_tag : int
PCM, float, or compressed format
channels : int
number of channels
fs : int
sampling frequency in samples per second
bytes_per_second : int
overall byte rate for the file
block_align : int
bytes per sample, including all channels
bit_depth : int
bits per sample
"""
if is_big_endian:
fmt = '>'
else:
fmt = '<'
size = res = struct.unpack(fmt+'I', fid.read(4))[0]
bytes_read = 0
if size < 16:
raise ValueError("Binary structure of wave file is not compliant")
res = struct.unpack(fmt+'HHIIHH', fid.read(16))
bytes_read += 16
format_tag, channels, fs, bytes_per_second, block_align, bit_depth = res
if format_tag == WAVE_FORMAT_EXTENSIBLE and size >= (16+2):
ext_chunk_size = struct.unpack(fmt+'H', fid.read(2))[0]
bytes_read += 2
if ext_chunk_size >= 22:
extensible_chunk_data = fid.read(22)
bytes_read += 22
raw_guid = extensible_chunk_data[2+4:2+4+16]
# GUID template {XXXXXXXX-0000-0010-8000-00AA00389B71} (RFC-2361)
# MS GUID byte order: first three groups are native byte order,
# rest is Big Endian
if is_big_endian:
tail = b'\x00\x00\x00\x10\x80\x00\x00\xAA\x00\x38\x9B\x71'
else:
tail = b'\x00\x00\x10\x00\x80\x00\x00\xAA\x00\x38\x9B\x71'
if raw_guid.endswith(tail):
format_tag = struct.unpack(fmt+'I', raw_guid[:4])[0]
else:
raise ValueError("Binary structure of wave file is not compliant")
if format_tag not in KNOWN_WAVE_FORMATS:
raise ValueError("Unknown wave file format")
# move file pointer to next chunk
if size > (bytes_read):
fid.read(size - bytes_read)
return (size, format_tag, channels, fs, bytes_per_second, block_align,
bit_depth)
# assumes file pointer is immediately after the 'data' id
def _read_data_chunk(fid, format_tag, channels, bit_depth, is_big_endian,
mmap=False):
if is_big_endian:
fmt = '>I'
else:
fmt = '<I'
# Size of the data subchunk in bytes
size = struct.unpack(fmt, fid.read(4))[0]
# Number of bytes per sample
bytes_per_sample = bit_depth//8
if bit_depth == 8:
dtype = 'u1'
else:
if is_big_endian:
dtype = '>'
else:
dtype = '<'
if format_tag == WAVE_FORMAT_PCM:
dtype += 'i%d' % bytes_per_sample
else:
dtype += 'f%d' % bytes_per_sample
if not mmap:
data = numpy.frombuffer(fid.read(size), dtype=dtype)
else:
start = fid.tell()
data = numpy.memmap(fid, dtype=dtype, mode='c', offset=start,
shape=(size//bytes_per_sample,))
fid.seek(start + size)
if channels > 1:
data = data.reshape(-1, channels)
return data
def _skip_unknown_chunk(fid, is_big_endian):
if is_big_endian:
fmt = '>I'
else:
fmt = '<I'
data = fid.read(4)
# call unpack() and seek() only if we have really read data from file
# otherwise empty read at the end of the file would trigger
# unnecessary exception at unpack() call
# in case data equals somehow to 0, there is no need for seek() anyway
if data:
size = struct.unpack(fmt, data)[0]
fid.seek(size, 1)
def _read_riff_chunk(fid):
str1 = fid.read(4) # File signature
if str1 == b'RIFF':
is_big_endian = False
fmt = '<I'
elif str1 == b'RIFX':
is_big_endian = True
fmt = '>I'
else:
# There are also .wav files with "FFIR" or "XFIR" signatures?
raise ValueError("File format {}... not "
"understood.".format(repr(str1)))
# Size of entire file
file_size = struct.unpack(fmt, fid.read(4))[0] + 8
str2 = fid.read(4)
if str2 != b'WAVE':
raise ValueError("Not a WAV file.")
return file_size, is_big_endian
def read(filename, mmap=False):
"""
Open a WAV file
Return the sample rate (in samples/sec) and data from a WAV file.
Parameters
----------
filename : string or open file handle
Input wav file.
mmap : bool, optional
Whether to read data as memory-mapped.
Only to be used on real files (Default: False).
.. versionadded:: 0.12.0
Returns
-------
rate : int
Sample rate of wav file.
data : numpy array
Data read from wav file. Data-type is determined from the file;
see Notes.
Notes
-----
This function cannot read wav files with 24-bit data.
Common data types: [1]_
===================== =========== =========== =============
WAV format Min Max NumPy dtype
===================== =========== =========== =============
32-bit floating-point -1.0 +1.0 float32
32-bit PCM -2147483648 +2147483647 int32
16-bit PCM -32768 +32767 int16
8-bit PCM 0 255 uint8
===================== =========== =========== =============
Note that 8-bit PCM is unsigned.
References
----------
.. [1] IBM Corporation and Microsoft Corporation, "Multimedia Programming
Interface and Data Specifications 1.0", section "Data Format of the
Samples", August 1991
http://www.tactilemedia.com/info/MCI_Control_Info.html
"""
if hasattr(filename, 'read'):
fid = filename
mmap = False
else:
fid = open(filename, 'rb')
try:
file_size, is_big_endian = _read_riff_chunk(fid)
fmt_chunk_received = False
channels = 1
bit_depth = 8
format_tag = WAVE_FORMAT_PCM
while fid.tell() < file_size:
# read the next chunk
chunk_id = fid.read(4)
if not chunk_id:
raise ValueError("Unexpected end of file.")
elif len(chunk_id) < 4:
raise ValueError("Incomplete wav chunk.")
if chunk_id == b'fmt ':
fmt_chunk_received = True
fmt_chunk = _read_fmt_chunk(fid, is_big_endian)
format_tag, channels, fs = fmt_chunk[1:4]
bit_depth = fmt_chunk[6]
if bit_depth not in (8, 16, 32, 64, 96, 128):
raise ValueError("Unsupported bit depth: the wav file "
"has {}-bit data.".format(bit_depth))
elif chunk_id == b'fact':
_skip_unknown_chunk(fid, is_big_endian)
elif chunk_id == b'data':
if not fmt_chunk_received:
raise ValueError("No fmt chunk before data")
data = _read_data_chunk(fid, format_tag, channels, bit_depth,
is_big_endian, mmap)
elif chunk_id == b'LIST':
# Someday this could be handled properly but for now skip it
_skip_unknown_chunk(fid, is_big_endian)
elif chunk_id in (b'JUNK', b'Fake'):
# Skip alignment chunks without warning
_skip_unknown_chunk(fid, is_big_endian)
else:
warnings.warn("Chunk (non-data) not understood, skipping it.",
WavFileWarning)
_skip_unknown_chunk(fid, is_big_endian)
finally:
if not hasattr(filename, 'read'):
fid.close()
else:
fid.seek(0)
return fs, data
def write(filename, rate, data):
"""
Write a numpy array as a WAV file.
Parameters
----------
filename : string or open file handle
Output wav file.
rate : int
The sample rate (in samples/sec).
data : ndarray
A 1-D or 2-D numpy array of either integer or float data-type.
Notes
-----
* Writes a simple uncompressed WAV file.
* To write multiple-channels, use a 2-D array of shape
(Nsamples, Nchannels).
* The bits-per-sample and PCM/float will be determined by the data-type.
Common data types: [1]_
===================== =========== =========== =============
WAV format Min Max NumPy dtype
===================== =========== =========== =============
32-bit floating-point -1.0 +1.0 float32
32-bit PCM -2147483648 +2147483647 int32
16-bit PCM -32768 +32767 int16
8-bit PCM 0 255 uint8
===================== =========== =========== =============
Note that 8-bit PCM is unsigned.
References
----------
.. [1] IBM Corporation and Microsoft Corporation, "Multimedia Programming
Interface and Data Specifications 1.0", section "Data Format of the
Samples", August 1991
http://www.tactilemedia.com/info/MCI_Control_Info.html
"""
if hasattr(filename, 'write'):
fid = filename
else:
fid = open(filename, 'wb')
fs = rate
try:
dkind = data.dtype.kind
if not (dkind == 'i' or dkind == 'f' or (dkind == 'u' and
data.dtype.itemsize == 1)):
raise ValueError("Unsupported data type '%s'" % data.dtype)
header_data = b''
header_data += b'RIFF'
header_data += b'\x00\x00\x00\x00'
header_data += b'WAVE'
# fmt chunk
header_data += b'fmt '
if dkind == 'f':
format_tag = WAVE_FORMAT_IEEE_FLOAT
else:
format_tag = WAVE_FORMAT_PCM
if data.ndim == 1:
channels = 1
else:
channels = data.shape[1]
bit_depth = data.dtype.itemsize * 8
bytes_per_second = fs*(bit_depth // 8)*channels
block_align = channels * (bit_depth // 8)
fmt_chunk_data = struct.pack('<HHIIHH', format_tag, channels, fs,
bytes_per_second, block_align, bit_depth)
if not (dkind == 'i' or dkind == 'u'):
# add cbSize field for non-PCM files
fmt_chunk_data += b'\x00\x00'
header_data += struct.pack('<I', len(fmt_chunk_data))
header_data += fmt_chunk_data
# fact chunk (non-PCM files)
if not (dkind == 'i' or dkind == 'u'):
header_data += b'fact'
header_data += struct.pack('<II', 4, data.shape[0])
# check data size (needs to be immediately before the data chunk)
if ((len(header_data)-4-4) + (4+4+data.nbytes)) > 0xFFFFFFFF:
raise ValueError("Data exceeds wave file size limit")
fid.write(header_data)
# data chunk
fid.write(b'data')
fid.write(struct.pack('<I', data.nbytes))
if data.dtype.byteorder == '>' or (data.dtype.byteorder == '=' and
sys.byteorder == 'big'):
data = data.byteswap()
_array_tofile(fid, data)
# Determine file size and place it in correct
# position at start of the file.
size = fid.tell()
fid.seek(4)
fid.write(struct.pack('<I', size-8))
finally:
if not hasattr(filename, 'write'):
fid.close()
else:
fid.seek(0)
if sys.version_info[0] >= 3:
def _array_tofile(fid, data):
# ravel gives a c-contiguous buffer
fid.write(data.ravel().view('b').data)
else:
def _array_tofile(fid, data):
fid.write(data.tostring())