Source code for silx.io.dictdump
# coding: utf-8
# /*##########################################################################
# Copyright (C) 2016 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ############################################################################*/
"""This module offers a set of functions to dump a python dictionary indexed
by text strings to following file formats: `HDF5, INI, JSON`
"""
from collections import OrderedDict
import json
import logging
import numpy
import os.path
import sys
try:
import h5py
except ImportError as e:
h5py_missing = True
h5py_import_error = e
else:
h5py_missing = False
from .configdict import ConfigDict
__authors__ = ["P. Knobel"]
__license__ = "MIT"
__date__ = "15/09/2016"
logger = logging.getLogger(__name__)
string_types = (basestring,) if sys.version_info[0] == 2 else (str,) # noqa
def _prepare_hdf5_dataset(array_like):
"""Cast a python object into a numpy array in a HDF5 friendly format.
:param array_like: Input dataset in a type that can be digested by
``numpy.array()`` (`str`, `list`, `numpy.ndarray`…)
:return: ``numpy.ndarray`` ready to be written as an HDF5 dataset
"""
# simple strings
if isinstance(array_like, string_types):
array_like = numpy.string_(array_like)
# Ensure our data is a numpy.ndarray
if not isinstance(array_like, (numpy.ndarray, numpy.string_)):
array = numpy.array(array_like)
else:
array = array_like
# handle list of strings or numpy array of strings
if not isinstance(array, numpy.string_):
data_kind = array.dtype.kind
# unicode: convert to byte strings
# (http://docs.h5py.org/en/latest/strings.html)
if data_kind.lower() in ["s", "u"]:
array = numpy.asarray(array, dtype=numpy.string_)
return array
[docs]def dicttoh5(treedict, h5file, h5path='/',
mode="w", overwrite_data=False,
create_dataset_args=None):
"""Write a nested dictionary to a HDF5 file, using keys as member names.
If a dictionary value is a sub-dictionary, a group is created. If it is
any other data type, it is cast into a numpy array and written as a
:mod:`h5py` dataset. Dictionary keys must be strings and cannot contain
the ``/`` character.
.. note::
This function requires `h5py <http://www.h5py.org/>`_ to be installed.
:param treedict: Nested dictionary/tree structure with strings as keys
and array-like objects as leafs. The ``"/"`` character is not allowed
in keys.
:param h5file: HDF5 file name or handle. If a file name is provided, the
function opens the file in the specified mode and closes it again
before completing.
:param h5path: Target path in HDF5 file in which scan groups are created.
Default is root (``"/"``)
:param mode: Can be ``"r+"`` (read/write, file must exist),
``"w"`` (write, existing file is lost), ``"w-"`` (write, fail if
exists) or ``"a"`` (read/write if exists, create otherwise).
This parameter is ignored if ``h5file`` is a file handle.
:param overwrite_data: If ``True``, existing groups and datasets can be
overwritten, if ``False`` they are skipped. This parameter is only
relevant if ``h5file_mode`` is ``"r+"`` or ``"a"``.
:param create_dataset_args: Dictionary of args you want to pass to
``h5f.create_dataset``. This allows you to specify filters and
compression parameters. Don't specify ``name`` and ``data``.
Example::
from silx.io.dictdump import dicttoh5
city_area = {
"Europe": {
"France": {
"Isère": {
"Grenoble": "18.44 km2"
},
"Nord": {
"Tourcoing": "15.19 km2"
},
},
},
}
create_ds_args = {'compression': "gzip",
'shuffle': True,
'fletcher32': True}
dicttoh5(city_area, "cities.h5", h5path="/area",
create_dataset_args=create_ds_args)
"""
if h5py_missing:
raise h5py_import_error
if not isinstance(h5file, h5py.File):
h5f = h5py.File(h5file, mode)
else:
h5f = h5file
if not h5path.endswith("/"):
h5path += "/"
for key in treedict:
if isinstance(treedict[key], dict) and len(treedict[key]):
# non-empty group: recurse
dicttoh5(treedict[key], h5f, h5path + key,
overwrite_data=overwrite_data,
create_dataset_args=create_dataset_args)
elif treedict[key] is None or (isinstance(treedict[key], dict) and
not len(treedict[key])):
# Create empty group
h5f.create_group(h5path + key)
else:
ds = _prepare_hdf5_dataset(treedict[key])
# can't apply filters on scalars (datasets with shape == () )
if ds.shape == () or create_dataset_args is None:
h5f.create_dataset(h5path + key,
data=ds)
else:
h5f.create_dataset(h5path + key,
data=ds,
**create_dataset_args)
if isinstance(h5file, string_types):
h5f.close()
[docs]def h5todict(h5file, path="/"):
"""Read HDF5 file and return a nested dictionary with the complete file
structure and all data.
.. note:: This function requires `h5py <http://www.h5py.org/>`_ to be
installed.
.. note:: If you write a dictionary to a HDF5 file with
:func:`dicttoh5` and then read it back with :func:`h5todict`, data
types are not preserved. All values are cast to numpy arrays before
being written to file, and they are read back as numpy arrays (or
scalars). In some cases, you may find that a list of heterogeneous
data types is converted to a numpy array of strings.
:param h5file: File name or :class:`h5py.File` object
:return: dict
"""
if h5py_missing:
raise h5py_import_error
if not isinstance(h5file, h5py.File):
h5f = h5py.File(h5file, "r")
else:
h5f = h5file
ddict = {}
for key in h5f[path]:
if isinstance(h5f[path + "/" + key], h5py.Group):
ddict[key] = h5todict(h5f, path + "/" + key)
else:
# Convert HDF5 dataset to numpy array
ddict[key] = h5f[path + "/" + key][...]
return ddict
[docs]def dicttojson(ddict, jsonfile, indent=None, mode="w"):
"""Serialize ``ddict`` as a JSON formatted stream to ``jsonfile``.
:param ddict: Dictionary (or any object compatible with ``json.dump``).
:param jsonfile: JSON file name or file-like object.
If a file name is provided, the function opens the file in the
specified mode and closes it again.
:param indent: If indent is a non-negative integer, then JSON array
elements and object members will be pretty-printed with that indent
level. An indent level of ``0`` will only insert newlines.
``None`` (the default) selects the most compact representation.
:param mode: File opening mode (``w``, ``a``, ``w+``…)
"""
if not hasattr(jsonfile, "write"):
jsonf = open(jsonfile, mode)
else:
jsonf = jsonfile
json.dump(ddict, jsonf, indent=indent)
if not hasattr(jsonfile, "write"):
jsonf.close()
[docs]def dicttoini(ddict, inifile, mode="w"):
"""Output dict as configuration file (similar to Microsoft Windows INI).
:param dict: Dictionary of configuration parameters
:param inifile: INI file name or file-like object.
If a file name is provided, the function opens the file in the
specified mode and closes it again.
:param mode: File opening mode (``w``, ``a``, ``w+``…)
"""
if not hasattr(inifile, "write"):
inif = open(inifile, mode)
else:
inif = inifile
ConfigDict(initdict=ddict).write(inif)
if not hasattr(inifile, "write"):
inif.close()
[docs]def dump(ddict, ffile, mode="w", fmat=None):
"""Dump dictionary to a file
:param ddict: Dictionary with string keys
:param ffile: File name or file-like object with a ``write`` method
:param str fmat: Output format: ``"json"``, ``"hdf5"`` or ``"ini"``.
When None (the default), it uses the filename extension as the format.
Dumping to a HDF5 file requires `h5py <http://www.h5py.org/>`_ to be
installed.
:param str mode: File opening mode (``w``, ``a``, ``w+``…)
Default is *"w"*, write mode, overwrite if exists.
:raises IOError: if file format is not supported
"""
if fmat is None:
# If file-like object get its name, else use ffile as filename
filename = getattr(ffile, 'name', ffile)
fmat = os.path.splitext(filename)[1][1:] # Strip extension leading '.'
fmat = fmat.lower()
if fmat == "json":
dicttojson(ddict, ffile, indent=2, mode=mode)
elif fmat in ["hdf5", "h5"]:
if h5py_missing:
logger.error("Cannot dump to HDF5 format, missing h5py library")
raise h5py_import_error
dicttoh5(ddict, ffile, mode=mode)
elif fmat in ["ini", "cfg"]:
dicttoini(ddict, ffile, mode=mode)
else:
raise IOError("Unknown format " + fmat)
[docs]def load(ffile, fmat=None):
"""Load dictionary from a file
When loading from a JSON or INI file, an OrderedDict is returned to
preserve the values' insertion order.
:param ffile: File name or file-like object with a ``read`` method
:param fmat: Input format: ``json``, ``hdf5`` or ``ini``.
When None (the default), it uses the filename extension as the format.
Loading from a HDF5 file requires `h5py <http://www.h5py.org/>`_ to be
installed.
:return: Dictionary (ordered dictionary for JSON and INI)
:raises IOError: if file format is not supported
"""
if not hasattr(ffile, "read"):
f = open(ffile, "r")
fname = ffile
else:
f = ffile
fname = ffile.name
if fmat is None: # Use file extension as format
fmat = os.path.splitext(fname)[1][1:] # Strip extension leading '.'
fmat = fmat.lower()
if fmat == "json":
return json.load(f, object_pairs_hook=OrderedDict)
if fmat in ["hdf5", "h5"]:
if h5py_missing:
logger.error("Cannot load from HDF5 format, missing h5py library")
raise h5py_import_error
return h5todict(fname)
elif fmat in ["ini", "cfg"]:
return ConfigDict(filelist=[fname])
else:
raise IOError("Unknown format " + fmat)
