dtkFileTools

Support for three formats of serialized population files: 1. "Original version": single payload chunk with simulation and all nodes, uncompressed or snappy or LZ4 2. "First chunked version": multiple payload chunks, one for simulation and one each for nodes 3. "Second chunked version": multiple payload chunks, simulation and node objects are "root" objects in each chunk 4. "Metadata update": compressed: true|false + engine: NONE|LZ4|SNAPPY replaced with compression: NONE|LZ4|SNAPPY 5. "Emod info added": emod_info added to header 6. "Per-node/human compression and chunk sizes": sim_compression, sim_chunk_size, node_compressions, node_chunk_sizes, human_compressions, human_node_suids, human_chunk_sizes added to header

DtkFileV6

Bases: object

The V6 file moves the humans out of the JSON serialized for the node and puts them into their own chunks. This helps to reduce the size of the JSON for one node and allows the memory for one collection of humans be freed before we get the next set. This greatly reduces the peak memory usage when processing populations that require lots of memory.

Source code in emod_api/serialization/dtkFileTools.py

class DtkFileV6(object):
    """
    The V6 file moves the humans out of the JSON serialized for the node and puts
    them into their own chunks.  This helps to reduce the size of the JSON for
    one node and allows the memory for one collection of humans be freed before
    we get the next set.  This greatly reduces the peak memory usage when processing
    populations that require lots of memory.
    """
    class Chunk(object):
        """
        Chunk represnts a compressed chunk of data in a V6 serialized population file.
        In the code, _json and _chunk are mutually exclusive - only one is populated at a time.

        Args:
            filename (str): The name of the file being read (for error messages).
            obj_type_str (str): The type of object in the chunk (for error messages).
            v6_compression_str (str): The V6 compression string for the chunk.
            node_suid (int): The SUID of the node the chunk belongs to.
            chunk_size (int): The size of the chunk in bytes.
            chunk (bytes): The compressed chunk data.
        """
        def __init__(self,
                     filename,
                     obj_type_str,
                     v6_compression_str,
                     node_suid,
                     chunk_size,
                     chunk):
            if chunk is None and chunk_size != 0:
                msg = f"Chunk is None but chunk size is {chunk_size} for {obj_type_str} chunk of file '{filename}'"
                raise UserWarning(msg)
            elif (chunk is not None) and (len(chunk) != chunk_size):
                msg = f"Only read {len(chunk)} bytes of {chunk_size} for {obj_type_str} chunk of file '{filename}'"
                raise UserWarning(msg)

            self._v6_compression_str = v6_compression_str
            self._node_suid = node_suid
            self._chunk_size = chunk_size
            self._chunk = chunk
            self._json = None
            return

        def get_json(self):
            """
            Return the JSON dictionary for the chunk, uncompressing and parsing it if necessary.
            """
            if self._json is None:
                old_compression_type = _compression_type_v6_to_old(self._v6_compression_str)
                uncomp_data = str(uncompress(self._chunk, old_compression_type), 'utf-8')
                try:
                    json_data = json.loads(uncomp_data, object_hook=support.SerialObject)
                except Exception:
                    raise UserWarning("Could not parse JSON in chunk with size {0}".format(self._chunk_size))
                self._json = json_data
                self._chunk = None
                self._chunk_size = 0
                gc.collect()
            return self._json

        def set_json(self, json_data):
            """
            Replace the existing JSON with the provided JSON dictionary.
            Also compresses and stores the chunk.
            """
            self._json = json_data
            self.store()
            return

        def store(self):
            """
            Compress and store the JSON dictionary as a chunk.
            """
            if self._chunk is None:
                json_data = json.dumps(self._json, separators=(',', ':'))
                self._v6_compression_str = _determine_v6_compression_type(json_data)
                old_compression_type = _compression_type_v6_to_old(self._v6_compression_str)
                self._chunk = compress(json_data.encode(), old_compression_type)
                self._chunk_size = len(self._chunk)
                self._json = None
                gc.collect()
            return

        @property
        def v6_compression_str(self):
            """
            Return the V6 compression string for the chunk - NON, LZ4, SNA.
            """
            return self._v6_compression_str

        @property
        def node_suid(self):
            """
            Return the SUID of the node the chunk belongs to - not the external ID.
            """
            return self._node_suid

        @property
        def chunk_size(self):
            """
            Return the size of the chunk in bytes.
            """
            return self._chunk_size

        @property
        def chunk(self):
            """
            Return the compressed chunk data.
            """
            return self._chunk

    class HumanCollectionChunkV6(Chunk):
        """
        This represents one collection of humans in a node.

        Args:
            filename (str): The name of the file being read (for error messages).
            obj_type_str (str): The type of object in the chunk (for error messages).
            v6_compression_str (str): The V6 compression string for the chunk.
            node_suid (int): The SUID of the node the chunk belongs to.
            num_humans (int): The number of humans in the collection.
            chunk_size (int): The size of the chunk in bytes.
            chunk (bytes): The compressed chunk data.
        """
        def __init__(self,
                     filename,
                     obj_type_str,
                     v6_compression_str,
                     node_suid,
                     num_humans,
                     chunk_size,
                     chunk):
            super(DtkFileV6.HumanCollectionChunkV6, self).__init__(filename,
                                                                   obj_type_str,
                                                                   v6_compression_str,
                                                                   node_suid,
                                                                   chunk_size,
                                                                   chunk)
            self._num_humans = num_humans
            return

        def get_json(self):
            """
            Return an list of JSON IndividualHuman dictionaries.
            """
            json_data = super(DtkFileV6.HumanCollectionChunkV6, self).get_json()
            return json_data['human_collection']

        def set_json(self, human_list):
            """
            Replace the existing JSON with the provided list of IndividualHuman dictionaries.
            """
            self._json = {}
            self._json['human_collection'] = human_list
            self._num_humans = len(human_list)
            self.store()
            return

        @property
        def num_humans(self):
            """
            Return the number of humans in the collection.
            """
            return self._num_humans

    class NodeV6(MutableMapping):
        """
        NodeV6 represents one node in a V6 serialized population file.
        The purpose of this class is to delay loading the full JSON for the node
        until it is actually needed and to provide a backwards compatible interface
        to the human data via the individualHumans property.

        Implementation notes:
            - This class inherits from MutableMapping to provide dictionary-like
            access to the node JSON data.  The individualHumans property is
            handled specially to return the human list.
            - __get_item__, __setitem__, __delitem__ each handle the key 'individualHumans'
              specially to return or raise an error as appropriate.  This helps to provide
              a backwards compatible interface to the human data.

        Args:
            parent (DtkFileV6): The parent DtkFileV6 object.
            node_chunk (DtkFileV6.Chunk): The chunk containing the node data.
            human_chunk_list (list of DtkFileV6.HumanCollectionChunkV6):
                The list of chunks containing the human data for the node.
        """
        def __init__(self, parent, node_chunk, human_chunk_list):
            super(DtkFileV6.NodeV6, self).__init__()
            self.__parent__ = parent
            self._node_chunk = node_chunk
            self._human_list = DtkFileV6.HumanListV6(self, human_chunk_list)
            self._json = None
            return

        def __getitem__(self, key):
            """
            Return the value for the given key in the node JSON dictionary.
            If the key is 'individualHumans', return the human list instead.
            """
            if key == 'individualHumans':
                return self._human_list
            else:
                self.load()
                return self._json[key]

        def __setitem__(self, key, value):
            """
            Set the value for the given key in the node JSON dictionary.
            Cannot set the 'individualHumans' key directly.
            """
            self.load()
            if key == 'individualHumans':
                self.individualHumans = value
            else:
                self._json[key] = value

        def __delitem__(self, key):
            """
            Delete the given key from the node JSON dictionary.
            Cannot delete the 'individualHumans' key.
            """
            if key == 'individualHumans':
                raise RuntimeError("Cannot set individualHumans property directly")
            self.load()
            del self._json[key]

        def __iter__(self):
            self.load()
            return iter(self._json)

        def __len__(self):
            self.load()
            return len(self._json)

        def __repr__(self):
            self.load()
            return repr(self._json)

        def keys(self):
            """
            Return the keys of the node JSON dictionary as a list.
            """
            self.load()
            return list(super(DtkFileV6.NodeV6, self).keys())

        def load(self):
            """
            Load the node JSON dictionary from the chunk if it is not already loaded.
            """
            if self._json is None:
                keys = list(self.__dict__.keys())
                values = list(self.__dict__.values())
                tmp_json = self._node_chunk.get_json()
                self.__dict__ = tmp_json
                for key, value in zip(keys, values):
                    self.__dict__[key] = value
                self._json = tmp_json
                self._node_chunk._chunk = None
                self._node_chunk._chunk_size = 0
                gc.collect()
            return

        def store(self):
            """
            Store the node JSON dictionary back to the chunk if it is loaded.

            Implementation note:
                We need to temporarily remove references to the member variables of
                this class from the _json/__dict__ before storing it back to the chunk.
                This keeps us from compressing the wrong stuff.  We add them back afterwards.
            """
            if self._json is not None:
                # save member variables
                parent = self.__parent__
                node_chunk = self._node_chunk
                human_list = self._human_list
                tmp_json = self._json

                # remove member variables from json
                keys_to_remove = ['__parent__', '_node_chunk', '_human_list', '_json']
                for key in keys_to_remove:
                    del tmp_json[key]

                # compress json
                node_chunk.set_json(tmp_json)

                # restore member variables
                self.__parent__ = parent
                self._node_chunk = node_chunk
                self._human_list = human_list

                # clear json to free memory
                self._json = None
                gc.collect()
            return

        def _clear_human_list(self):
            """
            Clear the human list for the node.
            """
            self.__parent__._remove_humans_for_node(self._node_chunk.node_suid)
            self._human_list = DtkFileV6.HumanListV6(node=self, human_chunk_list=[])
            return

        @property
        def individualHumans(self):
            """
            Return a list of IndividualHuman dictionaries for the node.
            """
            return self._human_list

        @individualHumans.setter
        def individualHumans(self, json_dict_list):
            self._clear_human_list()
            human_chunk = DtkFileV6.HumanCollectionChunkV6(
                filename="no file",
                obj_type_str="human",
                v6_compression_str=None,
                node_suid=self._node_chunk.node_suid,
                num_humans=0,
                chunk_size=0,
                chunk=None)
            human_chunk.set_json(json_dict_list)
            self.__parent__._human_chunks.append(human_chunk)
            self._human_list._add_human_chunk(human_chunk)
            return

    class NodeListV6(object):
        """
        The NodeListV6 provides an interface to a list of NodeV6 objects.
        The main purpose of this class is to manage loading and unloading
        the node data when iterating over the nodes.
        """
        def __init__(self, parent):
            self.__parent__ = parent
            self._node_list = []
            return

        def __iter__(self):
            index = 0
            while index < len(self):
                self._node_list[index].load()
                yield self.__getitem__(index)
                self._node_list[index].store()
                index += 1

        def __getitem__(self, index):
            node = self._node_list[index]
            node.load()
            return node

        def __setitem__(self, index, node):
            self._node_list[index] = node
            return

        def __len__(self):
            length = len(self._node_list)
            return length

        def append(self, node_chunk):
            self._node_list.append(node_chunk)
            return

    class HumanListV6(object):
        """
        A HumanListV6 provides an interface to a list of IndividualHuman dictionaries
        that may be stored in multiple HumanCollectionChunkV6 chunks.  The purpose of
        this class is to manage loading and unloading the human collection chunks
        when iterating over the humans.  It hides the fact that the humans for one node
        may be stored in multiple collections.
        """
        def __init__(self, node, human_chunk_list):
            self._node = node
            self._human_chunk_list = human_chunk_list
            self._num_humans = 0
            for human_chunk in self._human_chunk_list:
                self._num_humans += human_chunk.num_humans
            self._human_chunk_index = 0
            self._current_collection = None
            self._current_min_index = 0
            self._current_max_index = 0
            self.__init_current()
            return

        def __init_current(self):
            """
            Initialize the current human collection chunk.
            """
            if (len(self._human_chunk_list) > 0) and (self._current_collection is None):
                self._current_collection = self._human_chunk_list[0].get_json()
                self._current_min_index = 0
                self._current_max_index = len(self._current_collection) - 1
                if len(self._current_collection) != self._human_chunk_list[0].num_humans:
                    msg = f"Number of humans in first human chunk [{len(self._current_collection)}]"
                    msg += f" does not match num_humans attribute [{self._human_chunk_list[0].num_humans}]"
                    raise RuntimeError(msg)
            return

        def _add_human_chunk(self, human_chunk):
            """
            Add a new human collection chunk to the list.
            """
            self._human_chunk_list.append(human_chunk)
            self._num_humans += human_chunk.num_humans
            self.__init_current()
            return

        def __iter__(self):
            human_index = 0
            self.__update_current_collection__(human_index)
            while human_index < len(self):
                yield self.__getitem__(human_index)
                human_index += 1

        def __update_current_collection__(self, human_index):
            """
            Update/load the current human collection chunk to include the specified human index.
            0-based human_index is the index of the human in the full list of humans for the node.
            0-based _current_min_index and _current_max_index are the min and max indices of the
            currently loaded human collection chunk and are inclusive.
            """
            if self._num_humans == 0:
                return

            if human_index < self._current_min_index:
                while human_index < self._current_min_index:
                    self._human_chunk_list[self._human_chunk_index].store()
                    self._human_chunk_index -= 1
                    if self._human_chunk_index < 0:
                        raise IndexError("Index {0} is out of range for human collection".format(human_index))
                    self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
                    self._current_max_index = self._current_min_index - 1
                    self._current_min_index = self._current_max_index - len(self._current_collection) + 1
                    if len(self._current_collection) != self._human_chunk_list[self._human_chunk_index].num_humans:
                        raise RuntimeError("Number of humans in first human chunk does not match num_humans attribute")
            else:
                while human_index > self._current_max_index:
                    self._human_chunk_list[self._human_chunk_index].store()
                    self._human_chunk_index += 1
                    if self._human_chunk_index >= len(self._human_chunk_list):
                        raise IndexError("Index {0} is out of range for human collection".format(human_index))
                    self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
                    self._current_min_index = self._current_max_index + 1
                    self._current_max_index = self._current_min_index + len(self._current_collection) - 1
                    if len(self._current_collection) != self._human_chunk_list[self._human_chunk_index].num_humans:
                        raise RuntimeError(f"current collection = {len(self._current_collection)} but num_humans = {self._human_chunk_list[self._human_chunk_index].num_humans}")
            return

        def __getitem__(self, human_index):
            """
            Return the IndividualHuman dictionary at the specified index.
            """
            if human_index < self._current_min_index or human_index > self._current_max_index:
                self.__update_current_collection__(human_index)
            return self._current_collection[human_index - self._current_min_index]

        def __setitem__(self, human_index, value):
            """
            Set the IndividualHuman dictionary at the specified index.
            """
            if human_index < self._current_min_index or human_index > self._current_max_index:
                self.__update_current_collection__(human_index)
            self._current_collection[human_index - self._current_min_index] = value
            return

        def __len__(self):
            return self._num_humans

        def append(self, human_dict):
            if self._human_chunk_index != (len(self._human_chunk_list) - 1):
                self._human_chunk_list[self._human_chunk_index].store()
                self._human_chunk_index = len(self._human_chunk_list) - 1
                self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
                self._current_min_index = self._num_humans - len(self._current_collection)
                self._current_max_index = self._num_humans - 1
            self._current_collection.append(human_dict)
            self._current_max_index += 1
            self._num_humans += 1
            self._human_chunk_list[self._human_chunk_index]._num_humans += 1

    def __init__(self, header=None, filename='', handle=None):
        """
        Initialize a DtkFileV6 object from the provided header and file handle.
        This should read the file and create chunk objects for the simulation, nodes,
        and humans.  It will not uncompress or parse any of the JSON data until it is needed.

        Args:
            header (DtkHeaderV6): The header for the file.
            filename (str): The name of the file being read (for error messages).
            handle (file-like object): The file handle to read the data from.
        """
        if header is None:
            header = DtkHeaderV6()
        self.__header__ = header
        self._sim_chunk = None
        self._node_chunks = []
        self._human_chunks = []
        self._nodes = DtkFileV6.NodeListV6(self)

        if handle is not None:
            sim_chunk_size = int(header.sim_chunk_size, 16)
            sim_chunk_data = handle.read(sim_chunk_size)
            self._sim_chunk = DtkFileV6.Chunk(filename,
                                              "sim",
                                              header.sim_compression,
                                              -1,
                                              sim_chunk_size,
                                              sim_chunk_data)

            for index, size_string in enumerate(header.node_chunk_sizes):
                v6_compression_str = header.node_compressions[index]
                node_suid = int(header.node_suids[index], 16)
                chunk_size = int(size_string, 16)
                chunk_data = handle.read(chunk_size)
                node_chunk = DtkFileV6.Chunk(filename,
                                             "node",
                                             v6_compression_str,
                                             node_suid,
                                             chunk_size,
                                             chunk_data)
                self._node_chunks.append(node_chunk)

            for index, size_string in enumerate(header.human_chunk_sizes):
                v6_compression_str = header.human_compressions[index]
                node_suid_str = header.human_node_suids[index]
                num_humans_str = header.human_num_humans[index]
                node_suid = int(node_suid_str, 16)
                num_humans = int(num_humans_str, 16)
                chunk_size = int(size_string, 16)
                chunk_data = handle.read(chunk_size)
                human_chunk = DtkFileV6.HumanCollectionChunkV6(filename,
                                                               "human",
                                                               v6_compression_str,
                                                               node_suid,
                                                               num_humans,
                                                               chunk_size,
                                                               chunk_data)
                self._human_chunks.append(human_chunk)

            for node_chunk in self._node_chunks:
                human_chunk_list = []
                for human_chunk in self._human_chunks:
                    if human_chunk.node_suid == node_chunk.node_suid:
                        human_chunk_list.append(human_chunk)
                self._nodes.append(DtkFileV6.NodeV6(self, node_chunk, human_chunk_list))

        return

    def _remove_humans_for_node(self, node_suid):
        """
        Remove all human chunks for the specified node SUID.
        """
        new_human_chunks = []
        for human_chunk in self._human_chunks:
            if human_chunk.node_suid != node_suid:
                new_human_chunks.append(human_chunk)
        self._human_chunks = new_human_chunks
        return

    @property
    def header(self):
        return self.__header__

    # Optional header entries
    @property
    def author(self):
        return self.__header__.author if 'author' in self.__header__ else ''

    @author.setter
    def author(self, value):
        self.__header__['author'] = str(value)
        return

    @property
    def date(self):
        return self.__header__.date if 'date' in self.__header__ else ''

    @date.setter
    def date(self, value):
        self.__header__['date'] = str(value)

    @property
    def tool(self):
        return self.__header__.tool if 'tool' in self.__header__ else ''

    @tool.setter
    def tool(self, value):
        self.__header__['tool'] = str(value)
        return

    @property
    def version(self):
        return self.__header__.version

    @property
    def nodes(self):
        """
        Return the list of NodeV6 objects in the file.
        Do not try to access the nodes via the simulation property of this class.
        This keeps it backwards compatible.
        """
        return self._nodes

    def _sync_header(self):
        self._sim_chunk.store()
        for node in self.nodes:
            node.store()
        for human_chunk in self._human_chunks:
            human_chunk.store()

        self.__header__['date'] = time.strftime('%a %b %d %H:%M:%S %Y')
        self.__header__['sim_compression'] = self._sim_chunk.v6_compression_str
        self.__header__['sim_chunk_size'] = format(self._sim_chunk.chunk_size, '016x')
        self.__header__['node_compressions'] = []
        self.__header__['node_chunk_sizes'] = []
        self.__header__['node_suids'] = []
        for node_chunk in self._node_chunks:
            self.__header__['node_compressions'].append(node_chunk.v6_compression_str)
            self.__header__['node_chunk_sizes'].append(format(node_chunk.chunk_size, '016x'))
            self.__header__['node_suids'].append(format(node_chunk.node_suid, '016x'))
        self.__header__['human_compressions'] = []
        self.__header__['human_chunk_sizes'] = []
        self.__header__['human_node_suids'] = []
        self.__header__['human_num_humans'] = []
        for human_chunk in self._human_chunks:
            self.__header__['human_compressions'].append(human_chunk.v6_compression_str)
            self.__header__['human_chunk_sizes'].append(format(human_chunk.chunk_size, '016x'))
            self.__header__['human_node_suids'].append(format(human_chunk.node_suid, '016x'))
            self.__header__['human_num_humans'].append(format(human_chunk.num_humans, '016x'))
        return

    @property
    def simulation(self):
        """
        Return the simulation JSON dictionary.  Do not try to access the nodes
        from this dictionary - use the nodes property of this class instead.
        """
        return self._sim_chunk.get_json()

    @simulation.setter
    def simulation(self, value):
        value["nodes"] = []
        self._sim_chunk.set_json(value)
        return

nodes property

Return the list of NodeV6 objects in the file. Do not try to access the nodes via the simulation property of this class. This keeps it backwards compatible.

simulation property writable

Return the simulation JSON dictionary. Do not try to access the nodes from this dictionary - use the nodes property of this class instead.

Chunk

Bases: object

Chunk represnts a compressed chunk of data in a V6 serialized population file. In the code, _json and _chunk are mutually exclusive - only one is populated at a time.

Parameters:

Name	Type	Description	Default
filename	str	The name of the file being read (for error messages).	required
obj_type_str	str	The type of object in the chunk (for error messages).	required
v6_compression_str	str	The V6 compression string for the chunk.	required
node_suid	int	The SUID of the node the chunk belongs to.	required
chunk_size	int	The size of the chunk in bytes.	required
chunk	bytes	The compressed chunk data.	required

Source code in emod_api/serialization/dtkFileTools.py

class Chunk(object):
    """
    Chunk represnts a compressed chunk of data in a V6 serialized population file.
    In the code, _json and _chunk are mutually exclusive - only one is populated at a time.

    Args:
        filename (str): The name of the file being read (for error messages).
        obj_type_str (str): The type of object in the chunk (for error messages).
        v6_compression_str (str): The V6 compression string for the chunk.
        node_suid (int): The SUID of the node the chunk belongs to.
        chunk_size (int): The size of the chunk in bytes.
        chunk (bytes): The compressed chunk data.
    """
    def __init__(self,
                 filename,
                 obj_type_str,
                 v6_compression_str,
                 node_suid,
                 chunk_size,
                 chunk):
        if chunk is None and chunk_size != 0:
            msg = f"Chunk is None but chunk size is {chunk_size} for {obj_type_str} chunk of file '{filename}'"
            raise UserWarning(msg)
        elif (chunk is not None) and (len(chunk) != chunk_size):
            msg = f"Only read {len(chunk)} bytes of {chunk_size} for {obj_type_str} chunk of file '{filename}'"
            raise UserWarning(msg)

        self._v6_compression_str = v6_compression_str
        self._node_suid = node_suid
        self._chunk_size = chunk_size
        self._chunk = chunk
        self._json = None
        return

    def get_json(self):
        """
        Return the JSON dictionary for the chunk, uncompressing and parsing it if necessary.
        """
        if self._json is None:
            old_compression_type = _compression_type_v6_to_old(self._v6_compression_str)
            uncomp_data = str(uncompress(self._chunk, old_compression_type), 'utf-8')
            try:
                json_data = json.loads(uncomp_data, object_hook=support.SerialObject)
            except Exception:
                raise UserWarning("Could not parse JSON in chunk with size {0}".format(self._chunk_size))
            self._json = json_data
            self._chunk = None
            self._chunk_size = 0
            gc.collect()
        return self._json

    def set_json(self, json_data):
        """
        Replace the existing JSON with the provided JSON dictionary.
        Also compresses and stores the chunk.
        """
        self._json = json_data
        self.store()
        return

    def store(self):
        """
        Compress and store the JSON dictionary as a chunk.
        """
        if self._chunk is None:
            json_data = json.dumps(self._json, separators=(',', ':'))
            self._v6_compression_str = _determine_v6_compression_type(json_data)
            old_compression_type = _compression_type_v6_to_old(self._v6_compression_str)
            self._chunk = compress(json_data.encode(), old_compression_type)
            self._chunk_size = len(self._chunk)
            self._json = None
            gc.collect()
        return

    @property
    def v6_compression_str(self):
        """
        Return the V6 compression string for the chunk - NON, LZ4, SNA.
        """
        return self._v6_compression_str

    @property
    def node_suid(self):
        """
        Return the SUID of the node the chunk belongs to - not the external ID.
        """
        return self._node_suid

    @property
    def chunk_size(self):
        """
        Return the size of the chunk in bytes.
        """
        return self._chunk_size

    @property
    def chunk(self):
        """
        Return the compressed chunk data.
        """
        return self._chunk

chunk property

Return the compressed chunk data.

chunk_size property

Return the size of the chunk in bytes.

node_suid property

Return the SUID of the node the chunk belongs to - not the external ID.

v6_compression_str property

Return the V6 compression string for the chunk - NON, LZ4, SNA.

get_json()

Return the JSON dictionary for the chunk, uncompressing and parsing it if necessary.

Source code in emod_api/serialization/dtkFileTools.py

def get_json(self):
    """
    Return the JSON dictionary for the chunk, uncompressing and parsing it if necessary.
    """
    if self._json is None:
        old_compression_type = _compression_type_v6_to_old(self._v6_compression_str)
        uncomp_data = str(uncompress(self._chunk, old_compression_type), 'utf-8')
        try:
            json_data = json.loads(uncomp_data, object_hook=support.SerialObject)
        except Exception:
            raise UserWarning("Could not parse JSON in chunk with size {0}".format(self._chunk_size))
        self._json = json_data
        self._chunk = None
        self._chunk_size = 0
        gc.collect()
    return self._json

set_json(json_data)

Replace the existing JSON with the provided JSON dictionary. Also compresses and stores the chunk.

Source code in emod_api/serialization/dtkFileTools.py

def set_json(self, json_data):
    """
    Replace the existing JSON with the provided JSON dictionary.
    Also compresses and stores the chunk.
    """
    self._json = json_data
    self.store()
    return

store()

Compress and store the JSON dictionary as a chunk.

Source code in emod_api/serialization/dtkFileTools.py

def store(self):
    """
    Compress and store the JSON dictionary as a chunk.
    """
    if self._chunk is None:
        json_data = json.dumps(self._json, separators=(',', ':'))
        self._v6_compression_str = _determine_v6_compression_type(json_data)
        old_compression_type = _compression_type_v6_to_old(self._v6_compression_str)
        self._chunk = compress(json_data.encode(), old_compression_type)
        self._chunk_size = len(self._chunk)
        self._json = None
        gc.collect()
    return

HumanCollectionChunkV6

Bases: Chunk

This represents one collection of humans in a node.

Parameters:

Name	Type	Description	Default
filename	str	The name of the file being read (for error messages).	required
obj_type_str	str	The type of object in the chunk (for error messages).	required
v6_compression_str	str	The V6 compression string for the chunk.	required
node_suid	int	The SUID of the node the chunk belongs to.	required
num_humans	int	The number of humans in the collection.	required
chunk_size	int	The size of the chunk in bytes.	required
chunk	bytes	The compressed chunk data.	required

Source code in emod_api/serialization/dtkFileTools.py

class HumanCollectionChunkV6(Chunk):
    """
    This represents one collection of humans in a node.

    Args:
        filename (str): The name of the file being read (for error messages).
        obj_type_str (str): The type of object in the chunk (for error messages).
        v6_compression_str (str): The V6 compression string for the chunk.
        node_suid (int): The SUID of the node the chunk belongs to.
        num_humans (int): The number of humans in the collection.
        chunk_size (int): The size of the chunk in bytes.
        chunk (bytes): The compressed chunk data.
    """
    def __init__(self,
                 filename,
                 obj_type_str,
                 v6_compression_str,
                 node_suid,
                 num_humans,
                 chunk_size,
                 chunk):
        super(DtkFileV6.HumanCollectionChunkV6, self).__init__(filename,
                                                               obj_type_str,
                                                               v6_compression_str,
                                                               node_suid,
                                                               chunk_size,
                                                               chunk)
        self._num_humans = num_humans
        return

    def get_json(self):
        """
        Return an list of JSON IndividualHuman dictionaries.
        """
        json_data = super(DtkFileV6.HumanCollectionChunkV6, self).get_json()
        return json_data['human_collection']

    def set_json(self, human_list):
        """
        Replace the existing JSON with the provided list of IndividualHuman dictionaries.
        """
        self._json = {}
        self._json['human_collection'] = human_list
        self._num_humans = len(human_list)
        self.store()
        return

    @property
    def num_humans(self):
        """
        Return the number of humans in the collection.
        """
        return self._num_humans

num_humans property

Return the number of humans in the collection.

get_json()

Return an list of JSON IndividualHuman dictionaries.

Source code in emod_api/serialization/dtkFileTools.py

def get_json(self):
    """
    Return an list of JSON IndividualHuman dictionaries.
    """
    json_data = super(DtkFileV6.HumanCollectionChunkV6, self).get_json()
    return json_data['human_collection']

set_json(human_list)

Replace the existing JSON with the provided list of IndividualHuman dictionaries.

Source code in emod_api/serialization/dtkFileTools.py

def set_json(self, human_list):
    """
    Replace the existing JSON with the provided list of IndividualHuman dictionaries.
    """
    self._json = {}
    self._json['human_collection'] = human_list
    self._num_humans = len(human_list)
    self.store()
    return

HumanListV6

Bases: object

A HumanListV6 provides an interface to a list of IndividualHuman dictionaries that may be stored in multiple HumanCollectionChunkV6 chunks. The purpose of this class is to manage loading and unloading the human collection chunks when iterating over the humans. It hides the fact that the humans for one node may be stored in multiple collections.

Source code in emod_api/serialization/dtkFileTools.py

class HumanListV6(object):
    """
    A HumanListV6 provides an interface to a list of IndividualHuman dictionaries
    that may be stored in multiple HumanCollectionChunkV6 chunks.  The purpose of
    this class is to manage loading and unloading the human collection chunks
    when iterating over the humans.  It hides the fact that the humans for one node
    may be stored in multiple collections.
    """
    def __init__(self, node, human_chunk_list):
        self._node = node
        self._human_chunk_list = human_chunk_list
        self._num_humans = 0
        for human_chunk in self._human_chunk_list:
            self._num_humans += human_chunk.num_humans
        self._human_chunk_index = 0
        self._current_collection = None
        self._current_min_index = 0
        self._current_max_index = 0
        self.__init_current()
        return

    def __init_current(self):
        """
        Initialize the current human collection chunk.
        """
        if (len(self._human_chunk_list) > 0) and (self._current_collection is None):
            self._current_collection = self._human_chunk_list[0].get_json()
            self._current_min_index = 0
            self._current_max_index = len(self._current_collection) - 1
            if len(self._current_collection) != self._human_chunk_list[0].num_humans:
                msg = f"Number of humans in first human chunk [{len(self._current_collection)}]"
                msg += f" does not match num_humans attribute [{self._human_chunk_list[0].num_humans}]"
                raise RuntimeError(msg)
        return

    def _add_human_chunk(self, human_chunk):
        """
        Add a new human collection chunk to the list.
        """
        self._human_chunk_list.append(human_chunk)
        self._num_humans += human_chunk.num_humans
        self.__init_current()
        return

    def __iter__(self):
        human_index = 0
        self.__update_current_collection__(human_index)
        while human_index < len(self):
            yield self.__getitem__(human_index)
            human_index += 1

    def __update_current_collection__(self, human_index):
        """
        Update/load the current human collection chunk to include the specified human index.
        0-based human_index is the index of the human in the full list of humans for the node.
        0-based _current_min_index and _current_max_index are the min and max indices of the
        currently loaded human collection chunk and are inclusive.
        """
        if self._num_humans == 0:
            return

        if human_index < self._current_min_index:
            while human_index < self._current_min_index:
                self._human_chunk_list[self._human_chunk_index].store()
                self._human_chunk_index -= 1
                if self._human_chunk_index < 0:
                    raise IndexError("Index {0} is out of range for human collection".format(human_index))
                self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
                self._current_max_index = self._current_min_index - 1
                self._current_min_index = self._current_max_index - len(self._current_collection) + 1
                if len(self._current_collection) != self._human_chunk_list[self._human_chunk_index].num_humans:
                    raise RuntimeError("Number of humans in first human chunk does not match num_humans attribute")
        else:
            while human_index > self._current_max_index:
                self._human_chunk_list[self._human_chunk_index].store()
                self._human_chunk_index += 1
                if self._human_chunk_index >= len(self._human_chunk_list):
                    raise IndexError("Index {0} is out of range for human collection".format(human_index))
                self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
                self._current_min_index = self._current_max_index + 1
                self._current_max_index = self._current_min_index + len(self._current_collection) - 1
                if len(self._current_collection) != self._human_chunk_list[self._human_chunk_index].num_humans:
                    raise RuntimeError(f"current collection = {len(self._current_collection)} but num_humans = {self._human_chunk_list[self._human_chunk_index].num_humans}")
        return

    def __getitem__(self, human_index):
        """
        Return the IndividualHuman dictionary at the specified index.
        """
        if human_index < self._current_min_index or human_index > self._current_max_index:
            self.__update_current_collection__(human_index)
        return self._current_collection[human_index - self._current_min_index]

    def __setitem__(self, human_index, value):
        """
        Set the IndividualHuman dictionary at the specified index.
        """
        if human_index < self._current_min_index or human_index > self._current_max_index:
            self.__update_current_collection__(human_index)
        self._current_collection[human_index - self._current_min_index] = value
        return

    def __len__(self):
        return self._num_humans

    def append(self, human_dict):
        if self._human_chunk_index != (len(self._human_chunk_list) - 1):
            self._human_chunk_list[self._human_chunk_index].store()
            self._human_chunk_index = len(self._human_chunk_list) - 1
            self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
            self._current_min_index = self._num_humans - len(self._current_collection)
            self._current_max_index = self._num_humans - 1
        self._current_collection.append(human_dict)
        self._current_max_index += 1
        self._num_humans += 1
        self._human_chunk_list[self._human_chunk_index]._num_humans += 1

__getitem__(human_index)

Return the IndividualHuman dictionary at the specified index.

Source code in emod_api/serialization/dtkFileTools.py

def __getitem__(self, human_index):
    """
    Return the IndividualHuman dictionary at the specified index.
    """
    if human_index < self._current_min_index or human_index > self._current_max_index:
        self.__update_current_collection__(human_index)
    return self._current_collection[human_index - self._current_min_index]

__init_current()

Initialize the current human collection chunk.

Source code in emod_api/serialization/dtkFileTools.py

def __init_current(self):
    """
    Initialize the current human collection chunk.
    """
    if (len(self._human_chunk_list) > 0) and (self._current_collection is None):
        self._current_collection = self._human_chunk_list[0].get_json()
        self._current_min_index = 0
        self._current_max_index = len(self._current_collection) - 1
        if len(self._current_collection) != self._human_chunk_list[0].num_humans:
            msg = f"Number of humans in first human chunk [{len(self._current_collection)}]"
            msg += f" does not match num_humans attribute [{self._human_chunk_list[0].num_humans}]"
            raise RuntimeError(msg)
    return

__setitem__(human_index, value)

Set the IndividualHuman dictionary at the specified index.

Source code in emod_api/serialization/dtkFileTools.py

def __setitem__(self, human_index, value):
    """
    Set the IndividualHuman dictionary at the specified index.
    """
    if human_index < self._current_min_index or human_index > self._current_max_index:
        self.__update_current_collection__(human_index)
    self._current_collection[human_index - self._current_min_index] = value
    return

__update_current_collection__(human_index)

Update/load the current human collection chunk to include the specified human index. 0-based human_index is the index of the human in the full list of humans for the node. 0-based _current_min_index and _current_max_index are the min and max indices of the currently loaded human collection chunk and are inclusive.

Source code in emod_api/serialization/dtkFileTools.py

def __update_current_collection__(self, human_index):
    """
    Update/load the current human collection chunk to include the specified human index.
    0-based human_index is the index of the human in the full list of humans for the node.
    0-based _current_min_index and _current_max_index are the min and max indices of the
    currently loaded human collection chunk and are inclusive.
    """
    if self._num_humans == 0:
        return

    if human_index < self._current_min_index:
        while human_index < self._current_min_index:
            self._human_chunk_list[self._human_chunk_index].store()
            self._human_chunk_index -= 1
            if self._human_chunk_index < 0:
                raise IndexError("Index {0} is out of range for human collection".format(human_index))
            self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
            self._current_max_index = self._current_min_index - 1
            self._current_min_index = self._current_max_index - len(self._current_collection) + 1
            if len(self._current_collection) != self._human_chunk_list[self._human_chunk_index].num_humans:
                raise RuntimeError("Number of humans in first human chunk does not match num_humans attribute")
    else:
        while human_index > self._current_max_index:
            self._human_chunk_list[self._human_chunk_index].store()
            self._human_chunk_index += 1
            if self._human_chunk_index >= len(self._human_chunk_list):
                raise IndexError("Index {0} is out of range for human collection".format(human_index))
            self._current_collection = self._human_chunk_list[self._human_chunk_index].get_json()
            self._current_min_index = self._current_max_index + 1
            self._current_max_index = self._current_min_index + len(self._current_collection) - 1
            if len(self._current_collection) != self._human_chunk_list[self._human_chunk_index].num_humans:
                raise RuntimeError(f"current collection = {len(self._current_collection)} but num_humans = {self._human_chunk_list[self._human_chunk_index].num_humans}")
    return

NodeListV6

Bases: object

The NodeListV6 provides an interface to a list of NodeV6 objects. The main purpose of this class is to manage loading and unloading the node data when iterating over the nodes.

Source code in emod_api/serialization/dtkFileTools.py

class NodeListV6(object):
    """
    The NodeListV6 provides an interface to a list of NodeV6 objects.
    The main purpose of this class is to manage loading and unloading
    the node data when iterating over the nodes.
    """
    def __init__(self, parent):
        self.__parent__ = parent
        self._node_list = []
        return

    def __iter__(self):
        index = 0
        while index < len(self):
            self._node_list[index].load()
            yield self.__getitem__(index)
            self._node_list[index].store()
            index += 1

    def __getitem__(self, index):
        node = self._node_list[index]
        node.load()
        return node

    def __setitem__(self, index, node):
        self._node_list[index] = node
        return

    def __len__(self):
        length = len(self._node_list)
        return length

    def append(self, node_chunk):
        self._node_list.append(node_chunk)
        return

NodeV6

Bases: MutableMapping

NodeV6 represents one node in a V6 serialized population file. The purpose of this class is to delay loading the full JSON for the node until it is actually needed and to provide a backwards compatible interface to the human data via the individualHumans property.

Implementation notes

• This class inherits from MutableMapping to provide dictionary-like access to the node JSON data. The individualHumans property is handled specially to return the human list.
• get_item, setitem, delitem each handle the key 'individualHumans' specially to return or raise an error as appropriate. This helps to provide a backwards compatible interface to the human data.

Parameters:

Name	Type	Description	Default
parent	DtkFileV6	The parent DtkFileV6 object.	required
node_chunk	Chunk	The chunk containing the node data.	required
human_chunk_list	list of DtkFileV6.HumanCollectionChunkV6	The list of chunks containing the human data for the node.	required

Source code in emod_api/serialization/dtkFileTools.py

class NodeV6(MutableMapping):
    """
    NodeV6 represents one node in a V6 serialized population file.
    The purpose of this class is to delay loading the full JSON for the node
    until it is actually needed and to provide a backwards compatible interface
    to the human data via the individualHumans property.

    Implementation notes:
        - This class inherits from MutableMapping to provide dictionary-like
        access to the node JSON data.  The individualHumans property is
        handled specially to return the human list.
        - __get_item__, __setitem__, __delitem__ each handle the key 'individualHumans'
          specially to return or raise an error as appropriate.  This helps to provide
          a backwards compatible interface to the human data.

    Args:
        parent (DtkFileV6): The parent DtkFileV6 object.
        node_chunk (DtkFileV6.Chunk): The chunk containing the node data.
        human_chunk_list (list of DtkFileV6.HumanCollectionChunkV6):
            The list of chunks containing the human data for the node.
    """
    def __init__(self, parent, node_chunk, human_chunk_list):
        super(DtkFileV6.NodeV6, self).__init__()
        self.__parent__ = parent
        self._node_chunk = node_chunk
        self._human_list = DtkFileV6.HumanListV6(self, human_chunk_list)
        self._json = None
        return

    def __getitem__(self, key):
        """
        Return the value for the given key in the node JSON dictionary.
        If the key is 'individualHumans', return the human list instead.
        """
        if key == 'individualHumans':
            return self._human_list
        else:
            self.load()
            return self._json[key]

    def __setitem__(self, key, value):
        """
        Set the value for the given key in the node JSON dictionary.
        Cannot set the 'individualHumans' key directly.
        """
        self.load()
        if key == 'individualHumans':
            self.individualHumans = value
        else:
            self._json[key] = value

    def __delitem__(self, key):
        """
        Delete the given key from the node JSON dictionary.
        Cannot delete the 'individualHumans' key.
        """
        if key == 'individualHumans':
            raise RuntimeError("Cannot set individualHumans property directly")
        self.load()
        del self._json[key]

    def __iter__(self):
        self.load()
        return iter(self._json)

    def __len__(self):
        self.load()
        return len(self._json)

    def __repr__(self):
        self.load()
        return repr(self._json)

    def keys(self):
        """
        Return the keys of the node JSON dictionary as a list.
        """
        self.load()
        return list(super(DtkFileV6.NodeV6, self).keys())

    def load(self):
        """
        Load the node JSON dictionary from the chunk if it is not already loaded.
        """
        if self._json is None:
            keys = list(self.__dict__.keys())
            values = list(self.__dict__.values())
            tmp_json = self._node_chunk.get_json()
            self.__dict__ = tmp_json
            for key, value in zip(keys, values):
                self.__dict__[key] = value
            self._json = tmp_json
            self._node_chunk._chunk = None
            self._node_chunk._chunk_size = 0
            gc.collect()
        return

    def store(self):
        """
        Store the node JSON dictionary back to the chunk if it is loaded.

        Implementation note:
            We need to temporarily remove references to the member variables of
            this class from the _json/__dict__ before storing it back to the chunk.
            This keeps us from compressing the wrong stuff.  We add them back afterwards.
        """
        if self._json is not None:
            # save member variables
            parent = self.__parent__
            node_chunk = self._node_chunk
            human_list = self._human_list
            tmp_json = self._json

            # remove member variables from json
            keys_to_remove = ['__parent__', '_node_chunk', '_human_list', '_json']
            for key in keys_to_remove:
                del tmp_json[key]

            # compress json
            node_chunk.set_json(tmp_json)

            # restore member variables
            self.__parent__ = parent
            self._node_chunk = node_chunk
            self._human_list = human_list

            # clear json to free memory
            self._json = None
            gc.collect()
        return

    def _clear_human_list(self):
        """
        Clear the human list for the node.
        """
        self.__parent__._remove_humans_for_node(self._node_chunk.node_suid)
        self._human_list = DtkFileV6.HumanListV6(node=self, human_chunk_list=[])
        return

    @property
    def individualHumans(self):
        """
        Return a list of IndividualHuman dictionaries for the node.
        """
        return self._human_list

    @individualHumans.setter
    def individualHumans(self, json_dict_list):
        self._clear_human_list()
        human_chunk = DtkFileV6.HumanCollectionChunkV6(
            filename="no file",
            obj_type_str="human",
            v6_compression_str=None,
            node_suid=self._node_chunk.node_suid,
            num_humans=0,
            chunk_size=0,
            chunk=None)
        human_chunk.set_json(json_dict_list)
        self.__parent__._human_chunks.append(human_chunk)
        self._human_list._add_human_chunk(human_chunk)
        return

individualHumans property writable

Return a list of IndividualHuman dictionaries for the node.

__delitem__(key)

Delete the given key from the node JSON dictionary. Cannot delete the 'individualHumans' key.

Source code in emod_api/serialization/dtkFileTools.py

def __delitem__(self, key):
    """
    Delete the given key from the node JSON dictionary.
    Cannot delete the 'individualHumans' key.
    """
    if key == 'individualHumans':
        raise RuntimeError("Cannot set individualHumans property directly")
    self.load()
    del self._json[key]

__getitem__(key)

Return the value for the given key in the node JSON dictionary. If the key is 'individualHumans', return the human list instead.

Source code in emod_api/serialization/dtkFileTools.py

def __getitem__(self, key):
    """
    Return the value for the given key in the node JSON dictionary.
    If the key is 'individualHumans', return the human list instead.
    """
    if key == 'individualHumans':
        return self._human_list
    else:
        self.load()
        return self._json[key]

__setitem__(key, value)

Set the value for the given key in the node JSON dictionary. Cannot set the 'individualHumans' key directly.

Source code in emod_api/serialization/dtkFileTools.py

def __setitem__(self, key, value):
    """
    Set the value for the given key in the node JSON dictionary.
    Cannot set the 'individualHumans' key directly.
    """
    self.load()
    if key == 'individualHumans':
        self.individualHumans = value
    else:
        self._json[key] = value

keys()

Return the keys of the node JSON dictionary as a list.

Source code in emod_api/serialization/dtkFileTools.py

def keys(self):
    """
    Return the keys of the node JSON dictionary as a list.
    """
    self.load()
    return list(super(DtkFileV6.NodeV6, self).keys())

load()

Load the node JSON dictionary from the chunk if it is not already loaded.

Source code in emod_api/serialization/dtkFileTools.py

def load(self):
    """
    Load the node JSON dictionary from the chunk if it is not already loaded.
    """
    if self._json is None:
        keys = list(self.__dict__.keys())
        values = list(self.__dict__.values())
        tmp_json = self._node_chunk.get_json()
        self.__dict__ = tmp_json
        for key, value in zip(keys, values):
            self.__dict__[key] = value
        self._json = tmp_json
        self._node_chunk._chunk = None
        self._node_chunk._chunk_size = 0
        gc.collect()
    return

store()

Store the node JSON dictionary back to the chunk if it is loaded.

Implementation note

We need to temporarily remove references to the member variables of this class from the _json/dict before storing it back to the chunk. This keeps us from compressing the wrong stuff. We add them back afterwards.

Source code in emod_api/serialization/dtkFileTools.py

def store(self):
    """
    Store the node JSON dictionary back to the chunk if it is loaded.

    Implementation note:
        We need to temporarily remove references to the member variables of
        this class from the _json/__dict__ before storing it back to the chunk.
        This keeps us from compressing the wrong stuff.  We add them back afterwards.
    """
    if self._json is not None:
        # save member variables
        parent = self.__parent__
        node_chunk = self._node_chunk
        human_list = self._human_list
        tmp_json = self._json

        # remove member variables from json
        keys_to_remove = ['__parent__', '_node_chunk', '_human_list', '_json']
        for key in keys_to_remove:
            del tmp_json[key]

        # compress json
        node_chunk.set_json(tmp_json)

        # restore member variables
        self.__parent__ = parent
        self._node_chunk = node_chunk
        self._human_list = human_list

        # clear json to free memory
        self._json = None
        gc.collect()
    return

__init__(header=None, filename='', handle=None)

Initialize a DtkFileV6 object from the provided header and file handle. This should read the file and create chunk objects for the simulation, nodes, and humans. It will not uncompress or parse any of the JSON data until it is needed.

Parameters:

Name	Type	Description	Default
header	DtkHeaderV6	The header for the file.	None
filename	str	The name of the file being read (for error messages).	''
handle	file-like object	The file handle to read the data from.	None

Source code in emod_api/serialization/dtkFileTools.py

def __init__(self, header=None, filename='', handle=None):
    """
    Initialize a DtkFileV6 object from the provided header and file handle.
    This should read the file and create chunk objects for the simulation, nodes,
    and humans.  It will not uncompress or parse any of the JSON data until it is needed.

    Args:
        header (DtkHeaderV6): The header for the file.
        filename (str): The name of the file being read (for error messages).
        handle (file-like object): The file handle to read the data from.
    """
    if header is None:
        header = DtkHeaderV6()
    self.__header__ = header
    self._sim_chunk = None
    self._node_chunks = []
    self._human_chunks = []
    self._nodes = DtkFileV6.NodeListV6(self)

    if handle is not None:
        sim_chunk_size = int(header.sim_chunk_size, 16)
        sim_chunk_data = handle.read(sim_chunk_size)
        self._sim_chunk = DtkFileV6.Chunk(filename,
                                          "sim",
                                          header.sim_compression,
                                          -1,
                                          sim_chunk_size,
                                          sim_chunk_data)

        for index, size_string in enumerate(header.node_chunk_sizes):
            v6_compression_str = header.node_compressions[index]
            node_suid = int(header.node_suids[index], 16)
            chunk_size = int(size_string, 16)
            chunk_data = handle.read(chunk_size)
            node_chunk = DtkFileV6.Chunk(filename,
                                         "node",
                                         v6_compression_str,
                                         node_suid,
                                         chunk_size,
                                         chunk_data)
            self._node_chunks.append(node_chunk)

        for index, size_string in enumerate(header.human_chunk_sizes):
            v6_compression_str = header.human_compressions[index]
            node_suid_str = header.human_node_suids[index]
            num_humans_str = header.human_num_humans[index]
            node_suid = int(node_suid_str, 16)
            num_humans = int(num_humans_str, 16)
            chunk_size = int(size_string, 16)
            chunk_data = handle.read(chunk_size)
            human_chunk = DtkFileV6.HumanCollectionChunkV6(filename,
                                                           "human",
                                                           v6_compression_str,
                                                           node_suid,
                                                           num_humans,
                                                           chunk_size,
                                                           chunk_data)
            self._human_chunks.append(human_chunk)

        for node_chunk in self._node_chunks:
            human_chunk_list = []
            for human_chunk in self._human_chunks:
                if human_chunk.node_suid == node_chunk.node_suid:
                    human_chunk_list.append(human_chunk)
            self._nodes.append(DtkFileV6.NodeV6(self, node_chunk, human_chunk_list))

    return

DtkHeaderV6

Bases: SerialObject

The header for V6 is quite different because we distinguish the different types of chunks - sim, node, human collection. It also specifies the compression type for each type of chunk separately.

The 'human_num_humans' was added to the format to support this python code. It helps the code to know how many humans are in a particular chunk so that we can hide that the humans for one node are actually in different collections.

Source code in emod_api/serialization/dtkFileTools.py

class DtkHeaderV6(support.SerialObject):
    """
    The header for V6 is quite different because we distinguish the different types
    of chunks - sim, node, human collection.  It also specifies the compression type
    for each type of chunk separately.

    The 'human_num_humans' was added to the format to support this python code.
    It helps the code to know how many humans are in a particular chunk so that
    we can hide that the humans for one node are actually in different collections.
    """
    def __init__(self, dictionary=None):
        if dictionary is None:
            dictionary = {
                "version": 6,
                "author": "IDM",
                "tool": "DTK",
                "date": time.strftime('%a %b %d %H:%M:%S %Y'),
                "emod_info": {
                    'emod_major_version': 0,
                    'emod_minor_version': 0,
                    'emod_revision_number': 0,
                    'ser_pop_major_version': 0,
                    'ser_pop_minor_version': 0,
                    'ser_pop_patch_version': 0,
                    'emod_build_date': "Mon Jan 1 00:00:00 1970",
                    'emod_builder_name': "",
                    'emod_sccs_branch': 0,
                    'emod_sccs_date': "Mon Jan 1 00:00:00 1970"
                },
                "sim_compression": "NON",
                "sim_chunk_size": "0000000000000000",
                "node_suids": [],
                "node_compressions": [],
                "node_chunk_sizes": [],
                "human_compressions": [],
                "human_node_suids": [],
                "human_num_humans": [],
                "human_chunk_sizes": []
            }
        super(DtkHeaderV6, self).__init__(dictionary)
        return

    def __str__(self):
        text = json.dumps(self, separators=(',', ':'))
        return text

    def __len__(self):
        length = len(self.__str__())
        return length