onionshare/lib/stem/descriptor/networkstatus.py

# Copyright 2012-2013, Damian Johnson and The Tor Project
# See LICENSE for licensing information

"""
Parsing for Tor network status documents. This supports both the v2 and v3
dir-spec. Documents can be obtained from a few sources...

* the 'cached-consensus' file in tor's data directory
* tor metrics, at https://metrics.torproject.org/data.html
* directory authorities and mirrors via their DirPort

... and contain the following sections...

* document header
* list of :class:`stem.descriptor.networkstatus.DirectoryAuthority`
* list of :class:`stem.descriptor.router_status_entry.RouterStatusEntry`
* document footer

Of these, the router status entry section can be quite large (on the order of
hundreds of kilobytes). As such we provide a couple of methods for reading
network status documents through :func:`~stem.descriptor.__init__.parse_file`.
For more information see :func:`~stem.descriptor.__init__.DocumentHandler`...

::

  from stem.descriptor import parse_file, DocumentHandler

  with open('.tor/cached-consensus', 'rb') as consensus_file:
    # Processes the routers as we read them in. The routers refer to a document
    # with an unset 'routers' attribute.

    for router in parse_file(consensus_file, 'network-status-consensus-3 1.0', document_handler = DocumentHandler.ENTRIES):
      print router.nickname

**Module Overview:**

::

  NetworkStatusDocument - Network status document
    |- NetworkStatusDocumentV2 - Version 2 network status document
    |- NetworkStatusDocumentV3 - Version 3 network status document
    +- BridgeNetworkStatusDocument - Version 3 network status document for bridges

  KeyCertificate - Certificate used to authenticate an authority
  DocumentSignature - Signature of a document by a directory authority
  DirectoryAuthority - Directory authority as defined in a v3 network status document
"""

import datetime
import io

import stem.descriptor.router_status_entry
import stem.util.str_tools
import stem.util.tor_tools
import stem.version

from stem.descriptor import (
  PGP_BLOCK_END,
  Descriptor,
  DocumentHandler,
  _get_descriptor_components,
  _read_until_keywords,
)

# Version 2 network status document fields, tuples of the form...
# (keyword, is_mandatory)

NETWORK_STATUS_V2_FIELDS = (
  ("network-status-version", True),
  ("dir-source", True),
  ("fingerprint", True),
  ("contact", True),
  ("dir-signing-key", True),
  ("client-versions", False),
  ("server-versions", False),
  ("published", True),
  ("dir-options", False),
  ("directory-signature", True),
)

# Network status document are either a 'vote' or 'consensus', with different
# mandatory fields for each. Both though require that their fields appear in a
# specific order. This is an ordered listing of the following...
#
# (field, in_votes, in_consensus, is_mandatory)

HEADER_STATUS_DOCUMENT_FIELDS = (
  ("network-status-version", True, True, True),
  ("vote-status", True, True, True),
  ("consensus-methods", True, False, False),
  ("consensus-method", False, True, False),
  ("published", True, False, True),
  ("valid-after", True, True, True),
  ("fresh-until", True, True, True),
  ("valid-until", True, True, True),
  ("voting-delay", True, True, True),
  ("client-versions", True, True, False),
  ("server-versions", True, True, False),
  ("known-flags", True, True, True),
  ("flag-thresholds", True, False, False),
  ("params", True, True, False),
)

FOOTER_STATUS_DOCUMENT_FIELDS = (
  ("directory-footer", True, True, False),
  ("bandwidth-weights", False, True, False),
  ("directory-signature", True, True, True),
)

HEADER_FIELDS = [attr[0] for attr in HEADER_STATUS_DOCUMENT_FIELDS]
FOOTER_FIELDS = [attr[0] for attr in FOOTER_STATUS_DOCUMENT_FIELDS]

AUTH_START = "dir-source"
ROUTERS_START = "r"
FOOTER_START = "directory-footer"
V2_FOOTER_START = "directory-signature"

DEFAULT_PARAMS = {
  "bwweightscale": 10000,
  "cbtdisabled": 0,
  "cbtnummodes": 3,
  "cbtrecentcount": 20,
  "cbtmaxtimeouts": 18,
  "cbtmincircs": 100,
  "cbtquantile": 80,
  "cbtclosequantile": 95,
  "cbttestfreq": 60,
  "cbtmintimeout": 2000,
  "cbtinitialtimeout": 60000,
  "Support022HiddenServices": 1,
}

# KeyCertificate fields, tuple is of the form...
# (keyword, is_mandatory)

KEY_CERTIFICATE_PARAMS = (
  ('dir-key-certificate-version', True),
  ('dir-address', False),
  ('fingerprint', True),
  ('dir-identity-key', True),
  ('dir-key-published', True),
  ('dir-key-expires', True),
  ('dir-signing-key', True),
  ('dir-key-crosscert', False),
  ('dir-key-certification', True),
)


def _parse_file(document_file, document_type = None, validate = True, is_microdescriptor = False, document_handler = DocumentHandler.ENTRIES, **kwargs):
  """
  Parses a network status and iterates over the RouterStatusEntry in it. The
  document that these instances reference have an empty 'routers' attribute to
  allow for limited memory usage.

  :param file document_file: file with network status document content
  :param class document_type: NetworkStatusDocument subclass
  :param bool validate: checks the validity of the document's contents if
    **True**, skips these checks otherwise
  :param bool is_microdescriptor: **True** if this is for a microdescriptor
    consensus, **False** otherwise
  :param stem.descriptor.__init__.DocumentHandler document_handler: method in
    which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument`
  :param dict kwargs: additional arguments for the descriptor constructor

  :returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object

  :raises:
    * **ValueError** if the document_version is unrecognized or the contents is
      malformed and validate is **True**
    * **IOError** if the file can't be read
  """

  # we can't properly default this since NetworkStatusDocumentV3 isn't defined yet

  if document_type is None:
    document_type = NetworkStatusDocumentV3

  if document_type == NetworkStatusDocumentV2:
    document_type = NetworkStatusDocumentV2
    router_type = stem.descriptor.router_status_entry.RouterStatusEntryV2
  elif document_type == NetworkStatusDocumentV3:
    if not is_microdescriptor:
      router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3
    else:
      router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3
  elif document_type == BridgeNetworkStatusDocument:
    document_type = BridgeNetworkStatusDocument
    router_type = stem.descriptor.router_status_entry.RouterStatusEntryV2
  else:
    raise ValueError("Document type %i isn't recognized (only able to parse v2, v3, and bridge)" % document_type)

  if document_handler == DocumentHandler.DOCUMENT:
    yield document_type(document_file.read(), validate, **kwargs)
    return

  # getting the document without the routers section

  header = _read_until_keywords((ROUTERS_START, FOOTER_START, V2_FOOTER_START), document_file)

  routers_start = document_file.tell()
  _read_until_keywords((FOOTER_START, V2_FOOTER_START), document_file, skip = True)
  routers_end = document_file.tell()

  footer = document_file.readlines()
  document_content = bytes.join(b"", header + footer)

  if document_handler == DocumentHandler.BARE_DOCUMENT:
    yield document_type(document_content, validate, **kwargs)
  elif document_handler == DocumentHandler.ENTRIES:
    desc_iterator = stem.descriptor.router_status_entry._parse_file(
      document_file,
      validate,
      entry_class = router_type,
      entry_keyword = ROUTERS_START,
      start_position = routers_start,
      end_position = routers_end,
      extra_args = (document_type(document_content, validate),),
      **kwargs
    )

    for desc in desc_iterator:
      yield desc
  else:
    raise ValueError("Unrecognized document_handler: %s" % document_handler)


def _parse_file_key_certs(certificate_file, validate = True):
  """
  Parses a file containing one or more authority key certificates.

  :param file certificate_file: file with key certificates
  :param bool validate: checks the validity of the certificate's contents if
    **True**, skips these checks otherwise

  :returns: iterator for :class:`stem.descriptor.networkstatus.KeyCertificate`
    instance in the file

  :raises:
    * **ValueError** if the key certificate content is invalid and validate is
      **True**
    * **IOError** if the file can't be read
  """

  while True:
    keycert_content = _read_until_keywords("dir-key-certification", certificate_file)

    # we've reached the 'router-signature', now include the pgp style block
    block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0]
    keycert_content += _read_until_keywords(block_end_prefix, certificate_file, True)

    if keycert_content:
      yield stem.descriptor.networkstatus.KeyCertificate(bytes.join(b"", keycert_content), validate = validate)
    else:
      break  # done parsing file


class NetworkStatusDocument(Descriptor):
  """
  Common parent for network status documents.
  """

  def __init__(self, raw_content):
    super(NetworkStatusDocument, self).__init__(raw_content)
    self._unrecognized_lines = []

  def get_unrecognized_lines(self):
    return list(self._unrecognized_lines)


class NetworkStatusDocumentV2(NetworkStatusDocument):
  """
  Version 2 network status document. These have been deprecated and are no
  longer generated by Tor.

  :var dict routers: fingerprints to :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV2`
    contained in the document

  :var int version: **\*** document version

  :var str hostname: **\*** hostname of the authority
  :var str address: **\*** authority's IP address
  :var int dir_port: **\*** authority's DirPort
  :var str fingerprint: **\*** authority's fingerprint
  :var str contact: **\*** authority's contact information
  :var str signing_key: **\*** authority's public signing key

  :var list client_versions: list of recommended client tor version strings
  :var list server_versions: list of recommended server tor version strings
  :var datetime published: **\*** time when the document was published
  :var list options: **\*** list of things that this authority decides

  :var str signing_authority: **\*** name of the authority signing the document
  :var str signature: **\*** authority's signature for the document

  **\*** attribute is either required when we're parsed with validation or has
  a default value, others are left as **None** if undefined
  """

  def __init__(self, raw_content, validate = True):
    super(NetworkStatusDocumentV2, self).__init__(raw_content)

    self.version = None
    self.hostname = None
    self.address = None
    self.dir_port = None
    self.fingerprint = None
    self.contact = None
    self.signing_key = None

    self.client_versions = []
    self.server_versions = []
    self.published = None
    self.options = []

    self.signing_authority = None
    self.signatures = None

    # Splitting the document from the routers. Unlike v3 documents we're not
    # bending over backwards on the validation by checking the field order or
    # that header/footer attributes aren't in the wrong section. This is a
    # deprecated descriptor type - patches welcome if you want those checks.

    document_file = io.BytesIO(raw_content)
    document_content = bytes.join(b"", _read_until_keywords((ROUTERS_START, V2_FOOTER_START), document_file))

    router_iter = stem.descriptor.router_status_entry._parse_file(
      document_file,
      validate,
      entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV2,
      entry_keyword = ROUTERS_START,
      section_end_keywords = (V2_FOOTER_START,),
      extra_args = (self,),
    )

    self.routers = dict((desc.fingerprint, desc) for desc in router_iter)

    document_content += b"\n" + document_file.read()
    document_content = stem.util.str_tools._to_unicode(document_content)

    entries = _get_descriptor_components(document_content, validate)

    if validate:
      self._check_constraints(entries)

    self._parse(entries, validate)

  def _parse(self, entries, validate):
    for keyword, values in entries.items():
      value, block_contents = values[0]

      line = "%s %s" % (keyword, value)  # original line

      if block_contents:
        line += "\n%s" % block_contents

      if keyword == "network-status-version":
        if not value.isdigit():
          if not validate:
            continue

          raise ValueError("Network status document has a non-numeric version: %s" % line)

        self.version = int(value)

        if validate and self.version != 2:
          raise ValueError("Expected a version 2 network status document, got version '%s' instead" % self.version)
      elif keyword == "dir-source":
        dir_source_comp = value.split()

        if len(dir_source_comp) < 3:
          if not validate:
            continue

          raise ValueError("The 'dir-source' line of a v2 network status document must have three values: %s" % line)

        if validate:
          if not dir_source_comp[0]:
            # https://trac.torproject.org/7055
            raise ValueError("Authority's hostname can't be blank: %s" % line)
          elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[1]):
            raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[1])
          elif not stem.util.connection.is_valid_port(dir_source_comp[2], allow_zero = True):
            raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[2])
        elif not dir_source_comp[2].isdigit():
          continue

        self.hostname = dir_source_comp[0]
        self.address = dir_source_comp[1]
        self.dir_port = None if dir_source_comp[2] == '0' else int(dir_source_comp[2])
      elif keyword == "fingerprint":
        if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
          raise ValueError("Authority's fingerprint in a v2 network status document is malformed: %s" % line)

        self.fingerprint = value
      elif keyword == "contact":
        self.contact = value
      elif keyword == "dir-signing-key":
        self.signing_key = block_contents
      elif keyword in ("client-versions", "server-versions"):
        # v2 documents existed while there were tor versions using the 'old'
        # style, hence we aren't attempting to parse them

        for version_str in value.split(","):
          if keyword == 'client-versions':
            self.client_versions.append(version_str)
          elif keyword == 'server-versions':
            self.server_versions.append(version_str)
      elif keyword == "published":
        try:
          self.published = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
        except ValueError:
          if validate:
            raise ValueError("Version 2 network status document's 'published' time wasn't parsable: %s" % value)
      elif keyword == "dir-options":
        self.options = value.split()
      elif keyword == "directory-signature":
        self.signing_authority = value
        self.signature = block_contents
      else:
        self._unrecognized_lines.append(line)

    # 'client-versions' and 'server-versions' are only required if "Versions"
    # is among the options

    if validate and "Versions" in self.options:
      if not ('client-versions' in entries and 'server-versions' in entries):
        raise ValueError("Version 2 network status documents must have a 'client-versions' and 'server-versions' when 'Versions' is listed among its dir-options:\n%s" % str(self))

  def _check_constraints(self, entries):
    required_fields = [field for (field, is_mandatory) in NETWORK_STATUS_V2_FIELDS if is_mandatory]
    for keyword in required_fields:
      if not keyword in entries:
        raise ValueError("Network status document (v2) must have a '%s' line:\n%s" % (keyword, str(self)))

    # all recognized fields can only appear once
    single_fields = [field for (field, _) in NETWORK_STATUS_V2_FIELDS]
    for keyword in single_fields:
      if keyword in entries and len(entries[keyword]) > 1:
        raise ValueError("Network status document (v2) can only have a single '%s' line, got %i:\n%s" % (keyword, len(entries[keyword]), str(self)))

    if 'network-status-version' != entries.keys()[0]:
      raise ValueError("Network status document (v2) are expected to start with a 'network-status-version' line:\n%s" % str(self))


class NetworkStatusDocumentV3(NetworkStatusDocument):
  """
  Version 3 network status document. This could be either a vote or consensus.

  :var tuple routers: :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3`
    contained in the document

  :var int version: **\*** document version
  :var str version_flavor: **\*** flavor associated with the document (such as 'microdesc')
  :var bool is_consensus: **\*** **True** if the document is a consensus
  :var bool is_vote: **\*** **True** if the document is a vote
  :var bool is_microdescriptor: **\*** **True** if this is a microdescriptor
    flavored document, **False** otherwise
  :var datetime valid_after: **\*** time when the consensus became valid
  :var datetime fresh_until: **\*** time when the next consensus should be produced
  :var datetime valid_until: **\*** time when this consensus becomes obsolete
  :var int vote_delay: **\*** number of seconds allowed for collecting votes
    from all authorities
  :var int dist_delay: **\*** number of seconds allowed for collecting
    signatures from all authorities
  :var list client_versions: list of recommended client tor versions
  :var list server_versions: list of recommended server tor versions
  :var list known_flags: **\*** list of :data:`~stem.Flag` for the router's flags
  :var dict params: **\*** dict of parameter(**str**) => value(**int**) mappings
  :var list directory_authorities: **\*** list of :class:`~stem.descriptor.networkstatus.DirectoryAuthority`
    objects that have generated this document
  :var list signatures: **\*** :class:`~stem.descriptor.networkstatus.DocumentSignature`
    of the authorities that have signed the document

  **Consensus Attributes:**

  :var int consensus_method: method version used to generate this consensus
  :var dict bandwidth_weights: dict of weight(str) => value(int) mappings

  **Vote Attributes:**

  :var list consensus_methods: list of ints for the supported method versions
  :var datetime published: time when the document was published
  :var dict flag_thresholds: **\*** mapping of internal performance thresholds used while making the vote, values are **ints** or **floats**

  **\*** attribute is either required when we're parsed with validation or has
  a default value, others are left as None if undefined
  """

  def __init__(self, raw_content, validate = True, default_params = True):
    """
    Parse a v3 network status document.

    :param str raw_content: raw network status document data
    :param bool validate: **True** if the document is to be validated, **False** otherwise
    :param bool default_params: includes defaults in our params dict, otherwise
      it just contains values from the document

    :raises: **ValueError** if the document is invalid
    """

    super(NetworkStatusDocumentV3, self).__init__(raw_content)
    document_file = io.BytesIO(raw_content)

    self._header = _DocumentHeader(document_file, validate, default_params)

    # merge header attributes into us
    for attr, value in vars(self._header).items():
      if attr != "_unrecognized_lines":
        setattr(self, attr, value)
      else:
        self._unrecognized_lines += value

    self.directory_authorities = tuple(stem.descriptor.router_status_entry._parse_file(
      document_file,
      validate,
      entry_class = DirectoryAuthority,
      entry_keyword = AUTH_START,
      section_end_keywords = (ROUTERS_START, FOOTER_START, V2_FOOTER_START),
      extra_args = (self._header.is_vote,),
    ))

    if validate and self._header.is_vote and len(self.directory_authorities) != 1:
      raise ValueError("Votes should only have an authority entry for the one that issued it, got %i: %s" % (len(self.directory_authorities), self.directory_authorities))

    if not self._header.is_microdescriptor:
      router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3
    else:
      router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3

    router_iter = stem.descriptor.router_status_entry._parse_file(
      document_file,
      validate,
      entry_class = router_type,
      entry_keyword = ROUTERS_START,
      section_end_keywords = (FOOTER_START, V2_FOOTER_START),
      extra_args = (self,),
    )

    self.routers = dict((desc.fingerprint, desc) for desc in router_iter)

    self._footer = _DocumentFooter(document_file, validate, self._header)

    # merge header attributes into us
    for attr, value in vars(self._footer).items():
      if attr != "_unrecognized_lines":
        setattr(self, attr, value)
      else:
        self._unrecognized_lines += value

  def meets_consensus_method(self, method):
    """
    Checks if we meet the given consensus-method. This works for both votes and
    consensuses, checking our 'consensus-method' and 'consensus-methods'
    entries.

    :param int method: consensus-method to check for

    :returns: **True** if we meet the given consensus-method, and **False** otherwise
    """

    return self._header.meets_consensus_method(method)

  def _compare(self, other, method):
    if not isinstance(other, NetworkStatusDocumentV3):
      return False

    return method(str(self).strip(), str(other).strip())

  def __hash__(self):
    return hash(str(self).strip())

  def __eq__(self, other):
    return self._compare(other, lambda s, o: s == o)

  def __lt__(self, other):
    return self._compare(other, lambda s, o: s < o)

  def __le__(self, other):
    return self._compare(other, lambda s, o: s <= o)


class _DocumentHeader(object):
  def __init__(self, document_file, validate, default_params):
    self.version = None
    self.version_flavor = None
    self.is_consensus = True
    self.is_vote = False
    self.is_microdescriptor = False
    self.consensus_methods = []
    self.published = None
    self.consensus_method = None
    self.valid_after = None
    self.fresh_until = None
    self.valid_until = None
    self.vote_delay = None
    self.dist_delay = None
    self.client_versions = []
    self.server_versions = []
    self.known_flags = []
    self.flag_thresholds = {}
    self.params = dict(DEFAULT_PARAMS) if default_params else {}

    self._unrecognized_lines = []

    content = bytes.join(b"", _read_until_keywords((AUTH_START, ROUTERS_START, FOOTER_START), document_file))
    content = stem.util.str_tools._to_unicode(content)
    entries = _get_descriptor_components(content, validate)
    self._parse(entries, validate)

    # doing this validation afterward so we know our 'is_consensus' and
    # 'is_vote' attributes

    if validate:
      _check_for_missing_and_disallowed_fields(self, entries, HEADER_STATUS_DOCUMENT_FIELDS)
      _check_for_misordered_fields(entries, HEADER_FIELDS)

  def meets_consensus_method(self, method):
    if self.consensus_method is not None:
      return self.consensus_method >= method
    elif self.consensus_methods is not None:
      return bool(filter(lambda x: x >= method, self.consensus_methods))
    else:
      return False  # malformed document

  def _parse(self, entries, validate):
    for keyword, values in entries.items():
      value, _ = values[0]
      line = "%s %s" % (keyword, value)

      # all known header fields can only appear once except
      if validate and len(values) > 1 and keyword in HEADER_FIELDS:
        raise ValueError("Network status documents can only have a single '%s' line, got %i" % (keyword, len(values)))

      if keyword == 'network-status-version':
        # "network-status-version" version

        if ' ' in value:
          version, flavor = value.split(' ', 1)
        else:
          version, flavor = value, None

        if not version.isdigit():
          if not validate:
            continue

          raise ValueError("Network status document has a non-numeric version: %s" % line)

        self.version = int(version)
        self.version_flavor = flavor
        self.is_microdescriptor = flavor == 'microdesc'

        if validate and self.version != 3:
          raise ValueError("Expected a version 3 network status document, got version '%s' instead" % self.version)
      elif keyword == 'vote-status':
        # "vote-status" type
        #
        # The consensus-method and consensus-methods fields are optional since
        # they weren't included in version 1. Setting a default now that we
        # know if we're a vote or not.

        if value == 'consensus':
          self.is_consensus, self.is_vote = True, False
          self.consensus_method = 1
        elif value == 'vote':
          self.is_consensus, self.is_vote = False, True
          self.consensus_methods = [1]
        elif validate:
          raise ValueError("A network status document's vote-status line can only be 'consensus' or 'vote', got '%s' instead" % value)
      elif keyword == 'consensus-methods':
        # "consensus-methods" IntegerList

        consensus_methods = []
        for entry in value.split(" "):
          if entry.isdigit():
            consensus_methods.append(int(entry))
          elif validate:
            raise ValueError("A network status document's consensus-methods must be a list of integer values, but was '%s'" % value)

        self.consensus_methods = consensus_methods

        if validate and not (1 in self.consensus_methods):
          raise ValueError("Network status votes must include consensus-method version 1")
      elif keyword == 'consensus-method':
        # "consensus-method" Integer

        if value.isdigit():
          self.consensus_method = int(value)
        elif validate:
          raise ValueError("A network status document's consensus-method must be an integer, but was '%s'" % value)
      elif keyword in ('published', 'valid-after', 'fresh-until', 'valid-until'):
        try:
          date_value = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S")

          if keyword == 'published':
            self.published = date_value
          elif keyword == 'valid-after':
            self.valid_after = date_value
          elif keyword == 'fresh-until':
            self.fresh_until = date_value
          elif keyword == 'valid-until':
            self.valid_until = date_value
        except ValueError:
          if validate:
            raise ValueError("Network status document's '%s' time wasn't parsable: %s" % (keyword, value))
      elif keyword == "voting-delay":
        # "voting-delay" VoteSeconds DistSeconds

        value_comp = value.split(' ')

        if len(value_comp) == 2 and value_comp[0].isdigit() and value_comp[1].isdigit():
          self.vote_delay = int(value_comp[0])
          self.dist_delay = int(value_comp[1])
        elif validate:
          raise ValueError("A network status document's 'voting-delay' line must be a pair of integer values, but was '%s'" % value)
      elif keyword in ("client-versions", "server-versions"):
        for entry in value.split(","):
          try:
            version_value = stem.version._get_version(entry)

            if keyword == 'client-versions':
              self.client_versions.append(version_value)
            elif keyword == 'server-versions':
              self.server_versions.append(version_value)
          except ValueError:
            if validate:
              raise ValueError("Network status document's '%s' line had '%s', which isn't a parsable tor version: %s" % (keyword, entry, line))
      elif keyword == "known-flags":
        # "known-flags" FlagList

        # simply fetches the entries, excluding empty strings
        self.known_flags = [entry for entry in value.split(" ") if entry]
      elif keyword == "flag-thresholds":
        # "flag-thresholds" SP THRESHOLDS

        value = value.strip()

        if value:
          for entry in value.split(" "):
            if not '=' in entry:
              if not validate:
                continue

              raise ValueError("Network status document's '%s' line is expected to be space separated key=value mappings, got: %s" % (keyword, line))

            entry_key, entry_value = entry.split("=", 1)

            try:
              if entry_value.endswith("%"):
                # opting for string manipulation rather than just
                # 'float(entry_value) / 100' because floating point arithmetic
                # will lose precision

                self.flag_thresholds[entry_key] = float("0." + entry_value[:-1].replace('.', '', 1))
              elif '.' in entry_value:
                self.flag_thresholds[entry_key] = float(entry_value)
              else:
                self.flag_thresholds[entry_key] = int(entry_value)
            except ValueError:
              if validate:
                raise ValueError("Network status document's '%s' line is expected to have float values, got: %s" % (keyword, line))
      elif keyword == "params":
        # "params" [Parameters]
        # Parameter ::= Keyword '=' Int32
        # Int32 ::= A decimal integer between -2147483648 and 2147483647.
        # Parameters ::= Parameter | Parameters SP Parameter

        # should only appear in consensus-method 7 or later

        if validate and not self.meets_consensus_method(7):
          raise ValueError("A network status document's 'params' line should only appear in consensus-method 7 or later")

        # skip if this is a blank line

        if value == "":
          continue

        self.params.update(_parse_int_mappings(keyword, value, validate))

        if validate:
          self._check_params_constraints()
      else:
        self._unrecognized_lines.append(line)

  def _check_params_constraints(self):
    """
    Checks that the params we know about are within their documented ranges.
    """

    for key, value in self.params.items():
      # all parameters are constrained to int32 range
      minimum, maximum = -2147483648, 2147483647

      if key == "circwindow":
        minimum, maximum = 100, 1000
      elif key == "CircuitPriorityHalflifeMsec":
        minimum = -1
      elif key in ("perconnbwrate", "perconnbwburst"):
        minimum = 1
      elif key == "refuseunknownexits":
        minimum, maximum = 0, 1
      elif key == "bwweightscale":
        minimum = 1
      elif key == "cbtdisabled":
        minimum, maximum = 0, 1
      elif key == "cbtnummodes":
        minimum, maximum = 1, 20
      elif key == "cbtrecentcount":
        minimum, maximum = 3, 1000
      elif key == "cbtmaxtimeouts":
        minimum, maximum = 3, 10000
      elif key == "cbtmincircs":
        minimum, maximum = 1, 10000
      elif key == "cbtquantile":
        minimum, maximum = 10, 99
      elif key == "cbtclosequantile":
        minimum, maximum = self.params.get("cbtquantile", minimum), 99
      elif key == "cbttestfreq":
        minimum = 1
      elif key == "cbtmintimeout":
        minimum = 500
      elif key == "cbtinitialtimeout":
        minimum = self.params.get("cbtmintimeout", minimum)
      elif key == "UseOptimisticData":
        minimum, maximum = 0, 1
      elif key == "Support022HiddenServices":
        minimum, maximum = 0, 1

      if value < minimum or value > maximum:
        raise ValueError("'%s' value on the params line must be in the range of %i - %i, was %i" % (key, minimum, maximum, value))


class _DocumentFooter(object):
  def __init__(self, document_file, validate, header):
    self.signatures = []
    self.bandwidth_weights = {}
    self._unrecognized_lines = []

    content = stem.util.str_tools._to_unicode(document_file.read())

    if not content:
      return  # footer is optional and there's nothing to parse

    entries = _get_descriptor_components(content, validate)
    self._parse(entries, validate, header)

    if validate:
      # Check that the footer has the right initial line. Prior to consensus
      # method 9 it's a 'directory-signature' and after that footers start with
      # 'directory-footer'.

      if header.meets_consensus_method(9):
        if entries.keys()[0] != 'directory-footer':
          raise ValueError("Network status document's footer should start with a 'directory-footer' line in consensus-method 9 or later")
      else:
        if entries.keys()[0] != 'directory-signature':
          raise ValueError("Network status document's footer should start with a 'directory-signature' line prior to consensus-method 9")

      _check_for_missing_and_disallowed_fields(header, entries, FOOTER_STATUS_DOCUMENT_FIELDS)
      _check_for_misordered_fields(entries, FOOTER_FIELDS)

  def _parse(self, entries, validate, header):
    for keyword, values in entries.items():
      value, block_contents = values[0]
      line = "%s %s" % (keyword, value)

      # all known footer fields can only appear once except...
      # * 'directory-signature' in a consensus

      if validate and len(values) > 1 and keyword in FOOTER_FIELDS:
        if not (keyword == 'directory-signature' and header.is_consensus):
          raise ValueError("Network status documents can only have a single '%s' line, got %i" % (keyword, len(values)))

      if keyword == "directory-footer":
        # nothing to parse, simply checking that we don't have a value

        if validate and value:
          raise ValueError("A network status document's 'directory-footer' line shouldn't have any content, got '%s'" % line)
      elif keyword == "bandwidth-weights":
        self.bandwidth_weights = _parse_int_mappings(keyword, value, validate)
      elif keyword == "directory-signature":
        for sig_value, block_contents in values:
          if not sig_value.count(" ") in (1, 2) or not block_contents:
            if not validate:
              continue

            raise ValueError("Authority signatures in a network status document are expected to be of the form 'directory-signature [METHOD] FINGERPRINT KEY_DIGEST\\nSIGNATURE', got:\n%s\n%s" % (sig_value, block_contents))

          if sig_value.count(" ") == 1:
            method = 'sha1'  # default if none was provided
            fingerprint, key_digest = sig_value.split(" ", 1)
          else:
            method, fingerprint, key_digest = sig_value.split(" ", 2)

          self.signatures.append(DocumentSignature(method, fingerprint, key_digest, block_contents, validate))


def _check_for_missing_and_disallowed_fields(header, entries, fields):
  """
  Checks that we have mandatory fields for our type, and that we don't have
  any fields exclusive to the other (ie, no vote-only fields appear in a
  consensus or vice versa).

  :param _DocumentHeader header: document header
  :param dict entries: ordered keyword/value mappings of the header or footer
  :param list fields: expected field attributes (either
    **HEADER_STATUS_DOCUMENT_FIELDS** or **FOOTER_STATUS_DOCUMENT_FIELDS**)

  :raises: **ValueError** if we're missing mandatory fields or have fields we shouldn't
  """

  missing_fields, disallowed_fields = [], []

  for field, in_votes, in_consensus, mandatory in fields:
    if mandatory and ((header.is_consensus and in_consensus) or (header.is_vote and in_votes)):
      # mandatory field, check that we have it
      if not field in entries.keys():
        missing_fields.append(field)
    elif (header.is_consensus and not in_consensus) or (header.is_vote and not in_votes):
      # field we shouldn't have, check that we don't
      if field in entries.keys():
        disallowed_fields.append(field)

  if missing_fields:
    raise ValueError("Network status document is missing mandatory field: %s" % ', '.join(missing_fields))

  if disallowed_fields:
    raise ValueError("Network status document has fields that shouldn't appear in this document type or version: %s" % ', '.join(disallowed_fields))


def _check_for_misordered_fields(entries, expected):
  """
  To be valid a network status document's fiends need to appear in a specific
  order. Checks that known fields appear in that order (unrecognized fields
  are ignored).

  :param dict entries: ordered keyword/value mappings of the header or footer
  :param list expected: ordered list of expected fields (either
    **HEADER_FIELDS** or **FOOTER_FIELDS**)

  :raises: **ValueError** if entries aren't properly ordered
  """

  # Earlier validation has ensured that our fields either belong to our
  # document type or are unknown. Remove the unknown fields since they
  # reflect a spec change and can appear anywhere in the document.

  actual = filter(lambda field: field in expected, entries.keys())

  # Narrow the expected to just what we have. If the lists then match then the
  # order's valid.

  expected = filter(lambda field: field in actual, expected)

  if actual != expected:
    actual_label = ', '.join(actual)
    expected_label = ', '.join(expected)
    raise ValueError("The fields in a section of the document are misordered. It should be '%s' but was '%s'" % (actual_label, expected_label))


def _parse_int_mappings(keyword, value, validate):
  # Parse a series of 'key=value' entries, checking the following:
  # - values are integers
  # - keys are sorted in lexical order

  results, seen_keys = {}, []
  for entry in value.split(" "):
    try:
      if not '=' in entry:
        raise ValueError("must only have 'key=value' entries")

      entry_key, entry_value = entry.split("=", 1)

      try:
        # the int() function accepts things like '+123', but we don't want to
        if entry_value.startswith('+'):
          raise ValueError()

        entry_value = int(entry_value)
      except ValueError:
        raise ValueError("'%s' is a non-numeric value" % entry_value)

      if validate:
        # parameters should be in ascending order by their key
        for prior_key in seen_keys:
          if prior_key > entry_key:
            raise ValueError("parameters must be sorted by their key")

      results[entry_key] = entry_value
      seen_keys.append(entry_key)
    except ValueError as exc:
      if not validate:
        continue

      raise ValueError("Unable to parse network status document's '%s' line (%s): %s'" % (keyword, exc, value))

  return results


class DirectoryAuthority(Descriptor):
  """
  Directory authority information obtained from a v3 network status document.

  Authorities can optionally use a legacy format. These are no longer found in
  practice, but have the following differences...

  * The authority's nickname ends with '-legacy'.
  * There's no **contact** or **vote_digest** attribute.

  :var str nickname: **\*** authority's nickname
  :var str fingerprint: **\*** authority's fingerprint
  :var str hostname: **\*** hostname of the authority
  :var str address: **\*** authority's IP address
  :var int dir_port: **\*** authority's DirPort
  :var int or_port: **\*** authority's ORPort
  :var bool is_legacy: **\*** if the authority's using the legacy format
  :var str contact: contact information, this is included if is_legacy is **False**

  **Consensus Attributes:**

  :var str vote_digest: digest of the authority that contributed to the consensus, this is included if is_legacy is **False**

  **Vote Attributes:**

  :var str legacy_dir_key: fingerprint of and obsolete identity key
  :var stem.descriptor.networkstatus.KeyCertificate key_certificate: **\***
    authority's key certificate

  **\*** mandatory attribute
  """

  def __init__(self, raw_content, validate = True, is_vote = False):
    """
    Parse a directory authority entry in a v3 network status document.

    :param str raw_content: raw directory authority entry information
    :param bool validate: checks the validity of the content if True, skips
      these checks otherwise
    :param bool is_vote: True if this is for a vote, False if it's for a consensus

    :raises: ValueError if the descriptor data is invalid
    """

    super(DirectoryAuthority, self).__init__(raw_content)
    raw_content = stem.util.str_tools._to_unicode(raw_content)

    self.nickname = None
    self.fingerprint = None
    self.hostname = None
    self.address = None
    self.dir_port = None
    self.or_port = None
    self.is_legacy = False
    self.contact = None

    self.vote_digest = None

    self.legacy_dir_key = None
    self.key_certificate = None

    self._unrecognized_lines = []

    self._parse(raw_content, validate, is_vote)

  def _parse(self, content, validate, is_vote):
    """
    Parses the given content and applies the attributes.

    :param str content: descriptor content
    :param bool validate: checks validity if True
    :param bool is_vote: **True** if this is for a vote, **False** if it's for
      a consensus

    :raises: **ValueError** if a validity check fails
    """

    # separate the directory authority entry from its key certificate
    key_div = content.find('\ndir-key-certificate-version')

    if key_div != -1:
      key_cert_content = content[key_div + 1:]
      content = content[:key_div + 1]
    else:
      key_cert_content = None

    entries = _get_descriptor_components(content, validate)

    if validate and 'dir-source' != entries.keys()[0]:
      raise ValueError("Authority entries are expected to start with a 'dir-source' line:\n%s" % (content))

    # check that we have mandatory fields

    if validate:
      is_legacy, dir_source_entry = False, entries.get("dir-source")

      if dir_source_entry:
        is_legacy = dir_source_entry[0][0].split()[0].endswith("-legacy")

      required_fields, excluded_fields = ["dir-source"], []

      if not is_legacy:
        required_fields += ["contact"]

      if is_vote:
        if not key_cert_content:
          raise ValueError("Authority votes must have a key certificate:\n%s" % content)

        excluded_fields += ["vote-digest"]
      elif not is_vote:
        if key_cert_content:
          raise ValueError("Authority consensus entries shouldn't have a key certificate:\n%s" % content)

        if not is_legacy:
          required_fields += ["vote-digest"]

        excluded_fields += ["legacy-dir-key"]

      for keyword in required_fields:
        if not keyword in entries:
          raise ValueError("Authority entries must have a '%s' line:\n%s" % (keyword, content))

      for keyword in entries:
        if keyword in excluded_fields:
          type_label = "votes" if is_vote else "consensus entries"
          raise ValueError("Authority %s shouldn't have a '%s' line:\n%s" % (type_label, keyword, content))

    for keyword, values in entries.items():
      value, _ = values[0]
      line = "%s %s" % (keyword, value)

      # all known attributes can only appear at most once
      if validate and len(values) > 1 and keyword in ('dir-source', 'contact', 'legacy-dir-key', 'vote-digest'):
        raise ValueError("Authority entries can only have a single '%s' line, got %i:\n%s" % (keyword, len(values), content))

      if keyword == 'dir-source':
        # "dir-source" nickname identity address IP dirport orport

        dir_source_comp = value.split(" ")

        if len(dir_source_comp) < 6:
          if not validate:
            continue

          raise ValueError("Authority entry's 'dir-source' line must have six values: %s" % line)

        if validate:
          if not stem.util.tor_tools.is_valid_nickname(dir_source_comp[0].rstrip('-legacy')):
            raise ValueError("Authority's nickname is invalid: %s" % dir_source_comp[0])
          elif not stem.util.tor_tools.is_valid_fingerprint(dir_source_comp[1]):
            raise ValueError("Authority's fingerprint is invalid: %s" % dir_source_comp[1])
          elif not dir_source_comp[2]:
            # https://trac.torproject.org/7055
            raise ValueError("Authority's hostname can't be blank: %s" % line)
          elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[3]):
            raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[3])
          elif not stem.util.connection.is_valid_port(dir_source_comp[4], allow_zero = True):
            raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[4])
          elif not stem.util.connection.is_valid_port(dir_source_comp[5]):
            raise ValueError("Authority's ORPort is invalid: %s" % dir_source_comp[5])
        elif not (dir_source_comp[4].isdigit() and dir_source_comp[5].isdigit()):
          continue

        self.nickname = dir_source_comp[0]
        self.fingerprint = dir_source_comp[1]
        self.hostname = dir_source_comp[2]
        self.address = dir_source_comp[3]
        self.dir_port = None if dir_source_comp[4] == '0' else int(dir_source_comp[4])
        self.or_port = int(dir_source_comp[5])
        self.is_legacy = self.nickname.endswith("-legacy")
      elif keyword == 'contact':
        # "contact" string

        self.contact = value
      elif keyword == 'legacy-dir-key':
        # "legacy-dir-key" FINGERPRINT

        if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
          raise ValueError("Authority has a malformed legacy directory key: %s" % line)

        self.legacy_dir_key = value
      elif keyword == 'vote-digest':
        # "vote-digest" digest

        # technically not a fingerprint, but has the same characteristics
        if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
          raise ValueError("Authority has a malformed vote digest: %s" % line)

        self.vote_digest = value
      else:
        self._unrecognized_lines.append(line)

    if key_cert_content:
      self.key_certificate = KeyCertificate(key_cert_content, validate)

  def get_unrecognized_lines(self):
    """
    Returns any unrecognized lines.

    :returns: a list of unrecognized lines
    """

    return self._unrecognized_lines

  def _compare(self, other, method):
    if not isinstance(other, DirectoryAuthority):
      return False

    return method(str(self).strip(), str(other).strip())

  def __eq__(self, other):
    return self._compare(other, lambda s, o: s == o)

  def __lt__(self, other):
    return self._compare(other, lambda s, o: s < o)

  def __le__(self, other):
    return self._compare(other, lambda s, o: s <= o)


class KeyCertificate(Descriptor):
  """
  Directory key certificate for a v3 network status document.

  :var int version: **\*** version of the key certificate
  :var str address: authority's IP address
  :var int dir_port: authority's DirPort
  :var str fingerprint: **\*** authority's fingerprint
  :var str identity_key: **\*** long term authority identity key
  :var datetime published: **\*** time when this key was generated
  :var datetime expires: **\*** time after which this key becomes invalid
  :var str signing_key: **\*** directory server's public signing key
  :var str crosscert: signature made using certificate's signing key
  :var str certification: **\*** signature of this key certificate signed with
    the identity key

  **\*** mandatory attribute
  """

  def __init__(self, raw_content, validate = True):
    super(KeyCertificate, self).__init__(raw_content)
    raw_content = stem.util.str_tools._to_unicode(raw_content)

    self.version = None
    self.address = None
    self.dir_port = None
    self.fingerprint = None
    self.identity_key = None
    self.published = None
    self.expires = None
    self.signing_key = None
    self.crosscert = None
    self.certification = None

    self._unrecognized_lines = []

    self._parse(raw_content, validate)

  def _parse(self, content, validate):
    """
    Parses the given content and applies the attributes.

    :param str content: descriptor content
    :param bool validate: checks validity if **True**

    :raises: **ValueError** if a validity check fails
    """

    entries = _get_descriptor_components(content, validate)

    if validate:
      if 'dir-key-certificate-version' != entries.keys()[0]:
        raise ValueError("Key certificates must start with a 'dir-key-certificate-version' line:\n%s" % (content))
      elif 'dir-key-certification' != entries.keys()[-1]:
        raise ValueError("Key certificates must end with a 'dir-key-certification' line:\n%s" % (content))

      # check that we have mandatory fields and that our known fields only
      # appear once

      for keyword, is_mandatory in KEY_CERTIFICATE_PARAMS:
        if is_mandatory and not keyword in entries:
          raise ValueError("Key certificates must have a '%s' line:\n%s" % (keyword, content))

        entry_count = len(entries.get(keyword, []))
        if entry_count > 1:
          raise ValueError("Key certificates can only have a single '%s' line, got %i:\n%s" % (keyword, entry_count, content))

    for keyword, values in entries.items():
      value, block_contents = values[0]
      line = "%s %s" % (keyword, value)

      if keyword == 'dir-key-certificate-version':
        # "dir-key-certificate-version" version

        if not value.isdigit():
          if not validate:
            continue

          raise ValueError("Key certificate has a non-integer version: %s" % line)

        self.version = int(value)

        if validate and self.version != 3:
          raise ValueError("Expected a version 3 key certificate, got version '%i' instead" % self.version)
      elif keyword == 'dir-address':
        # "dir-address" IPPort

        if not ':' in value:
          if not validate:
            continue

          raise ValueError("Key certificate's 'dir-address' is expected to be of the form ADDRESS:PORT: %s" % line)

        address, dirport = value.split(':', 1)

        if validate:
          if not stem.util.connection.is_valid_ipv4_address(address):
            raise ValueError("Key certificate's address isn't a valid IPv4 address: %s" % line)
          elif not stem.util.connection.is_valid_port(dirport):
            raise ValueError("Key certificate's dirport is invalid: %s" % line)
        elif not dirport.isdigit():
          continue

        self.address = address
        self.dir_port = int(dirport)
      elif keyword == 'fingerprint':
        # "fingerprint" fingerprint

        if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
          raise ValueError("Key certificate's fingerprint is malformed: %s" % line)

        self.fingerprint = value
      elif keyword in ('dir-key-published', 'dir-key-expires'):
        # "dir-key-published" YYYY-MM-DD HH:MM:SS
        # "dir-key-expires" YYYY-MM-DD HH:MM:SS

        try:
          date_value = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S")

          if keyword == 'dir-key-published':
            self.published = date_value
          elif keyword == 'dir-key-expires':
            self.expires = date_value
        except ValueError:
          if validate:
            raise ValueError("Key certificate's '%s' time wasn't parsable: %s" % (keyword, value))
      elif keyword in ('dir-identity-key', 'dir-signing-key', 'dir-key-crosscert', 'dir-key-certification'):
        # "dir-identity-key" NL a public key in PEM format
        # "dir-signing-key" NL a key in PEM format
        # "dir-key-crosscert" NL CrossSignature
        # "dir-key-certification" NL Signature

        if validate and not block_contents:
          raise ValueError("Key certificate's '%s' line must be followed by a key block: %s" % (keyword, line))

        if keyword == 'dir-identity-key':
          self.identity_key = block_contents
        elif keyword == 'dir-signing-key':
          self.signing_key = block_contents
        elif keyword == 'dir-key-crosscert':
          self.crosscert = block_contents
        elif keyword == 'dir-key-certification':
          self.certification = block_contents
      else:
        self._unrecognized_lines.append(line)

  def get_unrecognized_lines(self):
    """
    Returns any unrecognized lines.

    :returns: **list** of unrecognized lines
    """

    return self._unrecognized_lines

  def _compare(self, other, method):
    if not isinstance(other, KeyCertificate):
      return False

    return method(str(self).strip(), str(other).strip())

  def __eq__(self, other):
    return self._compare(other, lambda s, o: s == o)

  def __lt__(self, other):
    return self._compare(other, lambda s, o: s < o)

  def __le__(self, other):
    return self._compare(other, lambda s, o: s <= o)


class DocumentSignature(object):
  """
  Directory signature of a v3 network status document.

  :var str method: algorithm used to make the signature
  :var str identity: fingerprint of the authority that made the signature
  :var str key_digest: digest of the signing key
  :var str signature: document signature
  :param bool validate: checks validity if **True**

  :raises: **ValueError** if a validity check fails
  """

  def __init__(self, method, identity, key_digest, signature, validate = True):
    # Checking that these attributes are valid. Technically the key
    # digest isn't a fingerprint, but it has the same characteristics.

    if validate:
      if not stem.util.tor_tools.is_valid_fingerprint(identity):
        raise ValueError("Malformed fingerprint (%s) in the document signature" % identity)

      if not stem.util.tor_tools.is_valid_fingerprint(key_digest):
        raise ValueError("Malformed key digest (%s) in the document signature" % key_digest)

    self.method = method
    self.identity = identity
    self.key_digest = key_digest
    self.signature = signature

  def _compare(self, other, method):
    if not isinstance(other, DocumentSignature):
      return False

    for attr in ("method", "identity", "key_digest", "signature"):
      if getattr(self, attr) != getattr(other, attr):
        return method(getattr(self, attr), getattr(other, attr))

    return method(True, True)  # we're equal

  def __eq__(self, other):
    return self._compare(other, lambda s, o: s == o)

  def __lt__(self, other):
    return self._compare(other, lambda s, o: s < o)

  def __le__(self, other):
    return self._compare(other, lambda s, o: s <= o)


class BridgeNetworkStatusDocument(NetworkStatusDocument):
  """
  Network status document containing bridges. This is only available through
  the metrics site.

  :var tuple routers: :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV2`
    contained in the document
  :var datetime published: time when the document was published
  """

  def __init__(self, raw_content, validate = True):
    super(BridgeNetworkStatusDocument, self).__init__(raw_content)

    self.published = None

    document_file = io.BytesIO(raw_content)
    published_line = stem.util.str_tools._to_unicode(document_file.readline())

    if published_line.startswith("published "):
      published_line = published_line.split(" ", 1)[1].strip()

      try:
        self.published = datetime.datetime.strptime(published_line, "%Y-%m-%d %H:%M:%S")
      except ValueError:
        if validate:
          raise ValueError("Bridge network status document's 'published' time wasn't parsable: %s" % published_line)
    elif validate:
      raise ValueError("Bridge network status documents must start with a 'published' line:\n%s" % stem.util.str_tools._to_unicode(raw_content))

    router_iter = stem.descriptor.router_status_entry._parse_file(
      document_file,
      validate,
      entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV2,
      extra_args = (self,),
    )

    self.routers = dict((desc.fingerprint, desc) for desc in router_iter)