WIP: in roledb, remove intermediate data format; #660

- Rename and alter some schemas that really address delegations,
to make that clear.
- Do away with the ROLEDB_SCHEMA, an intermediate metadata format
that is not necessary and which incorrectly flattens the delegation
graph, and similar schemas.
- Rewrite getters/setters in roledb to respect the delegation
graph rather than assuming that delegated targets roles have only
one delegation pointing to them (see Issue #660).
- Add a variety of TODOs for later.
- Clarify docstrings as a result of the above.

reinterpreting metadata

Signed-off-by: Sebastien Awwad <sebastien.awwad@gmail.com>

tuf/formats.py

@@ -110,22 +110,47 @@
 # A string representing a role's name.
 ROLENAME_SCHEMA = SCHEMA.AnyString()
 
+# THIS DEFINITION was previously overwritten further down in this file.
+# As a consequence, I'm not sure it used to do anything.....  The replacement
+# is gone now.
+# <~> In any case, this is a delegation, not a role!
 # Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1,
 # 'paths':[filepaths..]} format.
-ROLE_SCHEMA = SCHEMA.Object(
-  object_name = 'ROLE_SCHEMA',
+DELEGATION_SCHEMA = SCHEMA.Object(
+  object_name = 'DELEGATION_SCHEMA',
   name = SCHEMA.Optional(securesystemslib.formats.ROLENAME_SCHEMA),
   keyids = securesystemslib.formats.KEYIDS_SCHEMA,
   threshold = securesystemslib.formats.THRESHOLD_SCHEMA,
   terminating = SCHEMA.Optional(securesystemslib.formats.BOOLEAN_SCHEMA),
   paths = SCHEMA.Optional(securesystemslib.formats.RELPATHS_SCHEMA),
   path_hash_prefixes = SCHEMA.Optional(securesystemslib.formats.PATH_HASH_PREFIXES_SCHEMA))
 
+# This is the data stored for each top-level role, in root metadata.
+# TODO: Why is threshold schema in securesystemslib instead of here? Change?
+TOP_LEVEL_DELEGATION_SCHEMA = SCHEMA.Object(
+  object_name = 'TOP_LEVEL_DELEGATION_SCHEMA',
+  keyids = securesystemslib.formats.KEYIDS_SCHEMA,
+  threshold = securesystemslib.formats.THRESHOLD_SCHEMA)
+
+
+# # Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1,
+# # 'paths':[filepaths..]} format.
+# ROLE_SCHEMA = SCHEMA.Object(
+#   object_name = 'ROLE_SCHEMA',
+#   name = SCHEMA.Optional(securesystemslib.formats.ROLENAME_SCHEMA),
+#   keyids = securesystemslib.formats.KEYIDS_SCHEMA,
+#   threshold = securesystemslib.formats.THRESHOLD_SCHEMA,
+#   terminating = SCHEMA.Optional(securesystemslib.formats.BOOLEAN_SCHEMA),
+#   paths = SCHEMA.Optional(securesystemslib.formats.RELPATHS_SCHEMA),
+#   path_hash_prefixes = SCHEMA.Optional(securesystemslib.formats.PATH_HASH_PREFIXES_SCHEMA))
+
+
+# TODO: <~> Figure out what this is for and kill it or fix it.
 # A dict of roles where the dict keys are role names and the dict values holding
 # the role data/information.
-ROLEDICT_SCHEMA = SCHEMA.DictOf(
-  key_schema = ROLENAME_SCHEMA,
-  value_schema = ROLE_SCHEMA)
+# ROLEDICT_SCHEMA = SCHEMA.DictOf(
+#   key_schema = ROLENAME_SCHEMA,
+#   value_schema = ROLE_SCHEMA)
 
 # A dictionary of ROLEDICT, where dictionary keys can be repository names, and
 # dictionary values containing information for each role available on the
@@ -207,16 +232,17 @@
 # A list of path hash prefixes.
 PATH_HASH_PREFIXES_SCHEMA = SCHEMA.ListOf(PATH_HASH_PREFIX_SCHEMA)
 
-# Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1,
-# 'paths':[filepaths..]} format.
-ROLE_SCHEMA = SCHEMA.Object(
-  object_name = 'ROLE_SCHEMA',
-  name = SCHEMA.Optional(ROLENAME_SCHEMA),
-  keyids = KEYIDS_SCHEMA,
-  threshold = THRESHOLD_SCHEMA,
-  backtrack = SCHEMA.Optional(BOOLEAN_SCHEMA),
-  paths = SCHEMA.Optional(RELPATHS_SCHEMA),
-  path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA))
+# <~> This is a delegation, not a role.
+# # Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1,
+# # 'paths':[filepaths..]} format.
+# ROLE_SCHEMA = SCHEMA.Object(
+#   object_name = 'ROLE_SCHEMA',
+#   name = SCHEMA.Optional(ROLENAME_SCHEMA),
+#   keyids = KEYIDS_SCHEMA,
+#   threshold = THRESHOLD_SCHEMA,
+#   backtrack = SCHEMA.Optional(BOOLEAN_SCHEMA),
+#   paths = SCHEMA.Optional(RELPATHS_SCHEMA),
+#   path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA))
 
 # A dict of roles where the dict keys are role names and the dict values holding
 # the role data/information.
@@ -283,7 +309,7 @@
 # Like ROLEDICT_SCHEMA, except that ROLE_SCHEMA instances are stored in order.
 ROLELIST_SCHEMA = SCHEMA.ListOf(ROLE_SCHEMA)
 
-# The delegated roles of a Targets role (a parent).
+# The 'delegations' entry in a piece of targets role metadata.
 DELEGATIONS_SCHEMA = SCHEMA.Object(
   keys = KEYDICT_SCHEMA,
   roles = ROLELIST_SCHEMA)
@@ -303,21 +329,23 @@
   key_schema = RELPATH_SCHEMA,
   value_schema = CUSTOM_SCHEMA)
 
-# TUF roledb
-ROLEDB_SCHEMA = SCHEMA.Object(
-  object_name = 'ROLEDB_SCHEMA',
-  keyids = SCHEMA.Optional(KEYIDS_SCHEMA),
-  signing_keyids = SCHEMA.Optional(KEYIDS_SCHEMA),
-  previous_keyids = SCHEMA.Optional(KEYIDS_SCHEMA),
-  threshold = SCHEMA.Optional(THRESHOLD_SCHEMA),
-  previous_threshold = SCHEMA.Optional(THRESHOLD_SCHEMA),
-  version = SCHEMA.Optional(METADATAVERSION_SCHEMA),
-  expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA),
-  signatures = SCHEMA.Optional(securesystemslib.formats.SIGNATURES_SCHEMA),
-  paths = SCHEMA.Optional(SCHEMA.OneOf([RELPATHS_SCHEMA, PATH_FILEINFO_SCHEMA])),
-  path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA),
-  delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA),
-  partial_loaded = SCHEMA.Optional(BOOLEAN_SCHEMA))
+# TODO: <~> Kill it with fire.  This is nonsensical.  We use the actual
+#           metadata format.  Maybe we add partial_loaded if we need it.
+# # TUF roledb
+# ROLEDB_SCHEMA = SCHEMA.Object(
+#   object_name = 'ROLEDB_SCHEMA',
+#   keyids = SCHEMA.Optional(KEYIDS_SCHEMA),
+#   signing_keyids = SCHEMA.Optional(KEYIDS_SCHEMA),
+#   previous_keyids = SCHEMA.Optional(KEYIDS_SCHEMA),
+#   threshold = SCHEMA.Optional(THRESHOLD_SCHEMA),
+#   previous_threshold = SCHEMA.Optional(THRESHOLD_SCHEMA),
+#   version = SCHEMA.Optional(METADATAVERSION_SCHEMA),
+#   expires = SCHEMA.Optional(ISO8601_DATETIME_SCHEMA),
+#   signatures = SCHEMA.Optional(securesystemslib.formats.SIGNATURES_SCHEMA),
+#   paths = SCHEMA.Optional(SCHEMA.OneOf([RELPATHS_SCHEMA, PATH_FILEINFO_SCHEMA])),
+#   path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA),
+#   delegations = SCHEMA.Optional(DELEGATIONS_SCHEMA),
+#   partial_loaded = SCHEMA.Optional(BOOLEAN_SCHEMA))
 
 # A signable object.  Holds the signing role and its associated signatures.
 SIGNABLE_SCHEMA = SCHEMA.Object(