Concepts
Here is a generic solution that does what you need. The concept it uses is recursively looping through all values of the top-level "persons" dictionary. Based on the type of each value it finds, it proceeds.
So for all the non-dict/non-lists it finds in each dictionary, it puts those into the top-level object you need.
Or if it finds a dictionary or a list, it recursively does the same thing again, finding more non-dict/non-lists or lists or dictionaries.
Also using collections.defaultdict lets us easily populate an unknown number of lists for each key, into a dictionary, so that we can get those 4 top-level objects you want.
Code example
from collections import defaultdict
class DictFlattener(object):
def __init__(self, object_id_key, object_name):
"""Constructor.
:param object_id_key: String key that identifies each base object
:param object_name: String name given to the base object in data.
"""
self._object_id_key = object_id_key
self._object_name = object_name
# Store each of the top-level results lists.
self._collected_results = None
def parse(self, data):
"""Parse the given nested dictionary data into separate lists.
Each nested dictionary is transformed into its own list of objects,
associated with the original object via the object id.
:param data: Dictionary of data to parse.
:returns: Single dictionary containing the resulting lists of
objects, where each key is the object name combined with the
list name via an underscore.
"""
self._collected_results = defaultdict(list)
for value_to_parse in data[self._object_name]:
object_id = value_to_parse[self._object_id_key]
parsed_object = {}
for key, value in value_to_parse.items():
sub_object_name = self._object_name + "_" + key
parsed_value = self._parse_value(
value,
object_id,
sub_object_name,
)
if parsed_value:
parsed_object[key] = parsed_value
self._collected_results[self._object_name].append(parsed_object)
return self._collected_results
def _parse_value(self, value_to_parse, object_id, current_object_name, index=None):
"""Parse some value of an unknown type.
If it's a list or a dict, keep parsing, otherwise return it as-is.
:param value_to_parse: Value to parse
:param object_id: String id of the current top object being parsed.
:param current_object_name: Name of the current level being parsed.
:returns: None if value_to_parse is a dict or a list, otherwise returns
value_to_parse.
"""
if isinstance(value_to_parse, dict):
self._parse_dict(
value_to_parse,
object_id,
current_object_name,
index=index,
)
elif isinstance(value_to_parse, list):
self._parse_list(
value_to_parse,
object_id,
current_object_name,
)
else:
return value_to_parse
def _parse_dict(self, dict_to_parse, object_id, current_object_name,
index=None):
"""Parse some value of a dict type and store it in self._collected_results.
:param dict_to_parse: Dict to parse
:param object_id: String id of the current top object being parsed.
:param current_object_name: Name of the current level being parsed.
"""
parsed_dict = {
self._object_id_key: object_id,
}
if index is not None:
parsed_dict["__index"] = index
for key, value in dict_to_parse.items():
sub_object_name = current_object_name + "_" + key
parsed_value = self._parse_value(
value,
object_id,
sub_object_name,
index=index,
)
if parsed_value:
parsed_dict[key] = value
self._collected_results[current_object_name].append(parsed_dict)
def _parse_list(self, list_to_parse, object_id, current_object_name):
"""Parse some value of a list type and store it in self._collected_results.
:param list_to_parse: Dict to parse
:param object_id: String id of the current top object being parsed.
:param current_object_name: Name of the current level being parsed.
"""
for index, sub_dict in enumerate(list_to_parse):
self._parse_value(
sub_dict,
object_id,
current_object_name,
index=index,
)
Then to use it:
parser = DictFlattener("id", "persons")
results = parser.parse(test_data)
Notes
- that there were some inconsistencies in your example data vs expected, like scores were strings vs ints. So you'll need to tweak those when you compare given to expected.
- There's always more refactoring one could do, or it could be made more functional rather than being a class. But hopefully looking at this helps you understand how to do it.
- As @jbernardo said, if you will be inserting these into a relational database they shouldn't all just have "id" as the key, it should be "person_id".