将 json 嵌套到 csv - 通用方法
Posted
技术标签:
【中文标题】将 json 嵌套到 csv - 通用方法【英文标题】：Nested json to csv - generic approach 【发布时间】：2016-10-08 22:46:38 【问题描述】：我对 Python 非常陌生，我正在努力将嵌套的 json 文件转换为 cvs。为此，我首先加载json，然后将其转换为使用json_normalize 打印出漂亮输出的方式，然后使用pandas 包将标准化部分输出到cvs。
我的示例 json：
[
"_id":
"id": "123"
,
"device":
"browser": "Safari",
"category": "d",
"os": "Mac"
,
"exID":
"$oid": "123"
,
"extreme": false,
"geo":
"city": "London",
"country": "United Kingdom",
"countryCode": "UK",
"ip": "00.000.000.0"
,
"viewed":
"$date": "2011-02-12"
,
"attributes": [
"name": "gender",
"numeric": 0,
"value": 0
,
"name": "email",
"value": false
],
"change": [
"id":
"$id": "1231"
,
"seen": [
"$date": "2011-02-12"
]
]
,
"_id":
"id": "456"
,
"device":
"browser": "Chrome 47",
"category": "d",
"os": "Windows"
,
"exID":
"$oid": "345"
,
"extreme": false,
"geo":
"city": "Berlin",
"country": "Germany",
"countryCode": "DE",
"ip": "00.000.000.0"
,
"viewed":
"$date": "2011-05-12"
,
"attributes": [
"name": "gender",
"numeric": 1,
"value": 1
,
"name": "email",
"value": true
],
"change": [
"id":
"$id": "1231"
,
"seen": [
"$date": "2011-02-12"
]
]
]
使用以下代码（这里我排除了嵌套部分）：
import json
from pandas.io.json import json_normalize
def loading_file():
#File path
file_path = #file path here
#Loading json file
json_data = open(file_path)
data = json.load(json_data)
return data
#Storing avaliable keys
def data_keys(data):
keys =
for i in data:
for k in i.keys():
keys[k] = 1
keys = keys.keys()
#Excluding nested arrays from keys - hard coded -> IMPROVE
new_keys = [x for x in keys if
x != 'attributes' and
x != 'change']
return new_keys
#Excluding nested arrays from json dictionary
def new_data(data, keys):
new_data = []
for i in range(0, len(data)):
x = k:v for (k,v) in data[i].items() if k in keys
new_data.append(x)
return new_data
def csv_out(data):
data.to_csv('out.csv',encoding='utf-8')
def main():
data_file = loading_file()
keys = data_keys(data_file)
table = new_data(data_file, keys)
csv_out(json_normalize(table))
main()
我当前的输出如下所示：
| _id.id | device.browser | device.category | device.os | ... | viewed.$date |
|--------|----------------|-----------------|-----------|------|--------------|
| 123 | Safari | d | Mac | ... | 2011-02-12 |
| 456 | Chrome 47 | d | Windows | ... | 2011-05-12 |
| | | | | | |
我的问题是我想将嵌套数组包含到 cvs 中，所以我必须将它们展平。我不知道如何使它通用，所以我在创建表时不使用字典 keys (numeric, id, name) 和 values。我必须使其具有普遍性，因为attributes 和change 中的键数。因此，我希望有这样的输出：
| _id.id | device.browser | ... | attributes_gender_numeric | attributes_gender_value | attributes_email_value | change_id | change_seen |
|--------|----------------|-----|---------------------------|-------------------------|------------------------|-----------|-------------|
| 123 | Safari | ... | 0 | 0 | false | 1231 | 2011-02-12 |
| 456 | Chrome 47 | ... | 1 | 1 | true | 1231 | 2011-02-12 |
| | | | | | | | |
提前感谢您！非常欢迎任何关于如何改进我的代码并使其更高效的提示。
【问题讨论】：
【参考方案1】：感谢 Amir Ziai 的精彩博文，您可以找到 here 我设法以平面表格的形式输出我的数据。具有以下功能：
#Function that recursively extracts values out of the object into a flattened dictionary
def flatten_json(data):
flat = [] #list of flat dictionaries
def flatten(y):
out =
def flatten2(x, name=''):
if type(x) is dict:
for a in x:
if a == "name":
flatten2(x["value"], name + x[a] + '_')
else:
flatten2(x[a], name + a + '_')
elif type(x) is list:
for a in x:
flatten2(a, name + '_')
else:
out[name[:-1]] = x
flatten2(y)
return out
#Loop needed to flatten multiple objects
for i in range(len(data)):
flat.append(flatten(data[i]).copy())
return json_normalize(flat)
我知道由于名称-值 if 语句，它不是完全可推广的。但是，如果删除了创建名称-值字典的豁免，则该代码可以与其他嵌入式数组一起使用。
【讨论】：
【参考方案2】：几周前，我有一项任务是将带有嵌套键和值的 json 转换为 csv 文件。对于此任务，必须正确处理嵌套键以连接要用作值的唯一标头。结果是下面的代码，也可以找到here。
def get_flat_json(json_data, header_string, header, row):
"""Parse json files with nested key-vales into flat lists using nested column labeling"""
for root_key, root_value in json_data.items():
if isinstance(root_value, dict):
get_flat_json(root_value, header_string + '_' + str(root_key), header, row)
elif isinstance(root_value, list):
for value_index in range(len(root_value)):
for nested_key, nested_value in root_value[value_index].items():
header[0].append((header_string +
'_' + str(root_key) +
'_' + str(nested_key) +
'_' + str(value_index)).strip('_'))
if nested_value is None:
nested_value = ''
row[0].append(str(nested_value))
else:
if root_value is None:
root_value = ''
header[0].append((header_string + '_' + str(root_key)).strip('_'))
row[0].append(root_value)
return header, row
这是一种基于经济学家对此问题的回答的更通用的方法。
【讨论】：
【参考方案3】：以下代码处理了一个杂乱无章的 json 文件 字典和列表相互之间 7 层深：
import csv, json, os
def parse_json(data):
a_dict_accum =
for key, val in data.items():
print("key, val = ", key, val)
output.writerow([key])
output.writerow([val])
if isinstance(val, dict):
for a_key, a_val in val.items():
print("a_key, a_val = ", a_key, a_val)
output.writerow([a_key])
output.writerow([a_val])
a_dict_accum.update(a_key:a_val)
print("a_dict_accum = ", a_dict_accum)
parse_json(a_dict_accum)
elif isinstance(val, list):
print("val_list = ", val)
for a_list in val:
print("a_list = ", a_list)
output.writerow([a_list])
if isinstance(a_list, dict):
for a_key, a_val in a_list.items():
print("a_key, a_val = ", a_key, a_val)
output.writerow([a_key])
output.writerow([a_val])
a_dict_accum.update(a_key:a_val)
print("a_dict_accum = ", a_dict_accum)
parse_json(a_dict_accum)
os.chdir('C://Users/Robert/viirs/20200217_output')
fileInput = 'night_lights_points.json'
fileOutput = 'night_lights_points.csv'
inputFile = open(fileInput) #open json file
outputFile = open(fileOutput, 'w', newline='') #load csv file
data = json.load(inputFile) #load json content
output = csv.writer(outputFile) #create a csv.writer
output = parse_json(data)
inputFile.close() #close the input file
outputFile.close() #close the output file
【讨论】：
以上是关于将 json 嵌套到 csv - 通用方法的主要内容，如果未能解决你的问题，请参考以下文章
将具有多个嵌套级别的任何 XML 读取到结构化表中以写入 Excel 的通用方法