Python数据编码与处理

Posted 2022-11-28 HT . WANG

1.读写CSV数据

CSV数据：

Symbol,Price,Date,Time,Change,Volume
"AA",39.48,"6/11/2007","9:36am",-0.18,181800
"AIG",71.38,"6/11/2007","9:36am",-0.15,195500
"AXP",62.58,"6/11/2007","9:36am",-0.46,935000
"BA",98.31,"6/11/2007","9:36am",+0.12,104800
"C",53.08,"6/11/2007","9:36am",-0.25,360900
"CAT",78.29,"6/11/2007","9:36am",-0.23,225400

（1）CSV数据读取

from collections import namedtuple
with open('stock.csv') as f:
    f_csv = csv.reader(f)
    headings = next(f_csv)
    Row = namedtuple('Row', headings) #读取到命名元组中存储
    for r in f_csv:
        row = Row(*r)
        # Process row
        ...

import csv
with open('stocks.csv') as f:
    f_csv = csv.DictReader(f) #读取到字典中存储
    for row in f_csv:
        # process row
        ...

（2）CSV数据写入

headers = ['Symbol','Price','Date','Time','Change','Volume']
rows = [('AA', 39.48, '6/11/2007', '9:36am', -0.18, 181800),
         ('AIG', 71.38, '6/11/2007', '9:36am', -0.15, 195500),
         ('AXP', 62.58, '6/11/2007', '9:36am', -0.46, 935000),
       ]

with open('stocks.csv','w') as f:
    f_csv = csv.writer(f) #创建一个 writer 对象
    f_csv.writerow(headers)
    f_csv.writerows(rows)



headers = ['Symbol', 'Price', 'Date', 'Time', 'Change', 'Volume']
rows = ['Symbol':'AA', 'Price':39.48, 'Date':'6/11/2007',
        'Time':'9:36am', 'Change':-0.18, 'Volume':181800,
        'Symbol':'AIG', 'Price': 71.38, 'Date':'6/11/2007',
        'Time':'9:36am', 'Change':-0.15, 'Volume': 195500,
        'Symbol':'AXP', 'Price': 62.58, 'Date':'6/11/2007',
        'Time':'9:36am', 'Change':-0.46, 'Volume': 935000,
        ]

with open('stocks.csv','w') as f:
    f_csv = csv.DictWriter(f, headers) #创建一个 Dictwriter 对象
    f_csv.writeheader()
    f_csv.writerows(rows)

注意：csv产生的数据都是字符串类型的，如果需要做这样的类型转换，必须手动去实现

2.读写JSON数据

json 模块提供了一种很简单的方式来编码和解码JSON数据。 其中两个主要的函数是 json.dumps() 和 json.loads()

import json

data = 
    'name' : 'ACME',
    'shares' : 100,
    'price' : 542.23


json_str = json.dumps(data) #编码为json编码的字符串

data = json.loads(json_str) #将json编码的字符串解码

如果要处理的是文件而不是字符串，也可以使用 json.dump() 和 json.load() 来编码和解码JSON数据。

# Writing JSON data
with open('data.json', 'w') as f:
    json.dump(data, f)

# Reading data back
with open('data.json', 'r') as f:
    data = json.load(f)

3.解析简单的XML数据

可以使用 xml.etree.ElementTree 模块从简单的XML文档中提取数据

from urllib.request import urlopen
from xml.etree.ElementTree import parse

# Download the RSS feed and parse it
u = urlopen('http://planet.python.org/rss20.xml')
doc = parse(u)

# Extract and output tags of interest
for item in doc.iterfind('channel/item'):
    title = item.findtext('title')
    date = item.findtext('pubDate')
    link = item.findtext('link')

    print(title)
    print(date)
    print(link)
    print()

xml.etree.ElementTree.parse() 函数解析整个XML文档并将其转换成一个文档对象。 然后，你就能使用 find() 、iterfind() 和 findtext() 等方法来搜索特定的XML元素了。 这些函数的参数就是某个指定的标签名，例如 channel/item 或 title 。

4.增量式解析大型XML文件

采用迭代器和生成器

from xml.etree.ElementTree import iterparse

def parse_and_remove(filename, path):
    path_parts = path.split('/')
    doc = iterparse(filename, ('start', 'end'))
    # Skip the root element
    next(doc)

    tag_stack = []
    elem_stack = []
    for event, elem in doc:
        if event == 'start':
            tag_stack.append(elem.tag)
            elem_stack.append(elem)
        elif event == 'end':
            if tag_stack == path_parts:
                yield elem
                elem_stack[-2].remove(elem)
            try:
                tag_stack.pop()
                elem_stack.pop()
            except IndexError:
                pass

5.将字典转换为XML

使用一个Python字典存储数据，并将它转换成XML格式

from xml.etree.ElementTree import Element
from xml.etree.ElementTree import tostring

def dict_to_xml(tag, d):
elem = Element(tag)
for key, val in d.items():
    child = Element(key)
    child.text = str(val)
    elem.append(child)
return elem


>>> s =  'name': 'GOOG', 'shares': 100, 'price':490.1 
>>> e = dict_to_xml('stock', s)
>>> e
<Element 'stock' at 0x1004b64c8>
>>>

>>> tostring(e) #转换为字符串更直观显示xml格式
b'<stock><price>490.1</price><shares>100</shares><name>GOOG</name></stock>'
>>>

>>> e.set('_id','1234') #给某个元素添加属性值
>>> tostring(e)
b'<stock _id="1234"><price>490.1</price><shares>100</shares><name>GOOG</name>
</stock>'
>>>

注意：当字典的值中包含一些特殊字符，需要手动去转换这些字符， 可以使用 xml.sax.saxutils 中的 escape() 和 unescape() 函数

>>> d =  'name' : '<spam>' 

>>> e = dict_to_xml('item',d)
>>> tostring(e)
b'<item><name>&lt;spam&gt;</name></item>'
>>>
# 字符 ‘<’ 和 ‘>’ 被替换成了 &lt; 和 &gt;



>>> from xml.sax.saxutils import escape, unescape
>>> escape('<spam>')
'&lt;spam&gt;'
>>> unescape(_) #手动转换
'<spam>'
>>>

6.解析和修改XML

>>> from xml.etree.ElementTree import parse, Element
>>> doc = parse('pred.xml') #解析xml
>>> root = doc.getroot() #定位到root tag标签
>>> root
<Element 'stop' at 0x100770cb0>

>>> # 按照标签 删除元素
>>> root.remove(root.find('sri'))
>>> root.remove(root.find('cr'))

>>> # 定位到nm标签处
>>> root.getchildren().index(root.find('nm'))
1
>>> e = Element('spam')
>>> e.text = 'This is a test'
>>> root.insert(2, e) #插入新标签 其tag为spam 内容为'This is a test'

>>> # 修改后内容重新写回xml文件
>>> doc.write('newpred.xml', xml_declaration=True)
>>>

7.与关系型数据库的交互

在关系型数据库中查询、增加或删除记录

>>> import sqlite3
>>> db = sqlite3.connect('database.db') #执行 connect() 函数， 给它提供一些数据库名、主机、用户名、密码和其他必要的一些参数
>>>

>>> c = db.cursor() #创建一个操作句柄
>>> c.execute('create table portfolio (symbol text, shares integer, price real)') #建表
<sqlite3.Cursor object at 0x10067a730>
>>> db.commit()
>>>

>>> c.executemany('insert into portfolio values (?,?,?)', stocks) #插入多条数据
<sqlite3.Cursor object at 0x10067a730>
>>> db.commit()
>>>

>>> for row in db.execute('select * from portfolio'): #查询
...     print(row)
...
('GOOG', 100, 490.1)
('AAPL', 50, 545.75)
('FB', 150, 7.45)
('HPQ', 75, 33.2)
>>>

8.编码和解码十六进制数

将一个十六进制字符串解码成一个字节字符串或者将一个字节字符串编码成一个十六进制字符串。

>>> # Initial byte string
>>> s = b'hello'
>>> # Encode as hex
>>> import binascii
>>> h = binascii.b2a_hex(s) #将一个字节字符串编码成一个十六进制字符串。
>>> h
b'68656c6c6f'
>>> # Decode back to bytes
>>> binascii.a2b_hex(h) #将一个十六进制字符串解码成一个字节字符串
b'hello'

9.编码解码Base64数据

Base64编码仅仅用于面向字节的数据比如字节字符串和字节数组。 此外，编码处理的输出结果总是一个字节字符串

>>> # Some byte data
>>> s = b'hello'
>>> import base64

>>> # Encode as Base64
>>> a = base64.b64encode(s) #将字节字符串进行base64格式编码
>>> a
b'aGVsbG8='

>>> # Decode from Base64
>>> base64.b64decode(a) #将base64格式解码为字节字符串
b'hello'
>>>

10.读写二进制数组数据

读写一个二进制数组的结构化数据到Python元组中

（1）将一个Python元组列表写入一个二进制文件，并使用 struct 将每个元组编码为一个结构体。

from struct import Struct
def write_records(records, format, f):
    '''
    Write a sequence of tuples to a binary file of structures.
    '''
    record_struct = Struct(format)
    for r in records:
        f.write(record_struct.pack(*r))

# Example
if __name__ == '__main__':
    records = [ (1, 2.3, 4.5),
                (6, 7.8, 9.0),
                (12, 13.4, 56.7) ]
    with open('data.b', 'wb') as f:
        write_records(records, '<idd', f)

（2）通过numpy模块 将一个二进制数据读取到一个结构化数组中而不是一个元组列表中

>>> import numpy as np
>>> f = open('data.b', 'rb')
>>> records = np.fromfile(f, dtype='<i,<d,<d')
>>> records
array([(1, 2.3, 4.5), (6, 7.8, 9.0), (12, 13.4, 56.7)],
dtype=[('f0', '<i4'), ('f1', '<f8'), ('f2', '<f8')])
>>> records[0]
(1, 2.3, 4.5)
>>> records[1]
(6, 7.8, 9.0)
>>>