python读取MNIST image数据

Posted 2020-11-11 jude_python

Lecun Mnist数据集下载

import numpy as np
import struct

def loadImageSet(which=0):
    print "load image set"
    binfile=None
    if which==0:
        binfile = open("..//dataset//train-images-idx3-ubyte", ‘rb‘)
    else:
        binfile=  open("..//dataset//t10k-images-idx3-ubyte", ‘rb‘)
    buffers = binfile.read()

    head = struct.unpack_from(‘>IIII‘ , buffers ,0)
    print "head,",head

    offset=struct.calcsize(‘>IIII‘)
    imgNum=head[1]
    width=head[2]
    height=head[3]
    #[60000]*28*28
    bits=imgNum*width*height
    bitsString=‘>‘+str(bits)+‘B‘ #like ‘>47040000B‘

    imgs=struct.unpack_from(bitsString,buffers,offset)

    binfile.close()
    imgs=np.reshape(imgs,[imgNum,width,height])
    print "load imgs finished"
    return imgs

def loadLabelSet(which=0):
    print "load label set"
    binfile=None
    if which==0:
        binfile = open("..//dataset//train-labels-idx1-ubyte", ‘rb‘)
    else:
        binfile=  open("..//dataset//t10k-labels-idx1-ubyte", ‘rb‘)
    buffers = binfile.read()

    head = struct.unpack_from(‘>II‘ , buffers ,0)
    print "head,",head
    imgNum=head[1]

    offset = struct.calcsize(‘>II‘)
    numString=‘>‘+str(imgNum)+"B"
    labels= struct.unpack_from(numString , buffers , offset)
    binfile.close()
    labels=np.reshape(labels,[imgNum,1])

    #print labels
    print ‘load label finished‘
    return labels

if __name__=="__main__":
    imgs=loadImageSet()
    #import PlotUtil as pu
    #pu.showImgMatrix(imgs[0])
    loadLabelSet()

及方便训练的reader

import numpy as np
import struct
import gzip
import cPickle

class MnistReader():

    def __init__(self,mnist_path,data_dim=1,one_hot=True):
        ‘‘‘
        mnist_path: the path of mnist.pkl.gz
        data_dim=1 [N,784]
        data_dim=3 [N,28,28,1]
        one_hot: one hot encoding(like: [0,1,0,0,0,0,0,0,0,0]) if true
        ‘‘‘
        self.mnist_path=mnist_path
        self.data_dim=data_dim
        self.one_hot=one_hot
        self.load_minist(mnist_path)

        self.train_datalabel=zip(self.train_x,self.train_y)
        self.valid_datalabel=zip(self.valid_x,self.valid_y)

        self.batch_offset_train=0

    def next_batch_train(self,batch_size):
        ‘‘‘
        return list of images with shape [N,784] or [N,28,28,1] dependents on self.data_dim
               and list of labels with shape [N] or [N,10] dependents on self.one_hot
        ‘‘‘
        if self.batch_offset_train<len(self.train_datalabel)//batch_size:
            imgs=list();labels=list()
            for d,l in self.train_datalabel[self.batch_offset_train:self.batch_offset_train+batch_size]:
                if self.data_dim==3:
                    d=np.reshape(d, [28,28,1])
                imgs.append(d)
                if self.one_hot:
                    a=np.zeros(10)
                    a[l]=1
                    labels.append(l)
                else:
                    labels.append(l)
            self.batch_offset_train+=1
            return imgs,labels
        else:
            self.batch_offset_train=0
            np.random.shuffle(self.train_datalabel)
            return self.next_batch_train(batch_size)

    def next_batch_val(self,batch_size):
        ‘‘‘
        return list of images with shape [N,784] or [N,28,28,1] dependents on self.data_dim
               and list of labels with shape [N,1] or [N,10] dependents on self.one_hot
        ‘‘‘
        np.random.shuffle(self.valid_datalabel)
        imgs=list();labels=list()
        for d,l in self.train_datalabel[0:batch_size]:
            if self.data_dim==3:
                d=np.reshape(d, [28,28,1])
            imgs.append(d)
            if self.one_hot:
                a=np.zeros(10)
                a[l]=1
                labels.append(l)
            else:
                labels.append(l)
        return imgs,labels

    def load_minist(self,dataset):
        print "load dataset"
        f = gzip.open(dataset, ‘rb‘)
        train_set, valid_set, test_set = cPickle.load(f)
        f.close()
        self.train_x,self.train_y=train_set
        self.valid_x,self.valid_y=valid_set
        self.test_x , self.test_y=test_set
        print "train image,label shape:",self.train_x.shape,self.train_y.shape
        print "valid image,label shape:",self.valid_x.shape,self.valid_y.shape
        print "test  image,label shape:",self.test_x.shape,self.test_y.shape
        print "load dataset end"

if __name__=="__main__":
    mnist=MnistReader(‘../dataset/mnist.pkl.gz‘,data_dim=3)
    data,label=mnist.next_batch_train(batch_size=1)
    print data
    print label

第三种加载方式需要 gzip和struct

import gzip, struct

def _read(image,label):
    minist_dir = ‘your_dir/‘
    with gzip.open(minist_dir+label) as flbl:
        magic, num = struct.unpack(">II", flbl.read(8))
        label = np.fromstring(flbl.read(), dtype=np.int8)
    with gzip.open(minist_dir+image, ‘rb‘) as fimg:
        magic, num, rows, cols = struct.unpack(">IIII", fimg.read(16))
        image = np.fromstring(fimg.read(), dtype=np.uint8).reshape(len(label), rows, cols)
    return image,label

def get_data():
    train_img,train_label = _read(
            ‘train-images-idx3-ubyte.gz‘, 
            ‘train-labels-idx1-ubyte.gz‘)
    test_img,test_label = _read(
            ‘t10k-images-idx3-ubyte.gz‘, 
            ‘t10k-labels-idx1-ubyte.gz‘)
    return [train_img,train_label,test_img,test_label]