利用pytorch的载入训练npy类型数据代码
Posted Mario cai
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了利用pytorch的载入训练npy类型数据代码相关的知识，希望对你有一定的参考价值。
utils
import os
import sys
import json
import pickle
import random
import imageio
import numpy as np
import torch
from tqdm import tqdm
def train_one_epoch(model, optimizer, data_loader, device, epoch):
model.train()
loss_function = torch.nn.CrossEntropyLoss()
# from loss_function import FocalLoss1
# loss_function = FocalLoss1().cuda()
accu_loss = torch.zeros(1).to(device) # 累计损失
accu_num = torch.zeros(1).to(device) # 累计预测正确的样本数
optimizer.zero_grad()
k = 0
sample_num = 0
data_loader = tqdm(data_loader)
for step, data in enumerate(data_loader):
k = k+1
images, labels = data
sample_num += images.shape[0]
# print(images[0].shape)
# c = np.array(images[0])
# c = c.transpose((1, 2, 0))
# imageio.imwrite('data' + '%d.jpg' % (k + 1), c)
pred = model(images.to(device))
# print(images.shape)
pred_classes = torch.max(pred, dim=1)[1]
accu_num += torch.eq(pred_classes, labels.to(device)).sum()
loss = loss_function(pred, labels.to(device))
loss.backward()
accu_loss += loss.detach()
data_loader.desc = "[train epoch ] loss: :.3f, acc: :.3f".format(epoch,
accu_loss.item() / (step + 1),
accu_num.item() / sample_num)
if not torch.isfinite(loss):
print('WARNING: non-finite loss, ending training ', loss)
sys.exit(1)
optimizer.step()
optimizer.zero_grad()
return accu_loss.item() / (step + 1), accu_num.item() / sample_num
@torch.no_grad()
def evaluate(model, data_loader, device, epoch):
# from loss_function import FocalLoss1
# loss_function = FocalLoss1().cuda()
loss_function = torch.nn.CrossEntropyLoss()
model.eval()
accu_num = torch.zeros(1).to(device) # 累计预测正确的样本数
accu_loss = torch.zeros(1).to(device) # 累计损失
sample_num = 0
data_loader = tqdm(data_loader)
for step, data in enumerate(data_loader):
images, labels = data
sample_num += images.shape[0]
pred = model(images.to(device))
pred_classes = torch.max(pred, dim=1)[1]
accu_num += torch.eq(pred_classes, labels.to(device)).sum()
loss = loss_function(pred, labels.to(device))
accu_loss += loss
data_loader.desc = "[valid epoch ] loss: :.3f, acc: :.3f".format(epoch,
accu_loss.item() / (step + 1),
accu_num.item() / sample_num)
return accu_loss.item() / (step + 1), accu_num.item() / sample_numTRAIN
import os
import math
import argparse
from torch.utils.data import DataLoader
import torch
import torch.optim as optim
from torch.utils.tensorboard import SummaryWriter
from torchvision import transforms
import torch.optim.lr_scheduler as lr_scheduler
import matplotlib.pyplot as plt
from densenet_model import densenet121, load_state_dict,densenet201
# from model import densenet121
# from my_dataset import MyDataSet
import monai
# from utils import read_split_data, train_one_epoch, evaluate
from utils import train_one_epoch, evaluate
def main(args):
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
print(args)
# print('Start Tensorboard with "tensorboard --logdir=runs", view at http://localhost:6006/')
tb_writer = SummaryWriter()
if os.path.exists("./weights") is False:
os.makedirs("./weights")
#############################################################################################################################################################
# train_images_path, train_images_label, val_images_path, val_images_label = read_split_data(args.data_path)
batch_size = args.batch_size
# nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]) # number of workers
# print('Using dataloader workers every process'.format(nw))
#############################################################################################################################################################
data_transform =
"train": transforms.Compose([
# transforms.CenterCrop(224),
# transforms.RandomGrayscale(0.1),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.RandomRotation(degrees=(0, 180)),
# transforms.ToTensor(),
transforms.Normalize(mean=[0.5], std=[0.5]),
]),
"val": transforms.Compose([
# transforms.CenterCrop(224),
# transforms.RandomGrayscale(0.1),
# transforms.RandomHorizontalFlip(),
# transforms.RandomVerticalFlip(),
# transforms.RandomRotation(degrees=(0, 180)),
# transforms.ToTensor(),
transforms.Normalize(mean=[0.5], std=[0.5]),
])
#############################################################################################################################################################
from dataset import MyDataSet,valDataset
val_data_path = args.val_data_path
data_path = args.data_path
train_dataset =MyDataSet(
data=data_path,
label=data_path,
transform=data_transform["train"])
#
val_dataset = valDataset(
data=val_data_path,
label=val_data_path,
transform=data_transform["val"])
#############################################################################################################################################################
#############################################################################################################################################################
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
# num_workers=8,
pin_memory=True,
drop_last = True,
# collate_fn=train_dataset.collate_fn
)
val_loader = DataLoader(val_dataset,
batch_size=1,
shuffle=False,
# num_workers=8,
pin_memory=True,
# drop_last=True,
# ,collate_fn=train_dataset.collate_fn
)
#############################################################################################################################################################
#############################################################################################################################################################
# 如果存在预训练权重则载入
model = densenet121(num_classes=args.num_classes).to(device)
# model = densenet201(num_classes=args.num_classes,drop_rate = 0.7).to(device)
# from vit_model import vit_large_patch32_224_in21k as create_model
# model = create_model(num_classes=2, has_logits=False).to(device)
# model = monai.networks.nets.resnet34(
# spatial_dims=2,
# # in_channels=3,
# # block = 'SEBottleneck',
# # layers = (1, 1, 1, 1),
# num_classes = 2,
# # groups=32,
# # reduction=16,
# # pretrained = True,
# # progress = True,
# # stride = 1,
# # dropout_dim = 3,
# # dropout_prob=None
# )
model.cuda()
# *****************************************载预权重*************************************************************************
if args.weights != "":
if os.path.exists(args.weights):
# load_state_dict(model, args.weights)
model.load_state_dict(torch.load(args.weights, map_location=device))
else:
raise FileNotFoundError("not found weights file: ".format(args.weights))
# print(model.load_state_dict(args.weights, strict=False))
# checkpoint = torch.load(args.weights)
# model.load_state_dict(checkpoint['model'], strict=False)
# model.load_state_dict(checkpoint['state_dict'], strict=False)
# model.load_state_dict(checkpoint, strict=False)
model.to(device)
if args.freeze_layers:
for name, para in model.named_parameters():
# 除最后的全连接层外，其他权重全部冻结
if "classifier" not in name:
para.requires_grad_(False)
# ********************************拼接训练****************************# ************************************************************
#############################################################################################################################################################
pg = [p for p in model.parameters() if p.requires_grad]
# optimizer = optim.SGD(pg, lr=args.lr, momentum=0.9, weight_decay=1E-4, nesterov=True)
optimizer = torch.optim.Adam(pg, lr=0.0001, weight_decay=0.000) # L2正则
# Scheduler https://arxiv.org/pdf/1812.01187.pdf
lf = lambda x: ((1 + math.cos(x * math.pi / args.epochs)) / 2) * (1 - args.lrf) + args.lrf # cosine
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
# scheduler = CosineAnnealingLR(optimizer, T_max=32)
#############################################################################################################################################################
x_axis = []
t_loss = []
v_loss = []
t_acc = []
v_acc = []
j = 0
for epoch in range(1, args.epochs):
# train
j += 1
train_loss, train_acc = train_one_epoch(model=model,
optimizer=optimizer,
data_loader=train_loader,
device=device,
epoch=epoch)
scheduler.step()
val_loss, val_acc = evaluate(model=model,
data_loader=val_loader,
device=device,
epoch=epoch)
#############################################################################################################################################################
# ************************************************************# ************************************************************
x_axis.append(j)
t_loss.append(train_loss)
v_loss.append(val_loss)
t_acc.append(train_acc)
v_acc.append(val_acc)
try:
train_loss_lines.remove(train_loss_lines[0]) # 移除上一步曲线
val_loss_lines.remove(val_loss_lines[0])
except Exception:
pass
plt.figure(1)
train_loss_lines = plt.plot(x_axis, t_loss, 'r', lw=0.5) # lw为曲线宽度
val_loss_lines = plt.plot(x_axis, v_loss, 'b', lw=0.5)
plt.title("loss")
plt.xlabel("steps")
plt.ylabel("loss")
plt.legend(["train_loss", "val_loss"])
# plt.pause(0.1) # 图片停留0.1s
if epoch % 10 == 0:
plt.savefig('loss' + '%d.jpg' % j)
plt.figure(2)
train_loss_lines = plt.plot(x_axis, t_acc, 'r', lw=0.5) # lw为曲线宽度
val_loss_lines = plt.plot(x_axis, v_acc, 'b', lw=0.5)
plt.title("acc")
plt.xlabel("steps")
plt.ylabel("acc")
plt.legend(["train_acc", "val_acc"])
plt.savefig('acc' + '%d.jpg' % epoch)
plt.cla()
# ************************************************************# ************************************************************
tags = ["train_loss", "train_acc", "val_loss", "val_acc", "learning_rate"]
tb_writer.add_scalar(tags[0], train_loss, epoch)
tb_writer.add_scalar(tags[1], train_acc, epoch)
tb_writer.add_scalar(tags[2], val_loss, epoch)
tb_writer.add_scalar(tags[3], val_acc, epoch)
tb_writer.add_scalar(tags[4], optimizer.param_groups[0]["lr"], epoch)
#############################################################################################################################################################
if (epoch) % 200 == 0:
torch.save(model.state_dict(), "./weights/model-.pth".format(epoch))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--num_classes', type=int, default=2)
parser.add_argument('--epochs', type=int, default=1000)
parser.add_argument('--batch-size', type=int, default=64)
parser.add_argument('--lr', type=float, default=0.001)
parser.add_argument('--lrf', type=float, default=0.1)
# 数据集所在根目录
# http://download.tensorflow.org/example_images/flower_photos.tgz
parser.add_argument('--data-path', type=str,
default=r"C:\\Users\\Admin\\SIAT\\model\\caihaihua2d_NEW\\zhong")
parser.add_argument('--val-data-path', type=str,
default=r"C:\\Users\\Admin\\SIAT\\model\\caihaihua2d_NEW\\zhong")
# densenet121 官方权重下载地址
# https://download.pytorch.org/models/densenet121-a639ec97.pth
parser.add_argument('--weights', type=str, default=r'',#C:\\Users\\Admin\\SIAT\\model\\caihaihua2d_NEW\\weights\\model-800.pth
help='initial weights path')
parser.add_argument('--freeze_layers', type=bool, default=False)
parser.add_argument('--device', default='cuda:0', help='device id (i.e. 0 or 0,1 or cpu)')
opt = parser.parse_args()
main(opt)
#训练集验证集数据分布不一致，或者训练集过小，未包含验证集中所有情况DATASET
from PIL import Image
import torch
from torch.utils.data import Dataset
import csv
import torch
import imageio
import tensorflow as tf
import os
from torchvision import transforms
from medpy.io import load
# from albumentations import *
import numpy as np
from PIL import Image
from torch import nn
import torch.nn.functional as F
import random
import torchio as tio
from torchio import AFFINE, DATA
import torchio
from torchio import ScalarImage, LabelMap, Subject, SubjectsDataset, Queue
from torchio.data import UniformSampler
from torchio.transforms import (
RandomFlip,
RandomAffine,
RandomElasticDeformation,
RandomNoise,
RandomMotion,
RandomBiasField,
RescaleIntensity,
Resample,
ToCanonical,
ZNormalization,
CropOrPad,
RandomSpike,
RandomBlur,
RandomSwap,
HistogramStandardization,
OneOf,
Clamp,
Compose,
RandomGhosting,
)
from pathlib import Path
from glob import glob
from os.path import dirname, join, basename, isfile
import sys
class MyDataSet(Dataset):
"""自定义数据集"""
# def __init__(self, root,transform):
def __init__(self, data,label, transform=True):
self.data = data
self.label = label
self.images_path = torch.tensor(np.load(os.path.join(data,"train.npy")))
self.images_class = torch.tensor(np.load(os.path.join(label,"label.npy")))
self.transform = transform
def __len__(self):
return self.images_path.shape[0] # 返回数据的总个数
def __getitem__(self, index):
img = self.images_path[index, :, :] # 读取每一个npy的数据
label = self.images_class[index] # 读取每一个npy的数据
img = np.expand_dims(img, axis=0)
img = torch.Tensor(img)
img = torch.cat([img, img, img], dim=0)
###############################################################################################################3
###############################################################################################################3
label = label.type(torch.long)
if self.transform is not None:
img = self.transform(img)
return img, label # 返回数据还有标签
# def collate_fn(index):
# # 官方实现的default_collate可以参考
# print(index)
# # https://github.com/pytorch/pytorch/blob/67b7e751e6b5931a9f45274653f4f653a4e6cdf6/torch/utils/data/_utils/collate.py
# images, labels = tuple(zip(*index))
# #
# images = torch.stack(images, dim=0)
# labels = torch.as_tensor(labels)
# return images, labels
class valDataset(Dataset):
def __init__(self, data,label, transform=True):
self.data = data
self.label = label
self.images_path = torch.tensor(np.load(os.path.join(data,"val.npy")))
self.images_class = torch.tensor(np.load(os.path.join(label,"val_label.npy")))
self.transform = transform
def __len__(self):
return self.images_path.shape[0] #返回数据的总个数
def __getitem__(self, index):
img = self.images_path[index, :, :] # 读取每一个npy的数据
label = self.images_class[index] # 读取每一个npy的数据
img = np.expand_dims(img, axis=0)
img = torch.Tensor(img)
img = torch.cat([img, img, img], dim=0)
###############################################################################################################3
###############################################################################################################3
###############################################################################################################3
label = label.type(torch.long)
if self.transform is not None:
img = self.transform(img)
return img, label # 返回数据还有标签
#
# def collate_fn(batch):
# # 官方实现的default_collate可以参考
# # https://github.com/pytorch/pytorch/blob/67b7e751e6b5931a9f45274653f4f653a4e6cdf6/torch/utils/data/_utils/collate.py
# images, labels = tuple(zip(*batch))
#
# images = torch.stack(images, dim=0)
# labels = torch.as_tensor(labels)
# return images, labels
以上是关于利用pytorch的载入训练npy类型数据代码的主要内容，如果未能解决你的问题，请参考以下文章