如何使用我自己的数据在 PyTorch 上测试这个卷积神经网络？
Posted
技术标签:
【中文标题】如何使用我自己的数据在 PyTorch 上测试这个卷积神经网络？【英文标题】：How can I use my own data to test this Convolutional Neural Network on PyTorch? 【发布时间】：2020-09-03 09:21:15 【问题描述】：所以最近我一直在关注来自 sentdex 的关于卷积神经网络的教程，并且我一直在尝试实现他的代码，以使用我自己的图像测试经过训练的神经网络（在这种情况下，我只是从使用的数据集中随机挑选图片在他的程序中）。所以我的目的是训练神经网络，对其进行测试并最终保存它，以便稍后将其加载到单独的 python 文件中，以便在单个图像上使用已经训练好的 NN。
他使用的数据集是“来自微软的狗 vs 猫”。这是我编写神经网络程序的代码（“main.py”）。
import cv2
import numpy as np
from tqdm import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
REBUILD_DATA = False # set to true to one once, then back to false unless you want to change something in your training data.
class DogsVSCats():
IMG_SIZE = 100
CATS = "PetImages/Cat"
DOGS = "PetImages/Dog"
TESTING = "PetImages/Testing"
LABELS = CATS: 0, DOGS: 1
training_data = []
catcount = 0
dogcount = 0
def make_training_data(self):
for label in self.LABELS:
print(label)
for f in tqdm(os.listdir(label)):
if "jpg" in f:
try:
path = os.path.join(label, f)
img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
img = cv2.resize(img, (self.IMG_SIZE, self.IMG_SIZE))
self.training_data.append([np.array(img), np.eye(2)[self.LABELS[label]]]) # do something like print(np.eye(2)[1]), just makes one_hot
#print(np.eye(2)[self.LABELS[label]])
if label == self.CATS:
self.catcount += 1
elif label == self.DOGS:
self.dogcount += 1
except Exception as e:
pass
#print(label, f, str(e))
np.random.shuffle(self.training_data)
np.save("training_data.npy", self.training_data)
print('Cats:',dogsvcats.catcount)
print('Dogs:',dogsvcats.dogcount)
class Net(nn.Module):
def __init__(self):
super().__init__() # just run the init of parent class (nn.Module)
self.conv1 = nn.Conv2d(1, 32, 5) # input is 1 image, 32 output channels, 5x5 kernel / window
self.conv2 = nn.Conv2d(32, 64, 5) # input is 32, bc the first layer output 32. Then we say the output will be 64 channels, 5x5 kernel / window
self.conv3 = nn.Conv2d(64, 128, 5)
x = torch.randn(50, 50).view(-1, 1, 50, 50)
self._to_linear = None
self.convs(x)
self.fc1 = nn.Linear(self._to_linear, 512) #flattening.
self.fc2 = nn.Linear(512, 2) # 512 in, 2 out bc we're doing 2 classes (dog vs cat).
def convs(self, x):
# max pooling over 2x2
x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
x = F.max_pool2d(F.relu(self.conv2(x)), (2, 2))
x = F.max_pool2d(F.relu(self.conv3(x)), (2, 2))
if self._to_linear is None:
self._to_linear = x[0].shape[0]*x[0].shape[1]*x[0].shape[2]
return x
def forward(self, x):
x = self.convs(x)
x = x.view(-1, self._to_linear) # .view is reshape ... this flattens X before
x = F.relu(self.fc1(x))
x = self.fc2(x) # bc this is our output layer. No activation here.
return F.softmax(x, dim=1)
net = Net()
print(net)
if REBUILD_DATA:
dogsvcats = DogsVSCats()
dogsvcats.make_training_data()
training_data = np.load("training_data.npy", allow_pickle=True)
print(len(training_data))
optimizer = optim.Adam(net.parameters(), lr=0.001)
loss_function = nn.MSELoss()
X = torch.Tensor([i[0] for i in training_data]).view(-1,50,50)
X = X/255.0
y = torch.Tensor([i[1] for i in training_data])
VAL_PCT = 0.1 # lets reserve 10% of our data for validation
val_size = int(len(X)*VAL_PCT)
train_X = X[:-val_size]
train_y = y[:-val_size]
test_X = X[-val_size:]
test_y = y[-val_size:]
BATCH_SIZE = 100
EPOCHS = 1
def train(net):
for epoch in range(EPOCHS):
for i in tqdm(range(0, len(train_X), BATCH_SIZE)): # from 0, to the len of x, stepping BATCH_SIZE at a time. [:50] ..for now just to dev
#print(f"i:i+BATCH_SIZE")
batch_X = train_X[i:i+BATCH_SIZE].view(-1, 1, 50, 50)
batch_y = train_y[i:i+BATCH_SIZE]
net.zero_grad()
outputs = net(batch_X)
loss = loss_function(outputs, batch_y)
loss.backward()
optimizer.step() # Does the update
print(f"Epoch: epoch. Loss: loss")
def test(net):
correct = 0
total = 0
with torch.no_grad():
for i in tqdm(range(len(test_X))):
real_class = torch.argmax(test_y[i])
net_out = net(test_X[i].view(-1, 1, 50, 50))[0] # returns a list,
predicted_class = torch.argmax(net_out)
if predicted_class == real_class:
correct += 1
total += 1
print("Accuracy: ", round(correct/total, 3))
train(net)
test(net)
PATH = './object_detection.pth'
torch.save(net.state_dict(), PATH)
训练完神经网络后，我想将它加载到下一个程序中，并简单地测试 NN 上的图像。然而，每次我运行这个程序时，神经网络都会被再次训练和测试，这使得这个过程变得更长而且更烦人。而且，我认为当我运行这个程序然后将图像输入到神经网络时，整个“main.py”正在运行。
请问，有人可以帮我解决这个问题吗？这将是惊人的，因为我用它作为我的学士论文的基础。可能我还想修改此代码以通过它运行我自己的整个数据集，如果有人能帮助我这样做，那将是不可思议的，因为我是 pytorch 的新手。
import cv2
from main import Net, train, test
import numpy as np
classes = ('cat', 'dog')
imsize = 50
net = Net()
net.load_state_dict(torch.load('./object_detection.pth'))
def image_loader(image_name):
image = cv2.imread(image_name, cv2.IMREAD_GRAYSCALE)
image = cv2.resize(image, (imsize, imsize))
image = np.array(image)
image = torch.Tensor(image)/255
image = image.view(-1, 1, 50, 50)
return image
test_image = image_loader("./PetImages/Cat/1021.jpg")
result = net(test_image)
_, predicted = torch.max(result, 1)
print(result)
print(classes[predicted[0]])
【问题讨论】：
pytorch.org/tutorials/beginner/saving_loading_models.html 你需要保存你的模型。当您想在整个数据集上“评估”您的模型时，您只需使用新数据集加载模型并进行评估。 【参考方案1】：您面临的问题与 NN 无关，而是与导入部分有关。
在第二个代码 sn-p 中，您导入第一个代码 sn-p 的类和函数。同时语句也会执行里面的所有代码，这不是我们想要的。
解决它的最简单方法是将代码收集在 if 案例中，以避免在导入期间执行。
结果可能如下所示：
import cv2
import numpy as np
from tqdm import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
class DogsVSCats():
IMG_SIZE = 100
CATS = "PetImages/Cat"
DOGS = "PetImages/Dog"
TESTING = "PetImages/Testing"
LABELS = CATS: 0, DOGS: 1
training_data = []
catcount = 0
dogcount = 0
def make_training_data(self):
for label in self.LABELS:
print(label)
for f in tqdm(os.listdir(label)):
if "jpg" in f:
try:
path = os.path.join(label, f)
img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
img = cv2.resize(img, (self.IMG_SIZE, self.IMG_SIZE))
self.training_data.append([np.array(img), np.eye(2)[self.LABELS[label]]]) # do something like print(np.eye(2)[1]), just makes one_hot
#print(np.eye(2)[self.LABELS[label]])
if label == self.CATS:
self.catcount += 1
elif label == self.DOGS:
self.dogcount += 1
except Exception as e:
pass
#print(label, f, str(e))
np.random.shuffle(self.training_data)
np.save("training_data.npy", self.training_data)
print('Cats:',dogsvcats.catcount)
print('Dogs:',dogsvcats.dogcount)
class Net(nn.Module):
def __init__(self):
super().__init__() # just run the init of parent class (nn.Module)
self.conv1 = nn.Conv2d(1, 32, 5) # input is 1 image, 32 output channels, 5x5 kernel / window
self.conv2 = nn.Conv2d(32, 64, 5) # input is 32, bc the first layer output 32. Then we say the output will be 64 channels, 5x5 kernel / window
self.conv3 = nn.Conv2d(64, 128, 5)
x = torch.randn(50, 50).view(-1, 1, 50, 50)
self._to_linear = None
self.convs(x)
self.fc1 = nn.Linear(self._to_linear, 512) #flattening.
self.fc2 = nn.Linear(512, 2) # 512 in, 2 out bc we're doing 2 classes (dog vs cat).
def convs(self, x):
# max pooling over 2x2
x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
x = F.max_pool2d(F.relu(self.conv2(x)), (2, 2))
x = F.max_pool2d(F.relu(self.conv3(x)), (2, 2))
if self._to_linear is None:
self._to_linear = x[0].shape[0]*x[0].shape[1]*x[0].shape[2]
return x
def forward(self, x):
x = self.convs(x)
x = x.view(-1, self._to_linear) # .view is reshape ... this flattens X before
x = F.relu(self.fc1(x))
x = self.fc2(x) # bc this is our output layer. No activation here.
return F.softmax(x, dim=1)
def train(net):
for epoch in range(EPOCHS):
for i in tqdm(range(0, len(train_X), BATCH_SIZE)): # from 0, to the len of x, stepping BATCH_SIZE at a time. [:50] ..for now just to dev
#print(f"i:i+BATCH_SIZE")
batch_X = train_X[i:i+BATCH_SIZE].view(-1, 1, 50, 50)
batch_y = train_y[i:i+BATCH_SIZE]
net.zero_grad()
outputs = net(batch_X)
loss = loss_function(outputs, batch_y)
loss.backward()
optimizer.step() # Does the update
print(f"Epoch: epoch. Loss: loss")
def test(net):
correct = 0
total = 0
with torch.no_grad():
for i in tqdm(range(len(test_X))):
real_class = torch.argmax(test_y[i])
net_out = net(test_X[i].view(-1, 1, 50, 50))[0] # returns a list,
predicted_class = torch.argmax(net_out)
if predicted_class == real_class:
correct += 1
total += 1
print("Accuracy: ", round(correct/total, 3))
if __name__ == "__main__":
REBUILD_DATA = False # set to true to one once, then back to false unless you want to change something in your training data.
net = Net()
print(net)
if REBUILD_DATA:
dogsvcats = DogsVSCats()
dogsvcats.make_training_data()
training_data = np.load("training_data.npy", allow_pickle=True)
print(len(training_data))
optimizer = optim.Adam(net.parameters(), lr=0.001)
loss_function = nn.MSELoss()
X = torch.Tensor([i[0] for i in training_data]).view(-1,50,50)
X = X/255.0
y = torch.Tensor([i[1] for i in training_data])
VAL_PCT = 0.1 # lets reserve 10% of our data for validation
val_size = int(len(X)*VAL_PCT)
train_X = X[:-val_size]
train_y = y[:-val_size]
test_X = X[-val_size:]
test_y = y[-val_size:]
BATCH_SIZE = 100
EPOCHS = 1
train(net)
test(net)
PATH = './object_detection.pth'
torch.save(net.state_dict(), PATH)
您可以查看官方文档的更多信息：import 和 main。
【讨论】：
非常感谢您的帮助！这很有效，但是我还发现了一种更“更干净”的方法来解决这个问题，方法是使用神经网络类创建另一个 python 文件。这样我就可以只导入这个类，这样其余的就不会被导入和执行。【参考方案2】：您可以将模型保存为 pickle 文件，然后使用 torch.save 和 torch.load 加载它以用于另一个程序。所以在你的情况下，当你看到损失下降时，你可以打电话
torch.save(net.state_dict(), <save_path>) # to save
net.load_state_dict(torch.load(<save_path>)) # to load again
您需要在 train 函数中跟踪最小损失
【讨论】：
以上是关于如何使用我自己的数据在 PyTorch 上测试这个卷积神经网络？的主要内容，如果未能解决你的问题，请参考以下文章