Python重写jupyter notebook到python脚本问题

Posted 2023-02-23

【中文标题】Python重写jupyter notebook到python脚本问题

【英文标题】：Python re-write jupyter notebook to python script problem

【发布时间】：2021-05-03 22:22:28

【问题描述】：

我有一个非常简单的 RNN 模型，我可以在 Jupyter 笔记本上毫无问题地执行，现在我想将此模型嵌入到我的 Django 项目中，所以我下载了 .ipynb 文件，然后将其保存为 .py 文件，然后我就把它放在我的 Django 项目中。

但是，第一个问题是 VSCode 说我有 unresolved import 'MLutils，MLutils 是我在同一文件夹中的实用程序文件。

第二个问题是如果我只运行这个文件，我会得到这个错误RuntimeError: cannot perform reduction function argmax on a tensor with no elements because the operation does not have an identity 但是如果我使用 VSCode 的 Run Cell Above 按钮，我会得到正确的结果。 我要运行的代码是这样的，叫做MLTrain.py

#!/usr/bin/env python
# coding: utf-8

# In[1]:


import torch
import torch.nn as nn
import matplotlib.pyplot as plt 

from MLutils import ALL_LETTERS, N_LETTERS
from MLutils import load_data, letter_to_tensor, line_to_tensor, random_training_example


class RNN(nn.Module):
    # implement RNN from scratch rather than using nn.RNN
    # # number of possible letters, hidden size, categories number
    def __init__(self, input_size, hidden_size, output_size):
        super(RNN, self).__init__()
        
        self.hidden_size = hidden_size
        #Define 2 different liner layers
        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)
        self.i2o = nn.Linear(input_size + hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)
        
    def forward(self, input_tensor, hidden_tensor):
        # combine input and hidden tensor
        combined = torch.cat((input_tensor, hidden_tensor), 1)
        # apply the Linear layers and the softmax
        hidden = self.i2h(combined)
        output = self.i2o(combined)
        output = self.softmax(output)
        # return 2 different tensors
        return output, hidden
    
    # need some initial hidden states in the begining.
    def init_hidden(self):
        return torch.zeros(1, self.hidden_size)
# dictionary with the country as the key and names as values
category_lines, all_categories = load_data()
# number of categories
n_categories = len(all_categories)

# a Hyperparameter 
n_hidden = 128
# number of possible letters, hidden size, output size
rnn = RNN(N_LETTERS, n_hidden, n_categories)

# one step
input_tensor = letter_to_tensor('A')
hidden_tensor = rnn.init_hidden()

output, next_hidden = rnn(input_tensor, hidden_tensor)
#print(output.size())
#>>> size: [1,18]
#print(next_hidden.size())
#>>> size: [1,128]
# whole sequence/name
input_tensor = line_to_tensor('if')
hidden_tensor = rnn.init_hidden()

output, next_hidden = rnn(input_tensor[0], hidden_tensor)
print(output.size())
print(next_hidden.size())




# apply softmax in the end.
# this is the likelyhood of each character of each category
def category_from_output(output):
    # return index of the greatest value
    category_idx = torch.argmax(output).item()
    return all_categories[category_idx]
print(category_from_output(output))




criterion = nn.NLLLoss()
# hyperparameter
learning_rate = 0.005
optimizer = torch.optim.SGD(rnn.parameters(), lr=learning_rate)
# whole name as tensor,
def train(line_tensor, category_tensor):
    hidden = rnn.init_hidden()
    #line_tensor.size()[0]: the length of the name
    for i in range(line_tensor.size()[0]):
        # apply the current character and the previous hidden state.
        output, hidden = rnn(line_tensor[i], hidden)
        
    loss = criterion(output, category_tensor)
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    return output, loss.item()

current_loss = 0
all_losses = []
plot_steps, print_steps = 100, 500
n_iters = 2000
for i in range(n_iters):
    category, line, category_tensor, line_tensor = random_training_example(category_lines, all_categories)
    
    output, loss = train(line_tensor, category_tensor)
    current_loss += loss 
    
    if (i+1) % plot_steps == 0:
        all_losses.append(current_loss / plot_steps)
        current_loss = 0
        
    if (i+1) % print_steps == 0:
        guess = category_from_output(output)
        correct = "CORRECT" if guess == category else f"WRONG (category)"
        print(f"i+1 (i+1)/n_iters*100 loss:.4f line / guess correct")
        
    
plt.figure()
plt.plot(all_losses)
plt.show()
# model can be saved

def predict(input_line):
    print(f"\n> input_line")
    with torch.no_grad():
        line_tensor = line_to_tensor(input_line)
        
        hidden = rnn.init_hidden()
    
        for i in range(line_tensor.size()[0]):
            output, hidden = rnn(line_tensor[i], hidden)
        
        guess = category_from_output(output)
        print(guess)




if __name__ == "__main__":
    predict("abcde 1 ifelse")

# In[ ]:

MLutils.py是这个

import io
import os
import unicodedata
import string
import glob

import torch
import random

# alphabet small + capital letters + " .,;'"
ALL_LETTERS = string.ascii_letters + " .,;'"
N_LETTERS = len(ALL_LETTERS)

# Turn a Unicode string to plain ASCII, thanks to https://***.com/a/518232/2809427
def unicode_to_ascii(s):
    return ''.join(
        c for c in unicodedata.normalize('NFD', s)
        if unicodedata.category(c) != 'Mn'
        and c in ALL_LETTERS
    )

def load_data():
    # Build the category_lines dictionary, a list of names per language
    category_lines = 
    all_categories = []
    
    def find_files(path):
        return glob.glob(path)
    
    # Read a file and split into lines
    def read_lines(filename):
        lines = io.open(filename, encoding='utf-8').read().strip().split('\n')
        return [unicode_to_ascii(line) for line in lines]
    
    for filename in find_files('data/categories/*.txt'):
        category = os.path.splitext(os.path.basename(filename))[0]
        all_categories.append(category)
        
        lines = read_lines(filename)
        category_lines[category] = lines
        
    return category_lines, all_categories



# Find letter index from all_letters, e.g. "a" = 0
def letter_to_index(letter):
    return ALL_LETTERS.find(letter)

# Just for demonstration, turn a letter into a <1 x n_letters> Tensor
def letter_to_tensor(letter):
    tensor = torch.zeros(1, N_LETTERS)
    tensor[0][letter_to_index(letter)] = 1
    return tensor

# Turn a line into a <line_length x 1 x n_letters>,
# or an array of one-hot letter vectors
def line_to_tensor(line):
    tensor = torch.zeros(len(line), 1, N_LETTERS)
    for i, letter in enumerate(line):
        tensor[i][0][letter_to_index(letter)] = 1
    return tensor


def random_training_example(category_lines, all_categories):
    
    def random_choice(a):
        random_idx = random.randint(0, len(a) - 1)
        return a[random_idx]
    
    category = random_choice(all_categories)
    line = random_choice(category_lines[category])
    category_tensor = torch.tensor([all_categories.index(category)], dtype=torch.long)
    line_tensor = line_to_tensor(line)
    return category, line, category_tensor, line_tensor



if __name__ == '__main__':
    print(ALL_LETTERS)
    print(unicode_to_ascii('Ślusàrski'))
    
    # category_lines, all_categories = load_data()
    # print(category_lines['Flow'][:5])
    
    # print(letter_to_tensor('J')) # [1, 57]
    # print(line_to_tensor('Jones').size()) # [5, 1, 57]

Run Above按钮是这个

如果我在这里注释掉打印，我会得到这个错误 错误全文

Traceback (most recent call last):
  File "c:\Users\Leslie\Desktop\todo_drf\MLModel\MLTrain.py", line 106, in <module>
    category, line, category_tensor, line_tensor = random_training_example(category_lines, all_categories)
  File "c:\Users\Leslie\Desktop\todo_drf\MLModel\MLutils.py", line 84, in random_training_example
    category = random_choice(all_categories)
  File "c:\Users\Leslie\Desktop\todo_drf\MLModel\MLutils.py", line 81, in random_choice
    random_idx = random.randint(0, len(a) - 1)
  File "C:\Users\Leslie\AppData\Local\Programs\Python\Python39\lib\random.py", line 338, in randint
    return self.randrange(a, b+1)
  File "C:\Users\Leslie\AppData\Local\Programs\Python\Python39\lib\random.py", line 316, in randrange
    raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width))
ValueError: empty range for randrange() (0, 0, 0)

这真的是有线的，或者我在某个地方真的很愚蠢。 有人请帮助我！ 非常感谢。

【问题讨论】：

首先您可以尝试将所有代码放在一个文件中并运行它。系统可能会从您期望的不同文件夹中运行代码，然后import 可能会在您期望的不同文件夹中搜索MLutils - 您可能需要在导入MLutils 之前将您的文件夹添加到列表sys.path

@furas 是的，我试过了，我可以通过这样做解决第一个问题，但我仍然遇到第二个错误......

当你运行 Run Cell Above 时，你的单元格中有什么？也放入文件中。

@furas 我在问题中添加了一张图片，只有2个单元格，第一个包含文件中的所有代码，第二个没有任何内容..

下一个错误显示randint(0, len(a) - 1) 有问题，因此您可以检查print(a, len(a))，此代码用于函数random_choice，因此您可以找到使用random_choice 的位置并使用print() 来查看您在 random_choice 中使用哪些值作为参数。如果其中一个是错误的，那么您必须返回并找到创建此值的位置并再次检查创建它的方式 - 使用 print() 检查用于创建此值的变量。这样（后退）你可能最终会找到引发这个问题的地方。

【参考方案1】：

我想通了。 如果我使用这个文件结构 机器学习模型

数据 类别 数据集.txt MLTrain.py MLutils.py 对于文件路径，我应该在

.py

脚本中使用MLModel\data\categories\*.txt。

但我应该在 jupyter notebook 中使用data\categories\*.txt。

当你想从.ipynb重写为.py时，无论如何要注意文件路径

----来自一个刚刚失去整个夜晚的年轻人。