TensorFlow by Google 使用排序 APIMachine Learning Foundations: Ep #9 - Using the Sequencing APIs

Posted 2022-02-14 AI架构师易筋

练习

https://bit.ly/tfw-nlp2

import tensorflow as tf
from tensorflow import keras


from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

sentences = [
    'I love my dog',
    'I love my cat',
    'You love my dog!',
    'Do you think my dog is amazing?'
]

tokenizer = Tokenizer(num_words = 100, oov_token="<OOV>")
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index

sequences = tokenizer.texts_to_sequences(sentences)

padded = pad_sequences(sequences, maxlen=5)
print("\\nWord Index = " , word_index)
print("\\nSequences = " , sequences)
print("\\nPadded Sequences:")
print(padded)


# Try with words that the tokenizer wasn't fit to
test_data = [
    'i really love my dog',
    'my dog loves my manatee'
]

test_seq = tokenizer.texts_to_sequences(test_data)
print("\\nTest Sequence = ", test_seq)

padded = pad_sequences(test_seq, maxlen=10)
print("\\nPadded Test Sequence: ")
print(padded)

参考

https://youtu.be/L3suP4g8p7U