python generator.py

Posted 2021-05-09

import pandas as pd
import numpy as np
from process_data import DataMerger
from keras.models import Sequential, Model
from keras.layers import Input, Dense, Dropout, Activation, BatchNormalization, Lambda
from sklearn.model_selection import train_test_split

merger = DataMerger()
merger.createDataset()
merger.createCSV()

def converter(instr):
    x = np.fromstring(instr[1:-1],sep=' ')
    return x

dataset=pd.read_csv('dataset.csv',converters={'noun':converter, 'npro':converter})

print(dataset.isnull().any())

X1 = dataset.iloc[:,0].values
X2 = dataset.iloc[:,1].values
X = []
for i in range(0,len(X1)):
    X.append(np.concatenate((X1[i], X2[i]), axis = 0))
X = np.array(X)
print(X.shape)
y = dataset.iloc[:,2].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

seq = Sequential()
seq.add(Dense(300, input_shape=X_train[0].shape, activation='relu'))
seq.add(Dense(100, activation='relu'))
seq.add(Dense(1, activation='relu'))

seq.compile(optimizer ='adam',loss='binary_crossentropy', metrics =['accuracy'])
seq.fit(X_train,y_train, batch_size=300, epochs=50)

y_pred=seq.predict(X_test)
y_pred =(y_pred>0.5)
         

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)
print(cm)