如何通过浏览器从网络摄像头获取实时流视频详细信息到 python？

Posted 2023-03-16

【中文标题】如何通过浏览器从网络摄像头获取实时流视频详细信息到 python？

【英文标题】：How to fetch live stream video details from webcam via browser to python?

【发布时间】：2019-02-19 02:11:58

【问题描述】：

我正在使用 python、tensorflow、opencv 和 usbcamera 从实时流视频中检测面部和情绪。 它通过本地终端正常工作。我尝试通过浏览器运行网络摄像头。 如何通过 webrtc 将视频流发送到 python。 是否可以使用python逐帧获取webrtc结果，因为我想在这里做一些处理。

【问题讨论】：

如果您专注于 ML 的浏览器实现，我建议您使用他们的 javascript 实现。 Tensorflow.js 和 Keras.js 可用，您甚至可以在 tf.js 中使用经过 Python 训练的模型。在这种方法中，所有处理都在客户端机器上进行。检查此实现：github.com/ModelDepot/tfjs-yolo-tiny

感谢您的评论！如何在 tensorflow.js 中导入 python 训练模型

[pythonprogramming.net/loading-keras-model-tensorflowjs-tutorial/… 使用这个。附有文字说明的视频教程。

这些文件我想将 kerasmodel 转换为 tfjs Face_model_architecture.json,Face_model_weights.h5。我在下面发布了我的代码是否正确？

最后我找到了通过浏览器（如 Web 应用程序）检测流视频中的面部和情绪的解决方案。我参考这个链接：brendansudol.com/writing/tfjs-emotions, github.com/tupleblog/face-classification-js

【参考方案1】：

我将我的 keras 训练模型转换为我有 model.json 文件的 tfjs。 如何将转换后的模型加载到js文件中 我需要 model.predict_class() 来检测面部情绪。 如何在js中导入webcam.js进行直播以访问网络摄像头

convert_model_krs_tfjs.py
-------------------------
import keras
from keras.models import model_from_json 
from keras.optimizers import SGD
import numpy as np
from time import sleep
import tensorflowjs as tfjs

model = model_from_json(open('Face_model_architecture.json').read())
model.load_weights('Face_model_weights.h5')
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
model.save("Keras-64x2-10epoch")
tfjs.converters.save_keras_model(model, "face")

My python worked code:
----------------------
detectemotion.py
----------------
from keras.models import model_from_json
from keras.optimizers import SGD
import numpy as np
from time import sleep
model = model_from_json(open('./models/Face_model_architecture.json').read())
#model.load_weights('_model_weights.h5')
model.load_weights('./models/Face_model_weights.h5')
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
print(model)
def extract_face_features(gray, detected_face, offset_coefficients):
        (x, y, w, h) = detected_face
        #print x , y, w ,h
        horizontal_offset = np.int(np.floor(offset_coefficients[0] * w))
        vertical_offset = np.int(np.floor(offset_coefficients[1] * h))


        extracted_face = gray[y+vertical_offset:y+h, 
                          x+horizontal_offset:x-horizontal_offset+w]
        #print extracted_face.shape
        new_extracted_face = zoom(extracted_face, (48. / extracted_face.shape[0], 
                                               48. / extracted_face.shape[1]))
        new_extracted_face = new_extracted_face.astype(np.float32)
        new_extracted_face /= float(new_extracted_face.max())
        return new_extracted_face
from scipy.ndimage import zoom
def detect_face(frame):
        cascPath = "./models/haarcascade_frontalface_default.xml"
        faceCascade = cv2.CascadeClassifier(cascPath)
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        detected_faces = faceCascade.detectMultiScale(
                gray,
                scaleFactor=1.1,
                minNeighbors=6,
                minSize=(48, 48),
                flags=cv2.CASCADE_SCALE_IMAGE
            )
        return gray, detected_faces

import cv2
cascPath = "haarcascade_frontalface_default.xml"
faceCascade = cv2.CascadeClassifier(cascPath)

video_capture = cv2.VideoCapture(0)

while True:
    # Capture frame-by-frame
#    sleep(0.8)
    ret, frame = video_capture.read()

    # detect faces
    gray, detected_faces = detect_face(frame)

    face_index = 0

    # predict output
    for face in detected_faces:
        (x, y, w, h) = face
        if w > 100:
            # draw rectangle around face 
            cv2.rectangle(frame, (x, y), (x+w, y+h), (0,255,0), 2)

            # extract features
            extracted_face = extract_face_features(gray, face, (0.075, 0.05)) #(0.075, 0.05)

            # predict smile
            prediction_result = model.predict_classes(extracted_face.reshape(1,48,48,1))

            # draw extracted face in the top right corner
            frame[face_index * 48: (face_index + 1) * 48, -49:-1, :] = cv2.cvtColor(extracted_face * 255, cv2.COLOR_GRAY2RGB)

            # annotate main image with a label
            if prediction_result == 3:
                cv2.putText(frame, "Happy!!",(x,y), cv2.FONT_ITALIC, 2, (255,255,255), 2)
            elif prediction_result == 0:
                cv2.putText(frame, "Angry",(x,y), cv2.FONT_HERSHEY_SIMPLEX, 2, (255,255,255), 2)
        elif prediction_result == 1:
                cv2.putText(frame, "Disgust",(x,y), cv2.FONT_HERSHEY_SIMPLEX, 2, (255,255,255), 2)
        elif prediction_result == 2:
                cv2.putText(frame, "Fear",(x,y), cv2.FONT_HERSHEY_SIMPLEX, 2, (255,255,255), 2)
        elif prediction_result == 4:
                cv2.putText(frame, "Sad",(x,y), cv2.FONT_HERSHEY_SIMPLEX, 2, (255,255,255), 2)
        elif prediction_result == 5:
                cv2.putText(frame, "Surprise",(x,y), cv2.FONT_HERSHEY_SIMPLEX, 2,(255,255,255), 2)
        else :
                cv2.putText(frame, "Neutral",(x,y), cv2.FONT_HERSHEY_SIMPLEX, 2,(255,255,255), 2)


            # increment counter
            face_index += 1


    # Display the resulting frame
    cv2.imshow('Video', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# When everything is done, release the capture
video_capture.release()
cv2.destroyAllWindows()