将训练好的 Keras 图像分类模型转换为 coreml 并集成到 iOS11

Posted 2023-03-27

【中文标题】将训练好的 Keras 图像分类模型转换为 coreml 并集成到 iOS11

【英文标题】：Convert trained Keras image classification model to coreml and integrate in iOS11

【发布时间】：2017-12-15 14:03:30

【问题描述】：

使用https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html 上的教程，我训练了一个 Keras 模型来识别猫和狗之间的区别。

'''
Directory structure:

data/
    train/
        dogs/
            dog001.jpg
            dog002.jpg
            ...
        cats/
            cat001.jpg
            cat002.jpg
            ...
    validation/
        dogs/
            dog001.jpg
            dog002.jpg
            ...
        cats/
            cat001.jpg
            cat002.jpg
            ...
'''    
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
from PIL import Image
import numpy as np

# dimensions of our images.
img_width, img_height = 150, 150

train_data_dir = 'data/train'
validation_data_dir = 'data/validation'
nb_train_samples = 2000
nb_validation_samples = 800
epochs = 50
batch_size = 16

if K.image_data_format() == 'channels_first':
    input_shape = (3, img_width, img_height)
else:
    input_shape = (img_width, img_height, 3)

model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True)

# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1. / 255)

train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='binary')

validation_generator = test_datagen.flow_from_directory(
    validation_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode='binary')

model.fit_generator(
    train_generator,
    steps_per_epoch=nb_train_samples // batch_size,
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps=nb_validation_samples // batch_size)

model.save('first_try.h5')

使用 coremltools documentation 作为指导，我尝试将我的模型转换为 coreml 格式：

import coremltools
import h5py
coreml_model = coremltools.converters.keras.convert('first_try.h5',input_names='image',output_names='class',image_input_names = 'image',class_labels = ['cat', 'dog'], is_bgr=True)
coreml_model.save('cats_dogs.mlmodel')

当我将模型导入 XCode 并使用以下代码运行它时（适用于 Apple 网站上的 resnet50 和 inceptionv3 模型），代码 sn-p guard let prediction = try? model.prediction(image: pixelBuffer!) else print("error!") return 打印“错误！”并且永远无法到达textView.text = "I think this is a \(String(describing: prediction.classLabel))." 代码 sn-p。

    import UIKit
    import Vision
    import CoreML

    class ViewController: UIViewController, UINavigationControllerDelegate 

        //MARK: - Properties


        @IBOutlet weak var imageView: UIImageView!
        @IBOutlet weak var textView: UITextView!

        let imagePicker = UIImagePickerController()

        //MARK: - ViewController

        override func viewDidLoad() 
            super .viewDidLoad()
            self.imagePicker.delegate = self
        

        @IBAction func openImagePicker(_ sender: Any) 
            imagePicker.allowsEditing = false
            imagePicker.sourceType = .photoLibrary
            present(imagePicker, animated: true, completion: nil)
        

        @IBAction func camera(_ sender: Any) 

            if !UIImagePickerController.isSourceTypeAvailable(.camera) 
                return
            

            let cameraPicker = UIImagePickerController()
            cameraPicker.delegate = self
            cameraPicker.sourceType = .camera
            cameraPicker.allowsEditing = false

            present(cameraPicker, animated: true)
        

    

    extension ViewController: UIImagePickerControllerDelegate 
        func imagePickerControllerDidCancel(_ picker: UIImagePickerController) 
            dismiss(animated: true, completion: nil)
        

        func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [String : Any]) 
            picker.dismiss(animated: true)
            textView.text = "Analyzing Image..."
            guard let image = info["UIImagePickerControllerOriginalImage"] as? UIImage else 
                return
            

            UIGraphicsBeginImageContextWithOptions(CGSize(width: 150, height: 150), true, 2.0)
            image.draw(in: CGRect(x: 0, y: 0, width: 150, height: 150))
            let newImage = UIGraphicsGetImageFromCurrentImageContext()!
            UIGraphicsEndImageContext()

            let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
            var pixelBuffer : CVPixelBuffer?
            let status = CVPixelBufferCreate(kCFAllocatorDefault, Int(newImage.size.width), Int(newImage.size.height), kCVPixelFormatType_32ARGB, attrs, &pixelBuffer)
            guard (status == kCVReturnSuccess) else 
                return
            

            CVPixelBufferLockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
            let pixelData = CVPixelBufferGetBaseAddress(pixelBuffer!)

            let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
            let context = CGContext(data: pixelData, width: Int(newImage.size.width), height: Int(newImage.size.height), bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: rgbColorSpace, bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue) //3

            context?.translateBy(x: 0, y: newImage.size.height)
            context?.scaleBy(x: 1.0, y: -1.0)

            UIGraphicsPushContext(context!)
            newImage.draw(in: CGRect(x: 0, y: 0, width: newImage.size.width, height: newImage.size.height))
            UIGraphicsPopContext()
            CVPixelBufferUnlockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
            imageView.image = newImage

guard let prediction = try? model.prediction(image: pixelBuffer!) else 
        print("error!")
        return
    
    textView.text = "I think this is a \(String(describing: prediction.classLabel))."
        
    

XCode Error Implementing Custom Keras-CoreML model

我已经广泛搜索了 *** 来解决这个问题。非常感谢您帮助解决此问题！

================================================ ============================== 编辑＃1：

使用“print(try!model.prediction(image:pixelBuffer!) as Any)”我打印了以下错误：

2017-07-13 15:33:49.034967-0400cats_dogs[60441:1198094] 致命错误：“尝试！”表达式意外引发错误：错误域 = com.apple.CoreML 代码 = 0“层 '输出' 的尺寸与类标签的数量不同。” UserInfo=NSLocalizedDescription=“输出”层的维度与类标签的数量不同。：

不确定“层'输出'的尺寸与类标签的数量不同”是什么意思。

================================================ ==============================

编辑#2：

这是我用来将模型转换为 .mlmodel 格式的代码

import coremltools   
import h5py   
output_labels = ['cat','dog']   
coreml_model = coremltools.converters.keras.convert('first_try.h5',input_names='image',image_input_names = 'image',class_labels = output_labels, is_bgr=False)   
coreml_model.author = ''   
coreml_model.short_description = 'Model to classify images as either cats or dogs'   
coreml_model.input_description['image'] = 'Image of a cat or dog'   
print coreml_model   
coreml_model.save('cats_dogs.mlmodel') 

这是终端输出：

0 : conv2d_1_input, <keras.engine.topology.InputLayer object at 0x1194c6c50>
1 : conv2d_1, <keras.layers.convolutional.Conv2D object at 0x1194c6c90>
2 : activation_1, <keras.layers.core.Activation object at 0x119515b90>
3 : max_pooling2d_1, <keras.layers.pooling.MaxPooling2D object at 0x119501e50>
4 : conv2d_2, <keras.layers.convolutional.Conv2D object at 0x119520cd0>
5 : activation_2, <keras.layers.core.Activation object at 0x1194e8150>
6 : max_pooling2d_2, <keras.layers.pooling.MaxPooling2D object at 0x11955cc50>
7 : conv2d_3, <keras.layers.convolutional.Conv2D object at 0x11955ce50>
8 : activation_3, <keras.layers.core.Activation object at 0x11954d9d0>
9 : max_pooling2d_3, <keras.layers.pooling.MaxPooling2D object at 0x119594cd0>
10 : flatten_1, <keras.layers.core.Flatten object at 0x1195a08d0>
11 : dense_1, <keras.layers.core.Dense object at 0x119579f10>
12 : activation_4, <keras.layers.core.Activation object at 0x1195c94d0>
13 : dense_2, <keras.layers.core.Dense object at 0x1195ea450>
14 : activation_5, <keras.layers.core.Activation object at 0x119614b10>

input 
  name: "image"
  shortDescription: "Image of a cat or dog"
  type 
    imageType 
      width: 150
      height: 150
      colorSpace: RGB
    
  

output 
  name: "output1"
  type 
    dictionaryType 
      stringKeyType 
      
    
  

output 
  name: "classLabel"
  type 
    stringType 
    
  

predictedFeatureName: "classLabel"
predictedProbabilitiesName: "output1"
metadata 
  shortDescription: "Model to classify images as either cats or dogs"
  author: ""

【参考方案1】：

不确定“层'输出'的尺寸与类标签的数量不同”是什么意思。

这意味着您的模型的最后一层与您的类标签的维度不同（我假设是维度 2）。我建议删除此参数：

class_labels = output_labels

从您的模型转换中，看看它是否解决了问题