风格迁移试玩

Posted 2021-10-28 Debroon

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了风格迁移试玩相关的知识，希望对你有一定的参考价值。

风格迁移

现成工具：tensorflow hub
手工实现风格迁移
我们对风格有失恭敬

神经风格转换是深度学习领域中一个很有趣的技术。它可以改变图像的风格。

如下图所示，根据一张内容图片和一张风格图片，生成一张新图片，这张图片结合了第一张图像的内容和第二张图像的风格。

现成工具：tensorflow hub

在 tensorflow hub 中已经有现成的风格转换模型可以被免费调用了。

除了风格转换模型外，hub 中还包含了很多常见的模型，很强大很可怕！！

https://hub.tensorflow.google.cn/

我们将下面俩张图合成吧。

import os
import tensorflow as tf
os.environ['TFHUB_MODEL_LOAD_FORMAT'] = 'COMPRESSED'
import IPython.display as display
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.rcParams['figure.figsize'] = (12, 12)
mpl.rcParams['axes.grid'] = False

import numpy as np
import PIL.Image
import time
import functools

def load_img(path_to_img):
  max_dim = 512
  img = tf.io.read_file(path_to_img)
  img = tf.image.decode_image(img, channels=3)
  img = tf.image.convert_image_dtype(img, tf.float32)

  shape = tf.cast(tf.shape(img)[:-1], tf.float32)
  long_dim = max(shape)
  scale = max_dim / long_dim
  new_shape = tf.cast(shape * scale, tf.int32)
  img = tf.image.resize(img, new_shape)
  img = img[tf.newaxis, :]
  return img

def imshow(image, title=None):
  if len(image.shape) > 3:
    image = tf.squeeze(image, axis=0)

  plt.imshow(image)
  if title:
    plt.title(title)

content_path = tf.keras.utils.get_file('ebcf732904a54911be5967c5b072a8e4.jpeg', 'https://img-blog.csdnimg.cn/ebcf732904a54911be5967c5b072a8e4.jpg')
style_path = tf.keras.utils.get_file('b275d4b95c33488e93a829bb1e7da6c9.jpeg', 'https://img-blog.csdnimg.cn/b275d4b95c33488e93a829bb1e7da6c9.jpg')
content_image = load_img(content_path)
style_image = load_img(style_path)
plt.subplot(1, 2, 1)
imshow(content_image, 'Content Image')
plt.subplot(1, 2, 2)
imshow(style_image, 'Style Image')

def tensor_to_image(tensor):
  tensor = tensor * 255
  tensor = np.array(tensor, dtype=np.uint8)
  if np.ndim(tensor) > 3:
    assert tensor.shape[0] == 1
    tensor = tensor[0]
  return PIL.Image.fromarray(tensor)

import tensorflow_hub as hub
hub_model = hub.load('https://hub.tensorflow.google.cn/google/magenta/arbitrary-image-stylization-v1-256/2')
stylized_image = hub_model(tf.constant(content_image), tf.constant(style_image))[0]
tensor_to_image(stylized_image)

输出：

手工实现风格迁移

迁移学习其实就是利用已经训练好的模型来实现另一个任务，我们借用一个训练好了的 VGG-19 模型。

vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
content_layers = ['block5_conv2'] 
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

def vgg_layers(layer_names):
  vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
  vgg.trainable = False

  outputs = [vgg.get_layer(name).output for name in layer_names]

  model = tf.keras.Model([vgg.input], outputs)
  return model

def gram_matrix(input_tensor):
  result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)
  input_shape = tf.shape(input_tensor)
  num_locations = tf.cast(input_shape[1]*input_shape[2], tf.float32)
  return result/(num_locations)

class StyleContentModel(tf.keras.models.Model):
  def __init__(self, style_layers, content_layers):
    super(StyleContentModel, self).__init__()
    self.vgg = vgg_layers(style_layers + content_layers)
    self.style_layers = style_layers
    self.content_layers = content_layers
    self.num_style_layers = len(style_layers)
    self.vgg.trainable = False 

  def call(self, inputs):
    inputs = inputs*255.0
    preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs)
    outputs = self.vgg(preprocessed_input)
    style_outputs, content_outputs = (outputs[:self.num_style_layers],
                                      outputs[self.num_style_layers:])

    style_outputs = [gram_matrix(style_output)
                     for style_output in style_outputs]

    content_dict = {content_name: value
                    for content_name, value
                    in zip(self.content_layers, content_outputs)}

    style_dict = {style_name: value
                  for style_name, value
                  in zip(self.style_layers, style_outputs)}

    return {'content': content_dict, 'style': style_dict}

extractor = StyleContentModel(style_layers, content_layers)
style_targets = extractor(style_image)['style'] 
content_targets = extractor(content_image)['content'] 
image = tf.Variable(content_image)
def clip_0_1(image):
  return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)
opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)
style_weight = 1e-2 
content_weight = 1e4

def style_content_loss(outputs):
    style_outputs = outputs['style'] 
    content_outputs = outputs['content'] 
    style_loss = tf.add_n([tf.reduce_mean((style_outputs[name]-style_targets[name])**2) 
                           for name in style_outputs.keys()])
    style_loss *= style_weight / num_style_layers
    content_loss = tf.add_n([tf.reduce_mean((content_outputs[name]-content_targets[name])**2) 
                             for name in content_outputs.keys()])
    content_loss *= content_weight / num_content_layers
    loss = style_loss + content_loss
    return loss

def train_step(image):
  with tf.GradientTape() as tape:
    outputs = extractor(image) 
    loss = style_content_loss(outputs) 
  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)]) 
  image.assign(clip_0_1(image))

import time
start = time.time()

epochs = 10
steps_per_epoch = 100

# 只训练了三步，图片风格会稍稍变化
train_step(image)
train_step(image)
train_step(image)
tensor_to_image(image)

'''
真正训练的话，是要很多步的。会花很长时间，以下代码电脑配置不好的可能要花几个小时
step = 0
for n in range(epochs):
  for m in range(steps_per_epoch):
    step += 1
    train_step(image)
    print(".", end='', flush=True)
  display.clear_output(wait=True)
  display.display(tensor_to_image(image))
  print("Train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))
'''