Numpy实现激活函数

Posted 2021-10-06 AI浩

import numpy as np


# Collection of activation functions
# Reference: https://en.wikipedia.org/wiki/Activation_function

class Sigmoid():
    def __call__(self, x):
        return 1 / (1 + np.exp(-x))

    def gradient(self, x):
        return self.__call__(x) * (1 - self.__call__(x))


class Softmax():
    def __call__(self, x):
        e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
        return e_x / np.sum(e_x, axis=-1, keepdims=True)

    def gradient(self, x):
        p = self.__call__(x)
        return p * (1 - p)


class TanH():
    def __call__(self, x):
        return 2 / (1 + np.exp(-2 * x)) - 1

    def gradient(self, x):
        return 1 - np.power(self.__call__(x), 2)


class ReLU():
    def __call__(self, x):
        return np.where(x >= 0, x, 0)

    def gradient(self, x):
        return np.where(x >= 0, 1, 0)


class LeakyReLU():
    def __init__(self, alpha=0.2):
        self.alpha = alpha

    def __call__(self, x):
        return np.where(x >= 0, x, self.alpha * x)

    def gradient(self, x):
        return np.where(x >= 0, 1, self.alpha)


class ELU(object):
    def __init__(self, alpha=0.1):
        self.alpha = alpha

    def __call__(self, x):
        return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))

    def gradient(self, x):
        return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)


class SELU():
    # Reference : https://arxiv.org/abs/1706.02515,
    # https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
    def __init__(self):
        self.alpha = 1.6732632423543772848170429916717
        self.scale = 1.0507009873554804934193349852946

    def __call__(self, x):
        return self.scale * np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))

    def gradient(self, x):
        return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))


class SoftPlus(object):
    def __call__(self, x):
        return np.log(1 + np.exp(x))

    def gradient(self, x):
        return 1 / (1 + np.exp(-x))


class Swish(object):
    def __init__(self, b):
        self.b = b

    def __call__(self, x):
        return x * (np.exp(self.b * x) / (np.exp(self.b * x) + 1))

    def gradient(self, x):
        return self.b * x / (1 + np.exp(-self.b * x)) + (1 / (1 + np.exp(-self.b * x)))(
            1 - self.b * (x / (1 + np.exp(-self.b * x))))


def sech(x):
    """sech函数"""
    return 2 / (np.exp(x) + np.exp(-x))


def sigmoid(x):
    """sigmoid函数"""
    return 1 / (1 + np.exp(-x))


def soft_plus(x):
    """softplus函数"""
    return np.log(1 + np.exp(x))


def tan_h(x):
    """tanh函数"""
    return (np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x))


class Mish:

    def __call__(self, x):
        return x * tan_h(soft_plus(x))

    def gradient(self, x):
        return sech(soft_plus(x)) * sech(soft_plus(x)) * x * sigmoid(x) + tan_h(soft_plus(x))

class SILU(object):

    def __call__(self, x):
        return x * sigmoid(x)

    def gradient(self, x):
        return self.__call__(x) + sigmoid(x) * (1 - self.__call__(x))