192 lines
6.5 KiB
Python
192 lines
6.5 KiB
Python
import cv2
|
|
import h5py
|
|
import keras
|
|
import keras.backend as K
|
|
from keras.layers.core import Lambda
|
|
from keras.models import Sequential
|
|
from keras.models import load_model
|
|
import numpy as np
|
|
import tensorflow as tf
|
|
from tensorflow.python.framework import ops
|
|
|
|
from .preprocessor import preprocess_input
|
|
|
|
|
|
def reset_optimizer_weights(model_filename):
|
|
model = h5py.File(model_filename, 'r+')
|
|
del model['optimizer_weights']
|
|
model.close()
|
|
|
|
|
|
def target_category_loss(x, category_index, num_classes):
|
|
return tf.multiply(x, K.one_hot([category_index], num_classes))
|
|
|
|
|
|
def target_category_loss_output_shape(input_shape):
|
|
return input_shape
|
|
|
|
|
|
def normalize(x):
|
|
# utility function to normalize a tensor by its L2 norm
|
|
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
|
|
|
|
|
|
def load_image(image_array):
|
|
image_array = np.expand_dims(image_array, axis=0)
|
|
image_array = preprocess_input(image_array)
|
|
return image_array
|
|
|
|
|
|
def register_gradient():
|
|
if "GuidedBackProp" not in ops._gradient_registry._registry:
|
|
@ops.RegisterGradient("GuidedBackProp")
|
|
def _GuidedBackProp(op, gradient):
|
|
dtype = op.inputs[0].dtype
|
|
guided_gradient = (gradient * tf.cast(gradient > 0., dtype) *
|
|
tf.cast(op.inputs[0] > 0., dtype))
|
|
return guided_gradient
|
|
|
|
|
|
def compile_saliency_function(model, activation_layer='conv2d_7'):
|
|
input_image = model.input
|
|
layer_output = model.get_layer(activation_layer).output
|
|
max_output = K.max(layer_output, axis=3)
|
|
saliency = K.gradients(K.sum(max_output), input_image)[0]
|
|
return K.function([input_image, K.learning_phase()], [saliency])
|
|
|
|
|
|
def modify_backprop(model, name, task):
|
|
graph = tf.get_default_graph()
|
|
with graph.gradient_override_map({'Relu': name}):
|
|
|
|
# get layers that have an activation
|
|
activation_layers = [layer for layer in model.layers
|
|
if hasattr(layer, 'activation')]
|
|
|
|
# replace relu activation
|
|
for layer in activation_layers:
|
|
if layer.activation == keras.activations.relu:
|
|
layer.activation = tf.nn.relu
|
|
|
|
# re-instanciate a new model
|
|
if task == 'gender':
|
|
model_path = '../trained_models/gender_models/gender_mini_XCEPTION.21-0.95.hdf5'
|
|
elif task == 'emotion':
|
|
model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
|
|
# model_path = '../trained_models/fer2013_mini_XCEPTION.119-0.65.hdf5'
|
|
# model_path = '../trained_models/fer2013_big_XCEPTION.54-0.66.hdf5'
|
|
new_model = load_model(model_path, compile=False)
|
|
return new_model
|
|
|
|
|
|
def deprocess_image(x):
|
|
""" Same normalization as in:
|
|
https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py
|
|
"""
|
|
if np.ndim(x) > 3:
|
|
x = np.squeeze(x)
|
|
# normalize tensor: center on 0., ensure std is 0.1
|
|
x = x - x.mean()
|
|
x = x / (x.std() + 1e-5)
|
|
x = x * 0.1
|
|
|
|
# clip to [0, 1]
|
|
x = x + 0.5
|
|
x = np.clip(x, 0, 1)
|
|
|
|
# convert to RGB array
|
|
x = x * 255
|
|
if K.image_dim_ordering() == 'th':
|
|
x = x.transpose((1, 2, 0))
|
|
x = np.clip(x, 0, 255).astype('uint8')
|
|
return x
|
|
|
|
|
|
def compile_gradient_function(input_model, category_index, layer_name):
|
|
model = Sequential()
|
|
model.add(input_model)
|
|
|
|
num_classes = model.output_shape[1]
|
|
target_layer = lambda x: target_category_loss(x, category_index, num_classes)
|
|
model.add(Lambda(target_layer,
|
|
output_shape=target_category_loss_output_shape))
|
|
|
|
loss = K.sum(model.layers[-1].output)
|
|
conv_output = model.layers[0].get_layer(layer_name).output
|
|
gradients = normalize(K.gradients(loss, conv_output)[0])
|
|
gradient_function = K.function([model.layers[0].input, K.learning_phase()],
|
|
[conv_output, gradients])
|
|
return gradient_function
|
|
|
|
|
|
def calculate_gradient_weighted_CAM(gradient_function, image):
|
|
output, evaluated_gradients = gradient_function([image, False])
|
|
output, evaluated_gradients = output[0, :], evaluated_gradients[0, :, :, :]
|
|
weights = np.mean(evaluated_gradients, axis=(0, 1))
|
|
CAM = np.ones(output.shape[0: 2], dtype=np.float32)
|
|
for weight_arg, weight in enumerate(weights):
|
|
CAM = CAM + (weight * output[:, :, weight_arg])
|
|
CAM = cv2.resize(CAM, (64, 64))
|
|
CAM = np.maximum(CAM, 0)
|
|
heatmap = CAM / np.max(CAM)
|
|
|
|
# Return to BGR [0..255] from the preprocessed image
|
|
image = image[0, :]
|
|
image = image - np.min(image)
|
|
image = np.minimum(image, 255)
|
|
|
|
CAM = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
|
|
CAM = np.float32(CAM) + np.float32(image)
|
|
CAM = 255 * CAM / np.max(CAM)
|
|
return np.uint8(CAM), heatmap
|
|
|
|
|
|
def calculate_guided_gradient_CAM(
|
|
preprocessed_input, gradient_function, saliency_function):
|
|
CAM, heatmap = calculate_gradient_weighted_CAM(
|
|
gradient_function, preprocessed_input)
|
|
saliency = saliency_function([preprocessed_input, 0])
|
|
# gradCAM = saliency[0] * heatmap[..., np.newaxis]
|
|
# return deprocess_image(gradCAM)
|
|
return deprocess_image(saliency[0])
|
|
# return saliency[0]
|
|
|
|
|
|
def calculate_guided_gradient_CAM_v2(
|
|
preprocessed_input, gradient_function,
|
|
saliency_function, target_size=(128, 128)):
|
|
CAM, heatmap = calculate_gradient_weighted_CAM(
|
|
gradient_function, preprocessed_input)
|
|
heatmap = np.squeeze(heatmap)
|
|
heatmap = cv2.resize(heatmap.astype('uint8'), target_size)
|
|
saliency = saliency_function([preprocessed_input, 0])
|
|
saliency = np.squeeze(saliency[0])
|
|
saliency = cv2.resize(saliency.astype('uint8'), target_size)
|
|
gradCAM = saliency * heatmap
|
|
gradCAM = deprocess_image(gradCAM)
|
|
return np.expand_dims(gradCAM, -1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import pickle
|
|
faces = pickle.load(open('faces.pkl', 'rb'))
|
|
face = faces[0]
|
|
model_filename = '../../trained_models/emotion_models/mini_XCEPTION.523-0.65.hdf5'
|
|
# reset_optimizer_weights(model_filename)
|
|
model = load_model(model_filename)
|
|
|
|
preprocessed_input = load_image(face)
|
|
predictions = model.predict(preprocessed_input)
|
|
predicted_class = np.argmax(predictions)
|
|
gradient_function = compile_gradient_function(
|
|
model, predicted_class, 'conv2d_6')
|
|
register_gradient()
|
|
guided_model = modify_backprop(model, 'GuidedBackProp')
|
|
saliency_function = compile_saliency_function(guided_model)
|
|
guided_gradCAM = calculate_guided_gradient_CAM(
|
|
preprocessed_input, gradient_function, saliency_function)
|
|
|
|
cv2.imwrite('guided_gradCAM.jpg', guided_gradCAM)
|
|
|
|
|