# Setup library(keras) library(tensorflow) base_image_path = get_file('paris.jpg', 'https://i.imgur.com/aGBdQyK.jpg') result_prefix = 'sky_dream' # These are the names of the layers # for which we try to maximize activation, # as well as their weight in the final loss # we try to maximize. # You can tweak these setting to obtain new visual effects. layer_settings = list( 'mixed4' = 1.0, 'mixed5' = 1.5, 'mixed6' = 2.0, 'mixed7' = 2.5 ) # Playing with these hyperparameters will also allow you to achieve new effects step = 0.01 # Gradient ascent step size num_octave = 3 # Number of scales at which to run gradient ascent octave_scale = 1.4 # Size ratio between scales iterations = 20 # Number of ascent steps per scale max_loss = 15. # This is our base image: plot(magick::image_read(base_image_path)) # Let's set up some image preprocessing/deprocessing utilities: preprocess_image <- function(image_path) { # Util function to open, resize and format pictures # into appropriate arrays. img = tf$keras$preprocessing$image$load_img(image_path) img = tf$keras$preprocessing$image$img_to_array(img) img = tf$expand_dims(img, axis=0L) img = inception_v3_preprocess_input(img) img } deprocess_image <- function(x) { x = array_reshape(x, dim = c(dim(img)[[2]], dim(img)[[3]], 3)) # Undo inception v3 preprocession x = x / 2. x = x + 0.5 x = x * 255. # Convert to uint8 and clip to the valid range [0, 255] x = tf$clip_by_value(x, 0L, 255L) %>% tf$cast(dtype = 'uint8') x } save_img <- function(img, fname) { img <- deprocess_image(img) image_array_save(img, fname) } # Build an InceptionV3 model loaded with pre-trained ImageNet weights model <- application_inception_v3(weights = "imagenet", include_top = FALSE) # Get the symbolic outputs of each "key" layer (we gave them unique names). outputs_dict = list() for (layer_name in names(layer_settings)) { coeff <- layer_settings[[layer_name]] # Retrieves the layer's output activation <- get_layer(model, layer_name)$output outputs_dict[[layer_name]] <- activation } # Set up a model that returns the activation values for every target layer # (as a named list) feature_extractor = keras_model(inputs = model$inputs, outputs = outputs_dict) compute_loss <- function(input_image) { features = feature_extractor(input_image) names(features) = names(layer_settings) loss = tf$zeros(shape=list()) for (names in names(layer_settings)) { coeff = layer_settings[[names]] activation = features[[names]] # We avoid border artifacts by only involving non-border pixels in the loss. scaling = tf$reduce_prod(tf$cast(tf$shape(activation), 'float32')) loss = loss + coeff * tf$reduce_sum(tf$square(activation)) / scaling } loss } # Set up the gradient ascent loop for one octave gradient_ascent_step <- function(img, learning_rate) { with(tf$GradientTape() %as% tape, { tape$watch(img) loss = compute_loss(img) }) # Compute gradients. grads = tape$gradient(loss, img) # Normalize gradients. grads = grads / tf$maximum(tf$reduce_mean(tf$abs(grads)), 1e-6) img = img + learning_rate * grads list(loss, img) } gradient_ascent_loop <- function(img, iterations, learning_rate, max_loss = NULL) { for (i in 1:iterations) { c(loss, img) %<-% gradient_ascent_step(img, learning_rate) if (!is.null(max_loss) && as.array(loss) > max_loss) break cat("...Loss value at step", i, ":", as.array(loss), "\n") } img } # Run the training loop, iterating over different octaves original_img = preprocess_image(base_image_path) # Prepares a list of shape tuples defining the different scales at which to run gradient ascent original_shape <- dim(original_img)[2:3] successive_shapes <- list(original_shape) for (i in 1:num_octave) { shape <- as.integer(original_shape / (octave_scale ^ i)) successive_shapes[[length(successive_shapes) + 1]] <- shape } # Reverses the list of shapes so they're in increasing order successive_shapes <- rev(successive_shapes[1:3]) # Resizes the array of the image to the smallest scale shrunk_original_img <- tf$image$resize(original_img, successive_shapes[[1]]) img = tf$identity(original_img) # Make a copy for (i in 1:length(successive_shapes)) { shape = successive_shapes[[i]] cat("Processing octave", i, "with shape", shape, "\n") # Scales up the dream image img <- tf$image$resize(img, shape) # Runs gradient ascent, altering the dream img <- gradient_ascent_loop(img, iterations = iterations, learning_rate = step, max_loss = max_loss) # Scales up the smaller version of the original image: it will be pixellated upscaled_shrunk_original_img <- tf$image$resize(shrunk_original_img, shape) # Computes the high-quality version of the original image at this size same_size_original <- tf$image$resize(original_img, shape) # The difference between the two is the detail that was lost when scaling up lost_detail <- same_size_original - upscaled_shrunk_original_img # Reinjects lost detail into the dream img <- img + lost_detail shrunk_original_img <- tf$image$resize(original_img, shape) tf$keras$preprocessing$image$save_img(paste(result_prefix,'.png',sep = ''), deprocess_image(img$numpy())) } # Plot result plot(magick::image_read(paste(result_prefix,'.png',sep = '')))