visualization.py 文件源码

def get_superimposed_frame(video_id, frame_filename, saliency_frame, sentence):
    from matplotlib.font_manager import FontProperties
    font0 = FontProperties()
    font0.set_family("sans-serif")

    def generate_saliency(spatial_attention, image_size, norm = False):
        minimum = np.amin(spatial_attention)
        maximum = np.amax(spatial_attention)

        spatial_attention = np.pad(spatial_attention, pad_width = ((1, 1), (1, 1)), 
               mode = 'constant', constant_values=np.amin(spatial_attention))

        saliency_image = Image.fromarray(np.uint8((spatial_attention) * float(255)), 'L').resize(image_size, Image.BICUBIC)
        saliency_image = saliency_image.resize((int(image_size[0] * 1.2), int(image_size[1] * 1.2)), Image.BICUBIC)
        saliency_image = saliency_image.crop((int(image_size[0] * 0.1 ), int(image_size[1] * 0.1 ), int(image_size[0] * 1.1), int(image_size[0] * 1.1) ))

        return saliency_image 


    original_image = Image.open(frame_filename).resize((SCALE, SCALE),
                                                        Image.ANTIALIAS)
    n_words = saliency_frame.shape[0]

    w = np.floor(np.sqrt(n_words))
    h = np.ceil(np.float32(n_words) / w )
    figw, figh = int(h * 3), int(w * 3)
    f = plt.figure(figsize=(figw, figh), facecolor = "black", dpi = 150)

    for word_idx in range(saliency_frame.shape[0]):
        plt.subplot(w, h, word_idx+1)    
        plt.imshow(original_image)
        saliency = generate_saliency(saliency_frame[word_idx].reshape(8, 8),
                                     (SCALE, SCALE), norm = False)
        saliency = np.asarray(saliency) / 255.
        plt.imshow(saliency, vmin=0.0, vmax=1.0, alpha = 0.5, cmap="jet")

        fontsize = 12 + (h - 2) * 2 
        plt.text(6, 18, sentence[word_idx], fontproperties = font0,
                 color = "black", backgroundcolor='white', fontsize=fontsize)

        plt.axis('off')
        plt.tick_params(axis='both', left='off', top='off', right='off',
                        bottom='off', labelleft='off', labeltop='off',
                        labelright='off', labelbottom='off')

    bezel_thickness = 0.02
    plt.tight_layout(pad = bezel_thickness, w_pad=bezel_thickness, h_pad=bezel_thickness)
    plt.subplots_adjust(hspace = bezel_thickness , # ,
                        wspace = bezel_thickness )

    plt.savefig(path.join(path_to_save_figures, video_id, frame_filename.split("/")[-1] + ".png"),
                bbox_inches='tight',
                facecolor=f.get_facecolor(),
                dpi=90,
                edgecolor='none')
    plt.close()