python类merge()的实例源码

def __init__(self, lang_model: language_models.ALanguageModel,
                 img_model: image_models.AImageModel, language_only,
                 num_hidden_units, activation, dropout, num_hidden_layers, nb_classes):
        """
        :param lang_model: the language model to use
        :param img_model: the image model to use
        :param language_only: use a language model only to answer question and ignore images
        :param num_hidden_units: number of hidden units per hidden layer
        :param activation: activation function type
        :param dropout: fraction of nodes which should be dropped out in each training step,
        between 0 and 1.
        :param num_hidden_layers: the number of hidden layers
        :param nb_classes: the number of possible answers we allow (softmax size in the end)
        """
        # Start constructing the Keras model.
        model = Sequential()

        if language_only:
            # Language only means we *ignore the images* and only rely on the
            # question to compute an answers. Interestingly enough, this does not
            # suck horribly.
            model.add(Merge([lang_model.model()], mode='concat', concat_axis=1))
        else:
            model.add(Merge([lang_model.model(), img_model.model()], mode='concat', concat_axis=1))

        if dropout > 0:
            model.add(Dropout(dropout))

        for i in range(num_hidden_layers):
            model.add(Dense(num_hidden_units, init='uniform'))
            model.add(Activation(activation))
            if dropout > 0:
                model.add(Dropout(dropout))

        model.add(Dense(nb_classes, init='uniform'))
        model.add(Activation('softmax'))

        print('Compiling Keras model...')
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
        print('Compilation done...')
        self._model = model

def Wavenet(input_shape, filters, depth, stacks, last=0, h=None, build=True):
    # TODO: Soft targets? A float to make targets a gaussian with stdev.
    # TODO: Train only receptive field. The temporal-first outputs are computed from zero-padding.
    # TODO: Global conditioning?
    # TODO: Local conditioning?

    _, nb_bins = input_shape

    input_audio = Input(input_shape, name='audio_input')

    model = CausalAtrousConvolution1D(filters, 2, mask_type='A', atrous_rate=1, border_mode='valid')(input_audio)

    out, skip_connections = WavenetBlocks(filters, depth, stacks)(model)

    out = Merge(mode='sum', name='merging_skips')(skip_connections)
    out = PReLU()(out)

    out = Convolution1D(nb_bins, 1, border_mode='same')(out)
    out = PReLU()(out)

    out = Convolution1D(nb_bins, 1, border_mode='same')(out)

    # https://storage.googleapis.com/deepmind-live-cms/documents/BlogPost-Fig2-Anim-160908-r01.gif
    if last > 0:
        out = Lambda(lambda x: x[:, -last:], output_shape=(last, out._keras_shape[2]), name='last_out')(out)

    out = Activation('softmax')(out)

    if build:
        model = Model(input_audio, out)
        model.compile(Nadam(), 'sparse_categorical_crossentropy')

    return model

def __call__(self, model):
        # 2h -> h
        block = PReLU()(model)
        block = MaskedConvolution2D(self.filters//2, 1, 1)(block)

        # h 3x3 -> h
        block = PReLU()(block)
        block = MaskedConvolution2D(self.filters//2, 3, 3, border_mode='same')(block)

        # h -> 2h
        block = PReLU()(block)
        block = MaskedConvolution2D(self.filters, 1, 1)(block)

        return Merge(mode='sum')([model, block])

def prepare_model(ninputs=9600, n_feats=45,nclass=4,n_tfidf=10001):
    inp1 = Input(shape=(ninputs,))
    inp2 = Input(shape=(n_feats,))
    inp3 = Input(shape=(n_tfidf,))
    reg = 0.00005
    out_neurons1 = 500
    #out_neurons2 = 20
    #out_neurons2 = 10
    m1 = Dense(input_dim=ninputs, output_dim=out_neurons1,activation='sigmoid'\
                      ,kernel_regularizer=regularizers.l2(0.00000001))(inp1)
    m1 = Dropout(0.2)(m1)
    m1 = Dense(100,activation='sigmoid')(m1)
    #m1 = Dropout(0.2)(m1)
    #m1 = Dense(4, activation='sigmoid')(m1)

    #m2 = Dense(input_dim=n_feats, output_dim=n_feats,activation='relu')(inp2)
    m2 = Dense(50,activation='relu')(inp2)
    #m2=Dense(4,activation='relu')(m2)

    m3 = Dense(500, input_dim=n_tfidf, activation='relu',\
                    kernel_regularizer=regularizers.l2(reg))(inp3)

    m3 = Dropout(0.4)(m3)
    m3 = Dense(50, activation='relu')(m3)
    #m3 = Dropout(0.4)(m3)
    #m3 = Dense(4, activation='softmax')(m3)


    #m1 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='sigmoid')(m1)
    #m2 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='softmax')(m2)

    m = Merge(mode='concat')([m1,m2,m3])

    #mul = Multiply()([m1,m2])
    #add = Abs()([m1,m2])
    #m = Merge(mode='concat')([mul,add])

    score = Dense(output_dim=nclass,activation='softmax')(m)
    model = Model([inp1,inp2,inp3],score)
    model.compile(loss='categorical_crossentropy', optimizer='adam')
    return model

def prepare_model2(ninputs=9600, n_feats=45,nclass=4,n_tfidf=10001):
    inp1 = Input(shape=(ninputs,))
    inp2 = Input(shape=(n_feats,))
    inp3 = Input(shape=(n_tfidf,))
    reg = 0.00005
    out_neurons1 = 500
    #out_neurons2 = 20
    #out_neurons2 = 10
    m1 = Dense(input_dim=ninputs, output_dim=out_neurons1,activation='sigmoid'\
                      ,kernel_regularizer=regularizers.l2(0.00000001))(inp1)
    m1 = Dropout(0.2)(m1)
    m1 = Dense(100,activation='sigmoid')(m1)
    #m1 = Dropout(0.2)(m1)
    #m1 = Dense(4, activation='sigmoid')(m1)

    m2 = Dense(input_dim=n_feats, output_dim=n_feats,activation='relu')(inp2)
    m2 = Dense(4,activation='relu')(inp2)
    #m2=Dense(4,activation='relu')(m2)

    m3 = Dense(500, input_dim=n_tfidf, activation='relu',\
                    kernel_regularizer=regularizers.l2(reg))(inp3)

    m3 = Dropout(0.4)(m3)
    m3 = Dense(50, activation='relu')(m3)
    #m3 = Dropout(0.4)(m3)
    #m3 = Dense(4, activation='softmax')(m3)


    #m1 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='sigmoid')(m1)
    #m2 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='softmax')(m2)

    m = Merge(mode='concat')([m1,m2,m3])

    #mul = Multiply()([m1,m2])
    #add = Abs()([m1,m2])
    #m = Merge(mode='concat')([mul,add])

    score = Dense(output_dim=nclass,activation='softmax')(m)
    model = Model([inp1,inp2,inp3],score)
    model.compile(loss='categorical_crossentropy', optimizer='adam')
    return model

def skipgram_model(vocab_size, embedding_dim=100, paradigm='Functional'):
    # Sequential paradigm
    if paradigm == 'Sequential':
        target = Sequential()
        target.add(Embedding(vocab_size, embedding_dim, input_length=1))
        context = Sequential()
        context.add(Embedding(vocab_size, embedding_dim, input_length=1))

        # merge the pivot and context models
        model = Sequential()
        model.add(Merge([target, context], mode='dot'))
        model.add(Reshape((1,), input_shape=(1,1)))
        model.add(Activation('sigmoid'))
        model.compile(optimizer='adam', loss='binary_crossentropy')
        return model

    # Functional paradigm
    elif paradigm == 'Functional':
        target = Input(shape=(1,), name='target')
        context = Input(shape=(1,), name='context')
        #print target.shape, context.shape
        shared_embedding = Embedding(vocab_size, embedding_dim, input_length=1, name='shared_embedding')
        embedding_target = shared_embedding(target)
        embedding_context = shared_embedding(context)
        #print embedding_target.shape, embedding_context.shape

        merged_vector = dot([embedding_target, embedding_context], axes=-1)
        reshaped_vector = Reshape((1,), input_shape=(1,1))(merged_vector)
        #print merged_vector.shape
        prediction = Dense(1, input_shape=(1,), activation='sigmoid')(reshaped_vector)
        #print prediction.shape

        model = Model(inputs=[target, context], outputs=prediction)
        model.compile(optimizer='adam', loss='binary_crossentropy')
        return model

    else:
        print('paradigm error')
        return None

def load_model(self, frm_modelweights=''):

        first_model = Sequential()
        first_model.add(LSTM(2048,
                             input_shape=(m, self.feature_size),
                             return_sequences=False,
                             activation='relu',
                             name='fc1_left'))
        first_model.add(Dropout(0.5))
        first_model.add(LSTM(2048,
                             return_sequences=False,
                             activation='relu',
                             name='fc1_left'))

        second_model = Sequential()
        second_model.add(LSTM(2048,
                              input_shape=(n - m, self.feature_size),
                              return_sequences=False,
                              activation='relu',
                              name='fc1_right'))
        second_model.add(Dropout(0.5))
        second_model.add(LSTM(2048,
                              return_sequences=False,
                              activation='relu',
                              name='fc1_right'))

        model = Sequential()
        model.add(Merge([first_model, second_model], mode='concat'))
        model.add(Dense(4096, activation='relu', name='fc2'))
        model.add(Dropout(0.5))
        model.add(Dense(self.numclasses, activation='softmax', name='prediction'))

        if frm_modelweights:
            model.load_weights(frm_modelweights, by_name=True)
            print("Frame model loaded with weights from %s." % frm_modelweights)
        else:
            print "Empty frame model loaded."

        return model

def load_model(self, frm_modelweights='', frmdiff_modelweights=''):
        frm_model = Sequential()
        frm_model.add(GRU(4096,
                          return_sequences=True,
                          input_dim=self.feature_size,
                          input_length=MAX_FRAMES,
                          activation='relu',
                          name='fc1'))
        frm_model.add(Dropout(0.3))
        frm_model.add(GRU(4096,
                          return_sequences=False,
                          activation='relu',
                          name='fc2'))
        frm_model.add(Dropout(0.3))
        frm_model.add(Dense(self.numclasses, activation='softmax', name='frm_prediction'))
        if frm_modelweights:
            frm_model.load_weights(frm_modelweights, by_name=True)
            print("Frame model loaded with weights from %s." % frm_modelweights)
        else:
            print "Empty frame model loaded."

        '''
        frmdiff_model = Sequential()
        frmdiff_model.add(GRU(4096, input_dim=self.feature_size, activation='relu', name='fc1'))
        frmdiff_model.add(Dropout(0.3))
        frmdiff_model.add(GRU(4096, activation='relu', name='fc2'))
        frmdiff_model.add(Dropout(0.3))
        frmdiff_model.add(Dense(self.numclasses, activation='softmax', name='frmdiff_feature'))

        if frmdiff_modelweights:
            frmdiff_model.load_weights(frmdiff_modelweights, by_name=True)
            print('Frame model loaded with weights from %s.' % frmdiff_modelweights)
        else:
            print "Empty frame model loaded."

        model = Sequential()
        model.add(Merge([frm_model, frmdiff_model], mode='concat'))
        model.add(Dense(self.numclasses, activation='softmax', name='predictions'))
        '''

        return frm_model

def model(args):
# Image model
    model_image = Sequential()
    model_image.add(Reshape((args.img_vec_dim,), input_shape=(args.img_vec_dim,)))
    model_image.add(Dense(args.num_hidden_units_mlp))
    model_image.add(Activation(args.activation_1))
    model_image.add(Dropout(args.dropout))


    # Language Model
    model_language = Sequential()
    model_language.add(Embedding(args.vocabulary_size, args.word_emb_dim, input_length=args.max_ques_length))
    model_language.add(LSTM(args.num_hidden_units_lstm, return_sequences=True, input_shape=(args.max_ques_length, args.word_emb_dim)))
    model_language.add(LSTM(args.num_hidden_units_lstm, return_sequences=True))
    model_language.add(LSTM(args.num_hidden_units_lstm, return_sequences=False))
    model_language.add(Dense(args.num_hidden_units_mlp))
    model_language.add(Activation(args.activation_1))
    model_language.add(Dropout(args.dropout))


    # combined model
    model = Sequential()
    model.add(Merge([model_language, model_image], mode='mul'))

    # for _ in xrange(number_of_dense_layers):
    for i in xrange(args.num_hidden_layers_mlp):
        model.add(Dense(args.num_hidden_units_mlp))
        model.add(Activation(args.activation_1))
        model.add(Dropout(args.dropout))

    model.add(Dense(args.nb_classes))
    model.add(Activation(args.class_activation))


    return model

def model(args):

    # Image model
    model_image = Sequential()
    model_image.add(Reshape((args.img_vec_dim,), input_shape=(args.img_vec_dim,)))

    # Language Model
    model_language = Sequential()
    model_language.add(Embedding(args.vocabulary_size, args.word_emb_dim, input_length=args.max_ques_length))
    model_language.add(LSTM(args.num_hidden_units_lstm, return_sequences=True, input_shape=(args.max_ques_length, args.word_emb_dim)))
    model_language.add(LSTM(args.num_hidden_units_lstm, return_sequences=True))
    model_language.add(LSTM(args.num_hidden_units_lstm, return_sequences=False))

    # combined model
    model = Sequential()
    model.add(Merge([model_language, model_image], mode='concat', concat_axis=1))


    for i in xrange(args.num_hidden_layers_mlp):
        model.add(Dense(args.num_hidden_units_mlp))
        model.add(Dropout(args.dropout))

    model.add(Dense(args.nb_classes))
    model.add(Activation(args.class_activation))

    return model

def vqa_model(embedding_matrix, num_words, embedding_dim, seq_length, dropout_rate, num_classes):
    vgg_model = img_model(dropout_rate)
    lstm_model = Word2VecModel(embedding_matrix, num_words, embedding_dim, seq_length, dropout_rate)
    print "Merging final model..."
    fc_model = Sequential()
    fc_model.add(Merge([vgg_model, lstm_model], mode='mul'))
    fc_model.add(Dropout(dropout_rate))
    fc_model.add(Dense(1000, activation='tanh'))
    fc_model.add(Dropout(dropout_rate))
    fc_model.add(Dense(num_classes, activation='softmax'))
    fc_model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
        metrics=['accuracy'])
    return fc_model

def vis_lstm():
    embedding_matrix = embedding.load()
    embedding_model = Sequential()
    embedding_model.add(Embedding(
        embedding_matrix.shape[0],
        embedding_matrix.shape[1],
        weights = [embedding_matrix],
        trainable = False))

    image_model = Sequential()
    image_model.add(Dense(
        embedding_matrix.shape[1],
        input_dim=4096,
        activation='linear'))
    image_model.add(Reshape((1,embedding_matrix.shape[1])))

    main_model = Sequential()
    main_model.add(Merge(
        [image_model,embedding_model],
        mode = 'concat',        
        concat_axis = 1))
    main_model.add(LSTM(1001))
    main_model.add(Dropout(0.5))
    main_model.add(Dense(1001,activation='softmax'))

    return main_model

def vis_lstm_2():
    embedding_matrix = embedding.load()
    embedding_model = Sequential()
    embedding_model.add(Embedding(
        embedding_matrix.shape[0],
        embedding_matrix.shape[1],
        weights = [embedding_matrix],
        trainable = False))

    image_model_1 = Sequential()
    image_model_1.add(Dense(
        embedding_matrix.shape[1],
        input_dim=4096,
        activation='linear'))
    image_model_1.add(Reshape((1,embedding_matrix.shape[1])))

    image_model_2 = Sequential()
    image_model_2.add(Dense(
        embedding_matrix.shape[1],
        input_dim=4096,
        activation='linear'))
    image_model_2.add(Reshape((1,embedding_matrix.shape[1])))

    main_model = Sequential()
    main_model.add(Merge(
        [image_model_1,embedding_model,image_model_2],
        mode = 'concat',
        concat_axis = 1))
    main_model.add(LSTM(1001))
    main_model.add(Dropout(0.5))
    main_model.add(Dense(1001,activation='softmax'))

    return main_model

def generator_containing_discriminator_ae(generator, discriminator):
    model_left = Sequential()
    model_left.add(generator)
    discriminator.trainable = False
    model_left.add(discriminator)

    model_right = Sequential()
    model_right.add(generator)
    model_right.add(Reshape((784,)))

    model = Sequential()
    model.add(Merge([model_left, model_right], mode='concat', concat_axis=1))
    return model

def generator_containing_discriminator_ae(generator, discriminator):
    model_left = Sequential()
    model_left.add(generator)
    discriminator.trainable = False
    model_left.add(discriminator)

    model_right = Sequential()
    model_right.add(generator)
    model_right.add(Reshape((784,)))

    model = Sequential()
    model.add(Merge([model_left, model_right], mode='concat', concat_axis=1))
    return model

def generator_containing_discriminator_ae(self, generator, discriminator):
        model_left = Sequential()
        model_left.add(generator)
        discriminator.trainable = False
        model_left.add(discriminator)

        model_right = Sequential()
        model_right.add(generator)
        model_right.add(Reshape((144*144,)))

        model = Sequential()
        model.add(Merge([model_left, model_right], mode='concat', concat_axis=1))
        return model

def generator_containing_discriminator_ae(self, generator, discriminator):
        model_left = Sequential()
        model_left.add(generator)
        discriminator.trainable = False
        model_left.add(discriminator)

        model_right = Sequential()
        model_right.add(generator)
        model_right.add(Reshape((144*144,)))

        model = Sequential()
        model.add(Merge([model_left, model_right], mode='concat', concat_axis=1))
        return model

def generator_containing_discriminator_ae(generator, discriminator):
    model_left = Sequential()
    model_left.add(generator)
    discriminator.trainable = False
    model_left.add(discriminator)

    model_right = Sequential()
    model_right.add(generator)
    model_right.add(Reshape((784,)))

    model = Sequential()
    model.add(Merge([model_left, model_right], mode='concat', concat_axis=1))
    return model

def VQA_MODEL():
    image_feature_size          = 4096
    word_feature_size           = 300
    number_of_LSTM              = 3
    number_of_hidden_units_LSTM = 512
    max_length_questions        = 30
    number_of_dense_layers      = 3
    number_of_hidden_units      = 1024
    activation_function         = 'tanh'
    dropout_pct                 = 0.5


    # Image model
    model_image = Sequential()
    model_image.add(Reshape((image_feature_size,), input_shape=(image_feature_size,)))

    # Language Model
    model_language = Sequential()
    model_language.add(LSTM(number_of_hidden_units_LSTM, return_sequences=True, input_shape=(max_length_questions, word_feature_size)))
    model_language.add(LSTM(number_of_hidden_units_LSTM, return_sequences=True))
    model_language.add(LSTM(number_of_hidden_units_LSTM, return_sequences=False))

    # combined model
    model = Sequential()
    model.add(Merge([model_language, model_image], mode='concat', concat_axis=1))

    for _ in xrange(number_of_dense_layers):
        model.add(Dense(number_of_hidden_units, init='uniform'))
        model.add(Activation(activation_function))
        model.add(Dropout(dropout_pct))

    model.add(Dense(1000))
    model.add(Activation('softmax'))

    return model

def build_cnn_char_complex(input_dim, output_dim,nb_filter):
    randomEmbeddingLayer = Embedding(input_dim,32, input_length=maxlen,dropout=0.1)
    poolingLayer = Lambda(max_1d, output_shape=(nb_filter,))
    conv_filters = []
    for n_gram in range(2,4):
        ngramModel = Sequential()
        ngramModel.add(randomEmbeddingLayer)
        ngramModel.add(Convolution1D(nb_filter=nb_filter,
                                     filter_length=n_gram,
                                     border_mode="valid",
                                     activation="relu",
                                     subsample_length=1))
        ngramModel.add(poolingLayer)
        conv_filters.append(ngramModel)

    clf = Sequential()
    clf.add(Merge(conv_filters,mode="concat"))
    clf.add(Activation("relu"))
    clf.add(Dense(100))
    clf.add(Dropout(0.1))
    clf.add(Activation("tanh"))
    clf.add(Dense(output_dim=output_dim, activation='softmax'))

    clf.compile(optimizer='adagrad',
                     loss='categorical_crossentropy',
                     metrics=['accuracy'])
    return clf

def TextCNN(sequence_length, embedding_dim, filter_sizes, num_filters):
    ''' Convolutional Neural Network, including conv + pooling

    Args:
        sequence_length: ???????
        embedding_dim: ?????
        filter_sizes:  filter???
        num_filters: filter??

    Returns:
        features extracted by CNN
    '''
    graph_in = Input(shape=(sequence_length, embedding_dim))
    convs = []
    for fsz in filter_sizes:
        conv = Convolution1D(nb_filter=num_filters,
                         filter_length=fsz,
                         border_mode='valid',
                         activation='relu',
                         subsample_length=1)(graph_in)
        pool = MaxPooling1D()(conv)
        flatten = Flatten()(pool)
        convs.append(flatten)
    if len(filter_sizes)>1:
        out = Merge(mode='concat')(convs)
    else:
        out = convs[0]
    graph = Model(input=graph_in, output=out)
    return graph

def build_model(cat, hidden_dim):
    graph_in = Input(shape=(sequence_length, embedding_dim))
    convs = []
    for fsz in filter_sizes:
        conv = Convolution1D(nb_filter=num_filters,
                             filter_length=fsz,
                             border_mode='valid',
                             activation='relu',
                             subsample_length=1)(graph_in)
        pool = MaxPooling1D(pool_length=2)(conv)
        flatten = Flatten()(pool)
        convs.append(flatten)

    if len(filter_sizes)>1:
        out = Merge(mode='concat')(convs)
    else:
        out = convs[0]

    graph = Model(input=graph_in, output=out)

    # main sequential model
    model = Sequential()
    if not model_variation=='CNN-static':
        model.add(Embedding(len(vocabulary), embedding_dim, input_length=sequence_length,
                            weights=embedding_weights))
    model.add(Dropout(dropout_prob[0], input_shape=(sequence_length, embedding_dim)))
    model.add(graph)
    model.add(Dense(hidden_dim))
    model.add(Dropout(dropout_prob[1]))
    model.add(Activation('relu'))
    model.add(Dense(cat))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

def build_model(self, models, input_dims, output_dim):
        keras_models = [None] * len(models)
        for i, m in enumerate(models):
            m.build_model(input_dims[i], output_dim)
            keras_models[i] = m.model
        merged = Merge(keras_models, mode='concat')
        self.model.add(merged)
        self.model.add(Dense(output_dim, activation='softmax'))
        self.model.compile(optimizer='adam',
                           loss='categorical_crossentropy',
                           metrics=['accuracy'])

def create_model(self, ret_model = False):
        #base_model = VGG16(weights='imagenet', include_top=False, input_shape = (224, 224, 3))
        #base_model.trainable=False
        image_model = Sequential()
        #image_model.add(base_model)
        #image_model.add(Flatten())
        image_model.add(Dense(EMBEDDING_DIM, input_dim = 4096, activation='relu'))

        image_model.add(RepeatVector(self.max_cap_len))

        lang_model = Sequential()
        lang_model.add(Embedding(self.vocab_size, 256, input_length=self.max_cap_len))
        lang_model.add(LSTM(256,return_sequences=True))
        lang_model.add(TimeDistributed(Dense(EMBEDDING_DIM)))

        model = Sequential()
        model.add(Merge([image_model, lang_model], mode='concat'))
        model.add(LSTM(1000,return_sequences=False))
        model.add(Dense(self.vocab_size))
        model.add(Activation('softmax'))

        print "Model created!"

        if(ret_model==True):
            return model

        model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
        return model

def vis_lstm():
    embedding_matrix = embedding.load()
    embedding_model = Sequential()
    embedding_model.add(Embedding(
        embedding_matrix.shape[0],
        embedding_matrix.shape[1],
        weights = [embedding_matrix],
        trainable = False))

    image_model = Sequential()
    image_model.add(Dense(
        embedding_matrix.shape[1],
        input_dim=4096,
        activation='linear'))
    image_model.add(Reshape((1,embedding_matrix.shape[1])))

    main_model = Sequential()
    main_model.add(Merge(
        [image_model,embedding_model],
        mode = 'concat',        
        concat_axis = 1))
    main_model.add(LSTM(1001))
    main_model.add(Dropout(0.5))
    main_model.add(Dense(1001,activation='softmax'))

    return main_model

def vis_lstm_2():
    embedding_matrix = embedding.load()
    embedding_model = Sequential()
    embedding_model.add(Embedding(
        embedding_matrix.shape[0],
        embedding_matrix.shape[1],
        weights = [embedding_matrix],
        trainable = False))

    image_model_1 = Sequential()
    image_model_1.add(Dense(
        embedding_matrix.shape[1],
        input_dim=4096,
        activation='linear'))
    image_model_1.add(Reshape((1,embedding_matrix.shape[1])))

    image_model_2 = Sequential()
    image_model_2.add(Dense(
        embedding_matrix.shape[1],
        input_dim=4096,
        activation='linear'))
    image_model_2.add(Reshape((1,embedding_matrix.shape[1])))

    main_model = Sequential()
    main_model.add(Merge(
        [image_model_1,embedding_model,image_model_2],
        mode = 'concat',
        concat_axis = 1))
    main_model.add(LSTM(1001))
    main_model.add(Dropout(0.5))
    main_model.add(Dense(1001,activation='softmax'))

    return main_model

def create_cnn(W, max_length, dim=300,
               dropout=.5, output_dim=8):

    # Convolutional model
    filter_sizes=(2,3,4)
    num_filters = 3


    graph_in = Input(shape=(max_length, len(W[0])))
    convs = []
    for fsz in filter_sizes:
        conv = Convolution1D(nb_filter=num_filters,
                 filter_length=fsz,
                 border_mode='valid',
                 activation='relu',
                 subsample_length=1)(graph_in)
        pool = MaxPooling1D(pool_length=2)(conv)
        flatten = Flatten()(pool)
        convs.append(flatten)

    out = Merge(mode='concat')(convs)
    graph = Model(input=graph_in, output=out)

    # Full model
    model = Sequential()
    model.add(Embedding(output_dim=W.shape[1],
                        input_dim=W.shape[0],
                        input_length=max_length, weights=[W],
                        trainable=True))
    model.add(Dropout(dropout))
    model.add(graph)
    model.add(Dense(dim, activation='relu'))
    model.add(Dropout(dropout))
    model.add(Dense(output_dim, activation='softmax'))
    if output_dim == 2:
        model.compile('adam', 'binary_crossentropy',
                  metrics=['accuracy'])
    else:
        model.compile('adam', 'categorical_crossentropy',
                  metrics=['accuracy'])
    return model

    return model

def minst_attention(inc_noise=False, attention=True):
    #make layers
    inputs = Input(shape=(1,image_size,image_size),name='input')

    conv_1a = Convolution2D(32, 3, 3,activation='relu',name='conv_1')
    maxp_1a = MaxPooling2D((3, 3), strides=(2,2),name='convmax_1')
    norm_1a = crosschannelnormalization(name="convpool_1")
    zero_1a = ZeroPadding2D((2,2),name='convzero_1')

    conv_2a = Convolution2D(32,3,3,activation='relu',name='conv_2')
    maxp_2a = MaxPooling2D((3, 3), strides=(2,2),name='convmax_2')
    norm_2a = crosschannelnormalization(name="convpool_2")
    zero_2a = ZeroPadding2D((2,2),name='convzero_2')

    dense_1a = Lambda(global_average_pooling,output_shape=global_average_pooling_shape,name='dense_1')
    dense_2a = Dense(10, activation = 'softmax', init='uniform',name='dense_2')

    #make actual model
    if inc_noise:
        inputs_noise = noise.GaussianNoise(2.5)(inputs)
        input_pad = ZeroPadding2D((1,1),input_shape=(1,image_size,image_size),name='input_pad')(inputs_noise)
    else:
        input_pad = ZeroPadding2D((1,1),input_shape=(1,image_size,image_size),name='input_pad')(inputs)

    conv_1 = conv_1a(input_pad)
    conv_1 = maxp_1a(conv_1)
    conv_1 = norm_1a(conv_1)
    conv_1 = zero_1a(conv_1)

    conv_2_x = conv_2a(conv_1)
    conv_2 = maxp_2a(conv_2_x)
    conv_2 = norm_2a(conv_2)
    conv_2 = zero_2a(conv_2)
    conv_2 = Dropout(0.5)(conv_2)

    dense_1 = dense_1a(conv_2)
    dense_2 = dense_2a(dense_1)

    conv_shape1 = Lambda(change_shape1,output_shape=(32,),name='chg_shape')(conv_2_x)
    find_att = dense_2a(conv_shape1)

    if attention:
        find_att = Lambda(attention_control,output_shape=att_shape,name='att_con')([find_att,dense_2])
    else:
        find_att = Lambda(no_attention_control,output_shape=att_shape,name='att_con')([find_att,dense_2])

    zero_3a = ZeroPadding2D((1,1),name='convzero_3')(find_att)
    apply_attention  = Merge(mode='mul',name='attend')([zero_3a,conv_1])

    conv_3 = conv_2a(apply_attention)
    conv_3 = maxp_2a(conv_3)
    conv_3 = norm_2a(conv_3)
    conv_3 = zero_2a(conv_3)

    dense_3 = dense_1a(conv_3)
    dense_4 = dense_2a(dense_3)

    model = Model(input=inputs,output=dense_4)

    return model

def buildModel(loss_type,lamda):

    inpx = Input(shape=(dimx,))
    inpy = Input(shape=(dimy,))

    hx = Reshape((28,14,1))(inpx)
    hx = Conv2D(128, (3, 3), activation='relu', padding='same')(hx)
    hx = MaxPooling2D((2, 2), padding='same')(hx)
    hx = Conv2D(64, (3, 3), activation='relu', padding='same')(hx)
    hx = MaxPooling2D((2, 2), padding='same')(hx)
    hx = Conv2D(49, (3, 3), activation='relu', padding='same')(hx)
    hx = MaxPooling2D((2, 2), padding='same')(hx)
    hx = Flatten()(hx)
    hx1 = Dense(hdim_deep,activation='sigmoid')(hx)
    hx2 = Dense(hdim_deep2, activation='sigmoid',name='hid_l1')(hx1)
    hx = Dense(hdim, activation='sigmoid',name='hid_l')(hx2)

    hy = Reshape((28,14,1))(inpy)
    hy = Conv2D(128, (3, 3), activation='relu', padding='same')(hy)
    hy = MaxPooling2D((2, 2), padding='same')(hy)
    hy = Conv2D(64, (3, 3), activation='relu', padding='same')(hy)
    hy = MaxPooling2D((2, 2), padding='same')(hy)
    hy = Conv2D(49, (3, 3), activation='relu', padding='same')(hy)
    hy = MaxPooling2D((2, 2), padding='same')(hy)
    hy = Flatten()(hy)
    hy1 = Dense(hdim_deep,activation='sigmoid')(hy)
    hy2 = Dense(hdim_deep2, activation='sigmoid',name='hid_r1')(hy1)
    hy = Dense(hdim, activation='sigmoid',name='hid_r')(hy2)

    h =  Merge(mode="sum")([hx,hy]) 

    recx = Dense(dimx)(h)
    recy = Dense(dimy)(h)

    branchModel = Model( [inpx,inpy],[recx,recy,h,hx1,hy1,hx2,hy2])

    [recx1,recy1,h1,_,_,_,_] = branchModel( [inpx, ZeroPadding()(inpy)])
    [recx2,recy2,h2,_,_,_,_] = branchModel( [ZeroPadding()(inpx), inpy ])

    #you may probably add a reconstruction from combined
    [recx3,recy3,h3,hx_1,hy_1,hx_2,hy_2] = branchModel([inpx, inpy])

    lamda2,lamda3 = 0.001,0.05

    corr1=CorrnetCost(-lamda)([h1,h2])
    corr2=CorrnetCost(-lamda2)([hx_1,hy_1])
    corr3=CorrnetCost(-lamda3)([hx_2,hy_2])

    model = Model( [inpx,inpy],[recy1,recx2,recx1,recy2,corr1,corr2,corr3])
    model.compile( loss=["mse","mse","mse","mse",corr_loss,corr_loss,corr_loss],optimizer="rmsprop")

    return model, branchModel

def prepare_model(ninputs=9600,n_feats=47, nclass=5):
    """
    Set up and compile the model architecture (Logistic regression)
    """
    inp1 = Input(shape=(ninputs,))
    inp2 = Input(shape=(n_feats,))
    out_neurons1 = 50
    out_neurons2 = 20
    out_neurons2 = 10
    m1 = Dense(input_dim=ninputs, output_dim=out_neurons1,activation='sigmoid')(inp1)
    m2 = Dense(input_dim=ninputs, output_dim=out_neurons1,activation='softmax')(inp2)

    m1 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='sigmoid')(m1)
    m2 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='softmax')(m2)

    #m1 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='sigmoid')(m1)
    #m2 = Dense(input_dim=ninputs, output_dim=out_neurons2,activation='softmax')(m2)

    m = Merge(mode='concat')([m1,m2])

    #mul = Multiply()([m1,m2])
    #add = Abs()([m1,m2])
    #m = Merge(mode='concat')([mul,add])

    score = Dense(output_dim=nclass,activation='softmax')(m)
    model = Model([inp1,inp2],score)
    model.compile(loss='categorical_crossentropy', optimizer='adam')
    return model
    '''
    lrmodel = Sequential()
    lrmodel.add(Dense(input_dim=ninputs, output_dim=nclass))
    #lrmodel.add(Activation('softmax'))
    #lrmodel.compile(loss='categorical_crossentropy', optimizer='adam')

    #return lrmodel

    model_feat = Sequential()
    model_feat.add(Dense(input_dim=27, output_dim=nclass))
    merge_model = Sequential()
    merge_model.add(Merge([lrmodel, model_feat], mode='concat'))
    merge_model.add(Dense(output_dim=nclass))
    merge_model.add(Activation('softmax'))
    merge_model.compile(loss='categorical_crossentropy', optimizer='adam')
    return merge_model'''

    '''lrmodel.add(Dense(input_dim=ninputs, output_dim=1000,activation = 'relu'))
    lrmodel.add(Dropout(0.5))
    lrmodel.add(Dense(output_dim=500,activation = 'relu'))
    lrmodel.add(Dropout(0.5))
    lrmodel.add(Dense(output_dim=nclass))'''
    #return merge_model