I want to implement the model for Video Error concealment based on this paper: https://ieeexplore.ieee.org/document/8451090
I decode each three video frames to Numpy array with shape (1,3,48,352,1)
and use these three frames to Error Concealment. I use this code to generate datasets from video sources:
def dataset_generate(raw_data, num_frames_seq = 3, step_of_move_along_height = 16, data_width = 48):
dataset_X = []
label_X = []
for j in range(num_frames_seq,raw_data.shape[2]):
for i in range(0, raw_data.shape[0]-data_width+1, step_of_move_along_height):
b = range(i, i+data_width)
X_input = np.zeros((data_width,raw_data.shape[1],num_frames_seq))
X_label = np.zeros((data_width,raw_data.shape[1],1))
for t in range(num_frames_seq):
X_input [...,t] = raw_data[b,:,j+t-num_frames_seq]
X_label = raw_data[b,...,j]
dataset_X.append(X_input)
label_X.append (X_label)
dataset_X = np.asarray(dataset_X, dtype=np.float64)
label_X = np.asarray(label_X, dtype=np.float64)
dataset_X = np.transpose(np.expand_dims((dataset_X), axis =1), (0, 4, 2, 3, 1))
label_X= np.expand_dims(label_X, axis =-1)
return dataset_X, label_X
I use this code to create a Network to generate frames from previous frames and concealment the error.
model = Sequential()
model.add(ConvLSTM2D(filters=8,
kernel_size=(9,9),
input_shape = (3,48,352,1),
padding = 'same',
data_format='channels_last',
return_sequences=True))
model.add(Activation(activations.sigmoid))
model.add(BatchNormalization())
model.add(ConvLSTM2D(filters=16,
kernel_size=(5,5),
padding = 'same',
return_sequences = True))
model.add(Activation(activations.sigmoid))
model.add(BatchNormalization())
model.add(ConvLSTM2D(filters=32,
kernel_size=(3,3),
padding = 'same',
return_sequences = True))
model.add(Activation(activations.sigmoid))
model.add(BatchNormalization())
model.add(Conv3D(filters = 64,
kernel_size = (3,1,1),
activation='sigmoid',
padding='same',
data_format='channels_last'))
model.add(Activation(activations.sigmoid))
model.add(BatchNormalization())
model.add(Conv3D(filters = 32,
kernel_size = (1,3,3),
activation='sigmoid',
padding='same',
data_format='channels_last'))
model.add(Activation(activations.sigmoid))
model.add(BatchNormalization())
model.add(Conv3D(filters = 16,
kernel_size = (1,3,3),
activation='sigmoid',
padding='same',
data_format='channels_last'))
model.add(Activation(activations.sigmoid))
model.add(BatchNormalization())
model.add(Conv3D(filters = 1,
kernel_size = (1,1,1),
activation='sigmoid',
padding='same',
data_format='channels_last'))
model.add(Activation(activations.sigmoid))
model.add(BatchNormalization())
model.compile(loss='binary_crossentropy', optimizer='adadelta')
I use this segment of code to train the model:
available_ids = [i for i in range(0, 38944)]
shuffle(available_ids)
final_train_id = int(len(available_ids)*0.8)
train_ids = available_ids[:final_train_id]
val_ids = available_ids[final_train_id:]
model.fit(generate_array(train_ids),
steps_per_epoch = len(train_ids),
validation_data = generate_array(val_ids),
validation_steps = len(val_ids),
epochs = 30,
verbose = 1,
shuffle = False,
initial_epoch = 0)
where 38944 is the number of datasets and generate_array is:
def generate_array(available_ids):
while True:
#shuffle(available_ids)
for i in available_ids:
bFrame = np.load(r'\\dataset_{}.npy'.format(i))
aFrame = np.load(r'\\label_{}.npy'.format(i))
yield(bFrame, aFrame)
but I found this error:
Incompatible shapes: [1,48,352,1] vs. [1,3,48,352]