maskrcnn example is not training

24 views (last 30 days)
Sam Van Der Jeught
Sam Van Der Jeught on 17 Dec 2021
Commented: Thomas Cimiega on 17 Jul 2022 at 19:42
Hi all,
I'm trying to train a mask r-cnn network on my own data (two classes) and am therefore first trying out the example found on the mathworks website (https://www.mathworks.com/help/deeplearning/ug/instance-segmentation-using-mask-rcnn.html).
The code I'm using (unaltered from the website):
dataFolder = fullfile(tempdir,"coco");
trainedMaskRCNN_url = 'https://www.mathworks.com/supportfiles/vision/data/maskrcnn_object_person_car.mat';
helper.downloadTrainedMaskRCNN(trainedMaskRCNN_url,dataFolder);
pretrained = load(fullfile(dataFolder,'maskrcnn_object_person_car.mat'));
net = pretrained.net;
%%
imTest = imread('visionteam.jpg');
[masks,labels,scores,boxes] = segmentObjects(net,imTest);
overlayedImage = insertObjectMask(imTest,masks);
imshow(overlayedImage)
showShape("rectangle",gather(boxes),"Label",labels,"LineColor",'r')
%%
imageFolder = fullfile(dataFolder,"images");
captionsFolder = fullfile(dataFolder,"annotations");
if ~exist(imageFolder,'dir')
mkdir(imageFolder)
mkdir(captionsFolder)
end
annotationFile = fullfile(captionsFolder,"instances_train2014.json");
str = fileread(annotationFile);
trainClassNames = {'person', 'car'};
numClasses = length(trainClassNames);
imageSizeTrain = [800 800 3];
cocoAPIDir = fullfile(dataFolder,"cocoapi-master","MatlabAPI");
addpath(cocoAPIDir);
unpackAnnotationDir = fullfile(dataFolder,"annotations_unpacked","matFiles");
if ~exist(unpackAnnotationDir,'dir')
mkdir(unpackAnnotationDir)
end
helper.unpackAnnotations(trainClassNames,annotationFile,imageFolder,unpackAnnotationDir);
%%
ds = fileDatastore(unpackAnnotationDir, ...
'ReadFcn',@(x)helper.cocoAnnotationMATReader(x,imageFolder));
dsTrain = transform(ds,@(x)helper.preprocessData(x,imageSizeTrain));
data = preview(dsTrain)
%%
net = maskrcnn("resnet50-coco",trainClassNames,"InputSize",imageSizeTrain)
params = createMaskRCNNConfig(imageSizeTrain,numClasses,[trainClassNames {'background'}]);
params.ClassAgnosticMasks = false;
params.AnchorBoxes = net.AnchorBoxes;
params.FreezeBackbone = true;
initialLearnRate = 0.0012;
momentum = 0.9;
decay = 0.01;
velocity = [];
maxEpochs = 10;
miniBatchSize = 2;
miniBatchFcn = @(img,boxes,labels,masks) deal(cat(4,img{:}),boxes,labels,masks);
mbqTrain = minibatchqueue(dsTrain,4, ...
"MiniBatchFormat",["SSCB","","",""], ...
"MiniBatchSize",miniBatchSize, ...
"OutputCast",["single","","",""], ...
"OutputAsDlArray",[true,false,false,false], ...
"MiniBatchFcn",miniBatchFcn, ...
"OutputEnvironment",["auto","cpu","cpu","cpu"]);
%%
doTraining = true;
if doTraining
iteration = 1;
start = tic;
% Create subplots for the learning rate and mini-batch loss
fig = figure;
[lossPlotter, learningratePlotter] = helper.configureTrainingProgressPlotter(fig);
% Initialize verbose output
helper.initializeVerboseOutput([]);
% Custom training loop
for epoch = 1:maxEpochs
reset(mbqTrain)
shuffle(mbqTrain)
while hasdata(mbqTrain)
% Get next batch from minibatchqueue
[X,gtBox,gtClass,gtMask] = next(mbqTrain);
% Evaluate the model gradients and loss using dlfeval
[gradients,loss,state,learnables] = dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
%dlnet.State = state;
% Compute the learning rate for the current iteration
learnRate = initialLearnRate/(1 + decay*(epoch-1));
if(~isempty(gradients) && ~isempty(loss))
[net.AllLearnables,velocity] = sgdmupdate(learnables,gradients,velocity,learnRate,momentum);
else
continue;
end
% Plot loss/accuracy metric every 10 iterations
if(mod(iteration,10)==0)
helper.displayVerboseOutputEveryEpoch(start,learnRate,epoch,iteration,loss);
D = duration(0,0,toc(start),'Format','hh:mm:ss');
addpoints(learningratePlotter,iteration,learnRate)
addpoints(lossPlotter,iteration,double(gather(extractdata(loss))))
subplot(2,1,2)
title(strcat("Epoch: ",num2str(epoch),", Elapsed: "+string(D)))
drawnow
end
iteration = iteration + 1;
end
end
% Save the trained network
modelDateTime = string(datetime('now','Format',"yyyy-MM-dd-HH-mm-ss"));
save(strcat("trainedMaskRCNN-",modelDateTime,"-Epoch-",num2str(epoch),".mat"),'net');
end
When compiling, I get the following error:
Error using networkGradients
Too many output arguments.
Error in deep.internal.dlfeval (line 17)
[varargout{1:nargout}] = fun(x{:});
Error in dlfeval (line 40)
[varargout{1:nargout}] =
deep.internal.dlfeval(fun,varargin{:});
Error in person_car_example (line 80)
[gradients,loss,state,learnables] =
dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
so it seems something goes wrong in the line:
[gradients,loss,state,learnables] = dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
with the networkGradients file
I have used the networkGradients file from the example folder and have included it below:
function [gradients, totalLoss, state] = networkGradients(X, gTruthBoxes, gTruthLabels, gTruthMasks, dlnet, params)
% networkGradients - Gradient function to train MaskRCNN using a custom
% training loop.
% Copyright 2020 The MathWorks, Inc.
RPNRegDeltas = {'rpnConv1x1BoxDeltas'};regionProposal = {'rpl'};
outputNodes = [ RPNRegDeltas, regionProposal, dlnet.OutputNames(:)'];
% For training, the first step is to run a forward pass on the MaskRCNN
% network.
% We need the following outputs from the network to compute losses -
% YRPNReg - RPN regression output [1x1x4xN] {[deltaX deltaY deltaW deltaH]}
% YRPNClass - RPN classification output [1x1x2xN]
% YRCNNReg - RCNN regression output [1x1x4*(numClasses)xN] {[deltaX deltaY deltaW deltaH]}
% YRCNNClass - RCNN classification output [1x1x(numClasses+1)xN]
% YMask - RCNN Mask segmentation output [hxwxnumClassesxN]
% proposals - The regionproposals from the RPN. [5xN]
[YRPNRegDeltas, proposal, YRCNNClass, YRCNNReg, YRPNClass, YMask, state] = forward(...
dlnet, X, 'Outputs', outputNodes);
% If there are no proposals don't learn anything
if(isempty(proposal))
totalLoss = dlarray([]);
gradients= dlarray([]);
disp("Empty proposals");
return;
end
% Proposals are 5XNumProposals (Due to batch restrictions from custom RPL layer)
proposals = gather(extractdata(proposal));
% Convert proposals to numProposals x 5 (as expected by the rest of post processing code)
proposals =proposals';
proposals(:,1:4) = helper.boxUtils.x1y1x2y2ToXYWH(proposals(:,1:4));
numImagesInBatch = size(gTruthBoxes,1);
%Convert numProposalsx5 Proposals to numImagesInBatchx1 (Group by image index)
proposals = helper.groupProposalsByImageIndex(proposals, numImagesInBatch);
% Generate RCNN response targets
%--------------------------------
% Step 1: Match ground truth boxes to proposals
[assignment, positiveIndex, negativeIndex] = helper.bboxMatchAndAssign(...
proposals, gTruthBoxes,...
params.PositiveOverlapRange, params.NegativeOverlapRange,...
false);
% Step 2: Calcuate regression targets as (dx, dy, log(dw), log(dh))
regressionTargets = helper.generateRegressionTargets(gTruthBoxes, proposals,...
assignment, positiveIndex,...
params.NumClasses);
classNames = categories(gTruthLabels{1});
% Step 3: Assign groundtrutrh labels to proposals
classificationTargets = helper.generateClassificationTargets (gTruthLabels, assignment,...
positiveIndex, negativeIndex,...
classNames, params.BackgroundClass);
% Step 4: Calculate instance weights
instanceWeightsReg = helper.regressionResponseInstanceWeights (classificationTargets, params.BackgroundClass);
% Step 5: Generate mask targets
% Crop and resize the instances based on proposal bboxes and network output size
maskOutputSize = params.MaskOutputSize;
croppedMasks = helper.cropandResizeMasks (gTruthMasks, gTruthBoxes, maskOutputSize);
% Generate mask targets
maskTargets = helper.generateMaskTargets(croppedMasks, assignment, classificationTargets, params);
% Stage 2 (RCNN) Loss
% --------------------
% *Classification loss*
classificationTargets = cat(1, classificationTargets{:})';
% onehotencode labels
classificationTargets = onehotencode(classificationTargets,1);
classificationTargets(isnan(classificationTargets)) = 0;
LossRCNNClass = loss.CrossEntropy(YRCNNClass, classificationTargets);
% *Weighted regression loss*
regressionTargets = cat(1,regressionTargets{:});
instanceWeightsReg = cat(1, instanceWeightsReg{:});
LossRCNNReg = loss.smoothL1(YRCNNReg, single(regressionTargets'), single(instanceWeightsReg'));
% Mask Loss (Weighted cross entropy)
maskTargets= cat(4,maskTargets{:});
positiveIndex = cat(1,positiveIndex{:});
LossRCNNMask = loss.SpatialCrossEntropy(YMask, single(maskTargets), positiveIndex);
% Total Stage 2 loss
LossRCNN = LossRCNNReg + LossRCNNClass + LossRCNNMask;
% Generate RCNN response targets
%--------------------------------
featureSize = size(YRPNRegDeltas);
imageSize = params.ImageSize;
[RPNRegressionTargets, RPNRegWeights, assignedLabelsRPN] = helper.rpnRegressionResponse(featureSize, gTruthBoxes, imageSize, params);
RPNClassificationTargets = onehotencode(assignedLabelsRPN, 3);
RPNClassificationTargets(isnan(RPNClassificationTargets)) = 0;
% Stage 1 (RPN) Loss
% --------------------
LossRPNClass = loss.CrossEntropy(YRPNClass, RPNClassificationTargets);
LossRPNReg = loss.smoothL1(YRPNRegDeltas, RPNRegressionTargets, RPNRegWeights);
LossRPN = LossRPNClass + LossRPNReg;
% Total Loss
%------------
totalLoss = LossRCNN + LossRPN;
gradients = dlgradient(totalLoss, dlnet.Learnables);
end
It seems the "learnables" ouput is one too many, so I remove it but get the following error:
Unrecognized method, property, or field 'OutputNames' for class 'maskrcnn'.
Error in networkGradients (line 10)
outputNodes = [ RPNRegDeltas, regionProposal, dlnet.OutputNames(:)'];
Error in deep.internal.dlfeval (line 17)
[varargout{1:nargout}] = fun(x{:});
Error in dlfeval (line 40)
[varargout{1:nargout}] = deep.internal.dlfeval(fun,varargin{:});
Error in person_car_example (line 80)
[gradients,loss,state] = dlfeval(@networkGradients,X,gtBox,gtClass,gtMask,net,params);
The maskrcnn structure does not have an 'OutputNames' field... It seems likethe entire networkGradients file is incompatible with the example.
Any ideas/help would be very welcome!
Thanks,
Sam
  3 Comments
Thomas Cimiega
Thomas Cimiega on 17 Jul 2022 at 19:42
Unfortunately not. It seems that Matlab is not able to provide a working example... Instead I had to write an own example by using other cnn examples and reading the documentations.

Sign in to comment.

Answers (2)

David Willingham
David Willingham on 20 Dec 2021
Hi Sam,
What version of MATLAB are you running?
David
  1 Comment
Sam Van Der Jeught
Sam Van Der Jeught on 20 Dec 2021
Hi David,
I'm using Matlab 2021b, but I've also tried it with 2020b.
In the meantime I have rewritten the networkGradients.m file as follows and now it works. Hope someone else can benefit from this.
function [gradients, totalLoss, state, learnables] = networkGradients(X, gTruthBoxes, gTruthLabels, gTruthMasks, dlnet, params)
% networkGradients - Gradient function to train MaskRCNN using a custom
% training loop.
% Copyright 2020 The MathWorks, Inc.
%RPNRegDeltas = {'rpnConv1x1BoxDeltas'};regionProposal = {'rpl'};
%outputNodes = [ RPNRegDeltas, regionProposal, dlnet.OutputNames(:)'];
% For training, the first step is to run a forward pass on the MaskRCNN
% network.
% We need the following outputs from the network to compute losses -
% YRPNReg - RPN regression output [1x1x4xN] {[deltaX deltaY deltaW deltaH]}
% YRPNClass - RPN classification output [1x1x2xN]
% YRCNNReg - RCNN regression output [1x1x4*(numClasses)xN] {[deltaX deltaY deltaW deltaH]}
% YRCNNClass - RCNN classification output [1x1x(numClasses+1)xN]
% YMask - RCNN Mask segmentation output [hxwxnumClassesxN]
% % proposals - The regionproposals from the RPN. [5xN]
% [YRPNRegDeltas, proposal, YRCNNClass, YRCNNReg, YRPNClass, YMask, state] = forward(...
% dlnet, X, 'Outputs', outputNodes);
[netOut, state] = forward(dlnet, X);
YRPNClass = netOut{1};
YRPNRegDeltas = netOut{2};
proposal = netOut{3};
YRCNNClass = netOut{4};
YRCNNReg = netOut{5};
YMask = netOut{6};
%save proposal proposal
% If there are no proposals don't learn anything
if(isempty(proposal))
totalLoss = dlarray([]);
gradients= dlarray([]);
disp("Empty proposals");
return;
end
% Proposals are 5XNumProposals (Due to batch restrictions from custom RPL layer)
proposals = gather(extractdata(proposal));
% Convert proposals to numProposals x 5 (as expected by the rest of post processing code)
proposals =proposals';
proposals(:,1:4) = helper.boxUtils.x1y1x2y2ToXYWH(proposals(:,1:4));
numImagesInBatch = size(gTruthBoxes,1);
%Convert numProposalsx5 Proposals to numImagesInBatchx1 (Group by image index)
proposals = helper.groupProposalsByImageIndex(proposals, numImagesInBatch);
% Generate RCNN response targets
%--------------------------------
% Step 1: Match ground truth boxes to proposals
[assignment, positiveIndex, negativeIndex] = helper.bboxMatchAndAssign(...
proposals, gTruthBoxes,...
params.PositiveOverlapRange, params.NegativeOverlapRange,...
false);
%save assignment assignment
% Step 2: Calcuate regression targets as (dx, dy, log(dw), log(dh))
regressionTargets = helper.generateRegressionTargets(gTruthBoxes, proposals,...
assignment, positiveIndex,...
params.NumClasses);
% classNames = categories(gTruthLabels{1}); % !!! TROUBLE, change as below
classNames = params.ClassNames;
% Step 3: Assign groundtrutrh labels to proposals
classificationTargets = helper.generateClassificationTargets (gTruthLabels, assignment,...
positiveIndex, negativeIndex,...
classNames, params.BackgroundClass);
% Step 4: Calculate instance weights
instanceWeightsReg = helper.regressionResponseInstanceWeights (classificationTargets, params.BackgroundClass);
% Step 5: Generate mask targets
% Crop and resize the instances based on proposal bboxes and network output size
maskOutputSize = params.MaskOutputSize;
croppedMasks = helper.cropandResizeMasks (gTruthMasks, gTruthBoxes, maskOutputSize);
% Generate mask targets
maskTargets = helper.generateMaskTargets(croppedMasks, assignment, classificationTargets, params);
% Stage 2 (RCNN) Loss
% --------------------
% *Classification loss*
classificationTargets = cat(1, classificationTargets{:})';
summary(classificationTargets)
% onehotencode labels
classificationTargets = onehotencode(classificationTargets,1);
classificationTargets(isnan(classificationTargets)) = 0;
classificationTargets = reshape(classificationTargets ,1, 1, size(YRCNNClass,3),[]);
LossRCNNClass = loss.CrossEntropy(YRCNNClass, classificationTargets);
% *Weighted regression loss*
regressionTargets = cat(1,regressionTargets{:})';
instanceWeightsReg = cat(1, instanceWeightsReg{:})';
regressionTargets = reshape(regressionTargets, 1, 1, size(YRCNNReg,3),[]);
instanceWeightsReg = reshape(instanceWeightsReg, 1, 1, size(YRCNNReg,3),[]);
LossRCNNReg = loss.smoothL1(YRCNNReg, single(regressionTargets), single(instanceWeightsReg));
% Mask Loss (Weighted cross entropy)
maskTargets= cat(4,maskTargets{:});
positiveIndex = cat(1,positiveIndex{:});
LossRCNNMask = loss.SpatialCrossEntropy(YMask, single(maskTargets), positiveIndex);
% Total Stage 2 loss
LossRCNN = LossRCNNReg + LossRCNNClass + LossRCNNMask;
% Generate RCNN response targets
%--------------------------------
featureSize = size(YRPNRegDeltas);
imageSize = params.ImageSize;
[RPNRegressionTargets, RPNRegWeights, assignedLabelsRPN] = helper.rpnRegressionResponse(featureSize, gTruthBoxes, imageSize, params);
RPNClassificationTargets = onehotencode(assignedLabelsRPN, 3);
% detetron2 uses ony foreground class for classification.
RPNClassificationTargets(:,:,2,:) = [];
RPNClassificationTargets = reshape(RPNClassificationTargets, featureSize(1), featureSize(2), [], numImagesInBatch );
RPNClassificationTargets(isnan(RPNClassificationTargets)) = 0;
% Stage 1 (RPN) Loss
% --------------------
YRPNClass = softmax(YRPNClass);
LossRPNClass = loss.CrossEntropy(YRPNClass, RPNClassificationTargets);
LossRPNReg = loss.smoothL1(YRPNRegDeltas, RPNRegressionTargets, RPNRegWeights);
LossRPN = LossRPNClass + LossRPNReg;
% Total Loss
%------------
totalLoss = LossRCNN + LossRPN;
if(params.FreezeBackbone)
learnables = rmfield(dlnet.AllLearnables, {'FeatureExtractionNet','PostPoolFeatureExtractionNet'});
else
learnables = dlnet.AllLearnables;
end
gradients = dlgradient(totalLoss, learnables);
end

Sign in to comment.


Zakariya Abousabie
Zakariya Abousabie on 5 Jun 2022
hi there does anybody find solution to these problem, becuse im trying to train maskrcnn with my on data of two classes but im stuck too and the matworks example doesn't help

Community Treasure Hunt

Find the treasures in MATLAB Central and discover how the community can help you!

Start Hunting!