From 456cf086af521afe9720344bad97bd1c665228df Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Thu, 9 Jul 2015 11:57:05 +0200 Subject: [PATCH 01/19] Initial version of Fast-RCNN. Work in progress --- FRCNN.lua | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ ROIPooling.lua | 51 ++++++++++++++++++++++++++++++++++++++ data.lua | 2 ++ opts.lua | 1 + test.lua | 13 ++++++++++ 5 files changed, 134 insertions(+) create mode 100644 FRCNN.lua create mode 100644 ROIPooling.lua create mode 100644 test.lua diff --git a/FRCNN.lua b/FRCNN.lua new file mode 100644 index 0000000..8ab8527 --- /dev/null +++ b/FRCNN.lua @@ -0,0 +1,67 @@ +local FRCNN = torch.class('nnf.FRCNN') + +function FRCNN:__init(dataset) + self.dataset = dataset + + self.scale = {600} + self.max_dim = 1000 + self.randomscale = true + + --self.sz_conv_standard = 13 + self.step_standard = 16 + --self.offset0 = 21 + --self.offset = 6.5 + + --self.inputArea = 224^2 + +end + +local function rgb2bgr(I) + local out = I.new():resizeAs(I) + for i=1,I:size(1) do + out[i] = I[I:size(1)+1-i] + end + return out +end + +local function prepareImage(I,typ) + local typ = typ or 1 + local mean_pix = typ == 1 and {128.,128.,128.} or {103.939, 116.779, 123.68} + local I = I + if I:dim() == 2 then + I = I:view(1,I:size(1),I:size(2)) + end + if I:size(1) == 1 then + I = I:expand(3,I:size(2),I:size(3)) + end + I = rgb2bgr(I):mul(255) + for i=1,3 do + I[i]:add(-mean_pix[i]) + end + return I +end + +function FRCNN:getScale(I) + local min_size = math.min(I[2],I[3]) + local max_size = math.max(I[2],I[3]) + local scale + if max_size <= self.max_dim then + scale = self.scale[1]/min_size + else + scale = self.max_dim/max_size + end + return scale +end + +function FRCNN:projectBBoxes(bboxes,scale) + return (bboxes-1)*scale+1 +end + +function FRCNN:getFeatures(i,flip) + local I = self.dataset:getImage(i) + local bboxes = self.dataset:attachProposals(i) + I = prepareImage(I) + if flip then + + end +end diff --git a/ROIPooling.lua b/ROIPooling.lua new file mode 100644 index 0000000..23a5606 --- /dev/null +++ b/ROIPooling.lua @@ -0,0 +1,51 @@ +local ROIPooling,parent = torch.class('nnf.ROIPooling','nn.Module') + +function ROIPooling:__init(W,H) + parent.__init(self) + self.W = W + self.H = H + self.pooler = {}--nn.SpatialAdaptiveMaxPooling(W,H) +end + +-- not for batches for the moment +function ROIPooling:updateOutput(input) + local data = input[1] + local rois = input[2] + local num_rois = rois:size(1) + local s = data:size() + local ss = s:size(1) + self.output:resize(num_rois,s[ss-2],self.H,self.W) + + if #self.pooler < num_rois then + local diff = num_rois - #self.pooler + for i=1,diff do + table.insert(self.pooler,nn.SpatialAdaptiveMaxPooling(self.W,self.H)) + end + end + + for i=1,num_rois do + local roi = rois[i] + local im = data[{{},{roi[2],roi[4]},{roi[1],roi[3]}}] + self.output[i] = self.pooler[i]:forward(im) + end + return self.output +end + +function ROIPooling:updateGradInput(input,gradOutput) + local data = input[1] + local rois = input[2] + local num_rois = rois:size(1) + local s = data:size() + local ss = s:size(1) + self.gradInput:resizeAs(data):zero() + + for i=1,num_rois do + local roi = rois[i] + local r = {{},{roi[2],roi[3]},{roi[1],roi[3]}} + local im = data[r] + local g = self.pooler[i]:backward(im,gradOutput[i]) + self.gradInput[r]:add(g) + end + return self.gradInput + +end diff --git a/data.lua b/data.lua index 211f9a4..8d59522 100644 --- a/data.lua +++ b/data.lua @@ -51,6 +51,7 @@ else batch_provider = nnf.BatchProvider(feat_provider) batch_provider.iter_per_batch = opt.ipb batch_provider.nTimesMoreData = opt.ntmd + batch_provider.batch_size = opt.batch_size batch_provider.fg_fraction = opt.fg_frac batch_provider.bg_threshold = {0.0,0.5} batch_provider.do_flip = true @@ -89,6 +90,7 @@ else batch_provider_test = nnf.BatchProvider(feat_provider_test) batch_provider_test.iter_per_batch = 500--opt.ipb batch_provider_test.nTimesMoreData = 10--opt.ntmd + batch_provider_test.batch_size = opt.batch_size batch_provider_test.fg_fraction = opt.fg_frac batch_provider_test.bg_threshold = {0.0,0.5} batch_provider_test.do_flip = false diff --git a/opts.lua b/opts.lua index 68be1dd..4009956 100644 --- a/opts.lua +++ b/opts.lua @@ -36,6 +36,7 @@ function M.parse(arg) cmd:option('-nsmooth',10,'number of iterations before reducing learning rate') cmd:option('-nred',4,'number of divisions by 2 before stopping learning') cmd:option('-nildfdx',false,'erase memory of gradients when reducing learning rate') + cmd:option('-batch_size',128,'batch size') cmd:text() cmd:text('Others') cmd:option('-gpu',1,'gpu device to use') diff --git a/test.lua b/test.lua new file mode 100644 index 0000000..6057a07 --- /dev/null +++ b/test.lua @@ -0,0 +1,13 @@ +require 'nn' +nnf = {} +dofile 'ROIPooling.lua' + +m = nnf.ROIPooling(3,3) + +t = {torch.rand(1,10,10),torch.Tensor({{1,1,5,5},{2,3,7,8},{6,4,8,8},{6,4,10,10},{8,8,10,10}})} -- +g = torch.rand(t[2]:size(1),1,3,3) + +o = m:forward(t) +gg = m:backward(t,g) + + From d6a5caae7a68045cd0e72f9e88c09f008d14da55 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Fri, 7 Aug 2015 09:36:11 +0200 Subject: [PATCH 02/19] Add ROIDataLayer, still incomplete --- ROIDataLayer.lua | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 ROIDataLayer.lua diff --git a/ROIDataLayer.lua b/ROIDataLayer.lua new file mode 100644 index 0000000..c13a22c --- /dev/null +++ b/ROIDataLayer.lua @@ -0,0 +1,62 @@ +local ROIDataLayer,parent = torch.class('nnf.ROIDataLayer','nnf.BatchProvider') + +function ROIDataLayer:__init(dataset) + parent.__init(self) + self.dataset = dataset + self.image_transformer + self.imgs_per_batch = 2 + self.scale = 600 + self.max_size = 1000 +end + +local function shuffle_roidb_inds(self) + self._perm = torch.randperm(self.dataset:size()) + self._curr = 0 +end + +local function get_next_minibatch_inds(self) + if self._cur + cfg.TRAIN.IMS_PER_BATCH >= len(self._roidb) then + self:shuffle_roidb_inds() + end + + local db_inds = self._perm[{{self._cur,self._cur + self.imgs_per_batch}}] + self._cur = self._cur + self.imgs_per_batch + return db_inds +end + + +function ROIDataLayer:getBatch() + local dataset = self.dataset + local img_ids = self:get_next_minibatch_inds() + + local num_images = img_ids:size(1) + local imgs = {} + local im_sizes = {} + -- get images + -- prep_im_for_blob + for i=1,num_images do + local im = dataset:getImage(img_ids[i]) + im = self.image_transformer:preprocess(im) + local im_size = im[1]:size() + local im_size_min = math.min(im_size[1],im_size[2]) + local im_size_max = math.max(im_size[1],im_size[2]) + local im_scale = self.scale/im_size_min + if torch.round(im_scale*im_size_max) > self.max_size then + im_scale = self.max_size/im_size_max + end + local im_s = {im_size[1]*im_scale,im_size[2]*im_scale} + table.insert(imgs,image.scale(im,im_s[2],im_s[1])) + table.insert(im_sizes,im_s) + end + -- create single tensor with all images, padding with zero for different sizes + im_sizes = torch.IntTensor(im_sizes) + local max_shape = im_sizes:max(1) + local images = torch.FloatTensor(num_images,3,max_shape[1],max_shape[2]) + for i=1,num_images do + images[i][{{1,imgs[i]:size(2)},{imgs[i]:size(3)}}]:copy(imgs[i]) + end + + return images +end + + From 722c433826c9ca760eb9504716701c662e4aa2ac Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Thu, 13 Aug 2015 11:03:26 +0200 Subject: [PATCH 03/19] Commiting what we have for the moment --- FRCNN.lua | 25 ------------------------- ROIDataLayer.lua | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/FRCNN.lua b/FRCNN.lua index 8ab8527..6fd8061 100644 --- a/FRCNN.lua +++ b/FRCNN.lua @@ -16,31 +16,6 @@ function FRCNN:__init(dataset) end -local function rgb2bgr(I) - local out = I.new():resizeAs(I) - for i=1,I:size(1) do - out[i] = I[I:size(1)+1-i] - end - return out -end - -local function prepareImage(I,typ) - local typ = typ or 1 - local mean_pix = typ == 1 and {128.,128.,128.} or {103.939, 116.779, 123.68} - local I = I - if I:dim() == 2 then - I = I:view(1,I:size(1),I:size(2)) - end - if I:size(1) == 1 then - I = I:expand(3,I:size(2),I:size(3)) - end - I = rgb2bgr(I):mul(255) - for i=1,3 do - I[i]:add(-mean_pix[i]) - end - return I -end - function FRCNN:getScale(I) local min_size = math.min(I[2],I[3]) local max_size = math.max(I[2],I[3]) diff --git a/ROIDataLayer.lua b/ROIDataLayer.lua index c13a22c..9ba9eef 100644 --- a/ROIDataLayer.lua +++ b/ROIDataLayer.lua @@ -32,6 +32,7 @@ function ROIDataLayer:getBatch() local num_images = img_ids:size(1) local imgs = {} local im_sizes = {} + local im_scales = {} -- get images -- prep_im_for_blob for i=1,num_images do @@ -47,6 +48,7 @@ function ROIDataLayer:getBatch() local im_s = {im_size[1]*im_scale,im_size[2]*im_scale} table.insert(imgs,image.scale(im,im_s[2],im_s[1])) table.insert(im_sizes,im_s) + table.insert(im_scales,im_scale) end -- create single tensor with all images, padding with zero for different sizes im_sizes = torch.IntTensor(im_sizes) @@ -56,7 +58,32 @@ function ROIDataLayer:getBatch() images[i][{{1,imgs[i]:size(2)},{imgs[i]:size(3)}}]:copy(imgs[i]) end + return images end +local function sample_rois(self,i) + + local dataset = self.dataset + local rec = dataset:attachProposals(i) + local fg_inds = {} + local bg_inds = {} + for j=1,rec:size() do + local id = rec.label[j] + local is_fg = (rec.overlap[j] >= self.fg_threshold) + local is_bg = (rec.overlap[j] >= self.bg_threshold[1] and + rec.overlap[j] < self.bg_threshold[2]) + if is_fg then + table.insert(fg_inds,j) + elseif is_bg then + table.insert(bg_inds,j) + end + end + + local fg_rois_per_this_image = math.min(#fg_inds,self.fg_) + if #fg_inds > 0 then + + end + +end From 58074c5090bb8d1ccf23f371e273769094e18f87 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Thu, 13 Aug 2015 17:22:52 +0200 Subject: [PATCH 04/19] Updating BatchProviderROI --- BatchProviderROI.lua | 141 +++++++++++++++++++++++++++++++++++++++++++ ROIDataLayer.lua | 89 --------------------------- nnf.lua | 5 +- 3 files changed, 144 insertions(+), 91 deletions(-) create mode 100644 BatchProviderROI.lua delete mode 100644 ROIDataLayer.lua diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua new file mode 100644 index 0000000..6f28865 --- /dev/null +++ b/BatchProviderROI.lua @@ -0,0 +1,141 @@ +local BatchProviderROI, parent = torch.class('nnf.BatchProviderROI','nnf.BatchProvider') + +function BatchProviderROI:__init(dataset) + local fp = {dataset=dataset} + parent:__init(fp) + self.imgs_per_batch = 2 + self.scale = 600 + self.max_size = 1000 + self.image_transformer = nnf.ImageTransformer{} +end + +-- setup is the same + +function BatchProviderROI:permuteIdx() + local fg_num_each = self.fg_num_each + local bg_num_each = self.bg_num_each + local fg_num_total = self.fg_num_total + local bg_num_total = self.bg_num_total + local total_img = self.dataset:size() + local imgs_per_batch = self.imgs_per_batch + + self._cur = self._cur or math.huge + + if self._cur + imgs_per_batch > total_img then + self._perm = torch.randperm(total_img) + self._cur = 1 + end + + local img_idx = self._perm[{{self._cur,self._cur + self.imgs_per_batch - 1}}] + self._cur = self._cur + self.imgs_per_batch + + local img_idx_end = imgs_per_batch + --[[ + local fg_windows = {} + local bg_windows = {} + for i=1,img_idx_end do + local curr_idx = img_idx[i] + bg_windows[i] = {} + if self.bboxes[curr_idx][0] then + for j=1,self.bboxes[curr_idx][0]:size(1) do + table.insert(bg_windows[i],{curr_idx,j}) + end + end + fg_windows[i] = {} + if self.bboxes[curr_idx][1] then + for j=1,self.bboxes[curr_idx][1]:size(1) do + table.insert(fg_windows[i],{curr_idx,j}) + end + end + end + --]] + local opts = {img_idx=img_idx,img_idx_end=img_idx_end} + return fg_windows,bg_windows,opts + +end + +function BatchProviderROI:selectBBoxes(fg_windows,bg_windows) + local fg_w = {} + local bg_w = {} + + for im=1,self.imgs_per_batch do + + fg_w[im] = {} + bg_w[im] = {} + + local window_idx = torch.randperm(#bg_windows[im]) + for i=1,math.min(self.bg_num_each,#bg_windows[im]) do + local curr_idx = bg_windows[im][window_idx[i] ][1] + local position = bg_windows[im][window_idx[i] ][2] + local dd = self.bboxes[curr_idx][0][position] + table.insert(bg_w[im],dd) + end + + window_idx = torch.randperm(#fg_windows[im]) + for i=1,math.min(self.fg_num_each,#fg_windows[im]) do + local curr_idx = fg_windows[im][window_idx[i] ][1] + local position = fg_windows[im][window_idx[i] ][2] + local dd = self.bboxes[curr_idx][1][position] + table.insert(fg_w[im],dd) + end + + end + + return fg_w,bg_w +end + +local function getImages(self,img_ids,images) + local dataset = self.dataset + local num_images = img_ids:size(1) + + local imgs = {} + local im_sizes = {} + local im_scales = {} + + for i=1,num_images do + local im = dataset:getImage(img_ids[i]) + im = self.image_transformer:preprocess(im) + local im_size = im[1]:size() + local im_size_min = math.min(im_size[1],im_size[2]) + local im_size_max = math.max(im_size[1],im_size[2]) + local im_scale = self.scale/im_size_min + if torch.round(im_scale*im_size_max) > self.max_size then + im_scale = self.max_size/im_size_max + end + local im_s = {im_size[1]*im_scale,im_size[2]*im_scale} + table.insert(imgs,image.scale(im,im_s[2],im_s[1])) + table.insert(im_sizes,im_s) + table.insert(im_scales,im_scale) + end + -- create single tensor with all images, padding with zero for different sizes + im_sizes = torch.IntTensor(im_sizes) + local max_shape = im_sizes:max(1)[1] + images:resize(num_images,3,max_shape[1],max_shape[2]) + for i=1,num_images do + images[i][{{},{1,imgs[i]:size(2)},{1,imgs[i]:size(3)}}]:copy(imgs[i]) + end + return im_scales +end + + +function BatchProviderROI:getBatch(batches,targets) + local dataset = self.dataset + + self.fg_num_each = self.fg_fraction * self.batch_size + self.bg_num_each = self.batch_size - self.fg_num_each + --self.fg_num_total = self.fg_num_each * self.iter_per_batch + --self.bg_num_total = self.bg_num_each * self.iter_per_batch + + local fg_windows,bg_windows,opts = self:permuteIdx() + --local fg_w,bg_w = self:selectBBoxes(fg_windows,bg_windows) + + local batches = batches or {torch.FloatTensor(),torch.FloatTensor()} + local targets = targets or torch.IntTensor() + + -- batches[1]:resize(self.batch_size,unpack(self.batch_dim)) + local im_scales = getImages(self,opts.img_idx,batches[1]) + batches[2]:resize(self.batch_size,unpack(self.batch_dim)) + targets:resize(self.batch_size,self.target_dim) + + return batches, targets +end diff --git a/ROIDataLayer.lua b/ROIDataLayer.lua deleted file mode 100644 index 9ba9eef..0000000 --- a/ROIDataLayer.lua +++ /dev/null @@ -1,89 +0,0 @@ -local ROIDataLayer,parent = torch.class('nnf.ROIDataLayer','nnf.BatchProvider') - -function ROIDataLayer:__init(dataset) - parent.__init(self) - self.dataset = dataset - self.image_transformer - self.imgs_per_batch = 2 - self.scale = 600 - self.max_size = 1000 -end - -local function shuffle_roidb_inds(self) - self._perm = torch.randperm(self.dataset:size()) - self._curr = 0 -end - -local function get_next_minibatch_inds(self) - if self._cur + cfg.TRAIN.IMS_PER_BATCH >= len(self._roidb) then - self:shuffle_roidb_inds() - end - - local db_inds = self._perm[{{self._cur,self._cur + self.imgs_per_batch}}] - self._cur = self._cur + self.imgs_per_batch - return db_inds -end - - -function ROIDataLayer:getBatch() - local dataset = self.dataset - local img_ids = self:get_next_minibatch_inds() - - local num_images = img_ids:size(1) - local imgs = {} - local im_sizes = {} - local im_scales = {} - -- get images - -- prep_im_for_blob - for i=1,num_images do - local im = dataset:getImage(img_ids[i]) - im = self.image_transformer:preprocess(im) - local im_size = im[1]:size() - local im_size_min = math.min(im_size[1],im_size[2]) - local im_size_max = math.max(im_size[1],im_size[2]) - local im_scale = self.scale/im_size_min - if torch.round(im_scale*im_size_max) > self.max_size then - im_scale = self.max_size/im_size_max - end - local im_s = {im_size[1]*im_scale,im_size[2]*im_scale} - table.insert(imgs,image.scale(im,im_s[2],im_s[1])) - table.insert(im_sizes,im_s) - table.insert(im_scales,im_scale) - end - -- create single tensor with all images, padding with zero for different sizes - im_sizes = torch.IntTensor(im_sizes) - local max_shape = im_sizes:max(1) - local images = torch.FloatTensor(num_images,3,max_shape[1],max_shape[2]) - for i=1,num_images do - images[i][{{1,imgs[i]:size(2)},{imgs[i]:size(3)}}]:copy(imgs[i]) - end - - - return images -end - - -local function sample_rois(self,i) - - local dataset = self.dataset - local rec = dataset:attachProposals(i) - local fg_inds = {} - local bg_inds = {} - for j=1,rec:size() do - local id = rec.label[j] - local is_fg = (rec.overlap[j] >= self.fg_threshold) - local is_bg = (rec.overlap[j] >= self.bg_threshold[1] and - rec.overlap[j] < self.bg_threshold[2]) - if is_fg then - table.insert(fg_inds,j) - elseif is_bg then - table.insert(bg_inds,j) - end - end - - local fg_rois_per_this_image = math.min(#fg_inds,self.fg_) - if #fg_inds > 0 then - - end - -end diff --git a/nnf.lua b/nnf.lua index a2e7831..eee8d36 100644 --- a/nnf.lua +++ b/nnf.lua @@ -1,14 +1,15 @@ require 'nn' require 'image' -require 'inn' +--require 'inn' require 'xlua' nnf = {} torch.include('nnf','DataSetPascal.lua') torch.include('nnf','BatchProvider.lua') +torch.include('nnf','BatchProviderROI.lua') -torch.include('nnf','SPP.lua') +--torch.include('nnf','SPP.lua') torch.include('nnf','RCNN.lua') torch.include('nnf','Trainer.lua') From f1c256f2650033589a52b66dce3e0fed2264cce9 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Sat, 15 Aug 2015 18:38:24 +0200 Subject: [PATCH 05/19] Basics of Fast-RCNN seems to be working --- BatchProviderROI.lua | 46 ++++++++++++++++++++++++++------------------ ROIPooling.lua | 21 ++++++++++++++++---- model.lua | 11 +++++++++++ nnf.lua | 1 + 4 files changed, 56 insertions(+), 23 deletions(-) diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index 6f28865..9b66e61 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -30,7 +30,7 @@ function BatchProviderROI:permuteIdx() self._cur = self._cur + self.imgs_per_batch local img_idx_end = imgs_per_batch - --[[ + local fg_windows = {} local bg_windows = {} for i=1,img_idx_end do @@ -48,40 +48,50 @@ function BatchProviderROI:permuteIdx() end end end - --]] + local opts = {img_idx=img_idx,img_idx_end=img_idx_end} return fg_windows,bg_windows,opts end -function BatchProviderROI:selectBBoxes(fg_windows,bg_windows) - local fg_w = {} - local bg_w = {} +function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) + --local fg_w = {} + --local bg_w = {} + local rois = {} + local labels = {} for im=1,self.imgs_per_batch do - fg_w[im] = {} - bg_w[im] = {} + local im_scale = im_scales[im] + --fg_w[im] = {} + --bg_w[im] = {} local window_idx = torch.randperm(#bg_windows[im]) for i=1,math.min(self.bg_num_each,#bg_windows[im]) do local curr_idx = bg_windows[im][window_idx[i] ][1] local position = bg_windows[im][window_idx[i] ][2] - local dd = self.bboxes[curr_idx][0][position] - table.insert(bg_w[im],dd) + local dd = self.bboxes[curr_idx][0][position][{{2,5}}]--:totable() + dd:add(-1):mul(im_scale):add(1) + --table.insert(bg_w[im],dd) + table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) + table.insert(labels,self.bboxes[curr_idx][0][position][6]) end window_idx = torch.randperm(#fg_windows[im]) for i=1,math.min(self.fg_num_each,#fg_windows[im]) do local curr_idx = fg_windows[im][window_idx[i] ][1] local position = fg_windows[im][window_idx[i] ][2] - local dd = self.bboxes[curr_idx][1][position] - table.insert(fg_w[im],dd) + local dd = self.bboxes[curr_idx][1][position][{{2,5}}]--:totable() + dd:add(-1):mul(im_scale):add(1) + --table.insert(fg_w[im],dd) + table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) + table.insert(labels,self.bboxes[curr_idx][1][position][6]) end - end - - return fg_w,bg_w + rois = torch.FloatTensor(rois) + labels = torch.IntTensor(labels) + --return fg_w,bg_w + return rois, labels end local function getImages(self,img_ids,images) @@ -123,8 +133,6 @@ function BatchProviderROI:getBatch(batches,targets) self.fg_num_each = self.fg_fraction * self.batch_size self.bg_num_each = self.batch_size - self.fg_num_each - --self.fg_num_total = self.fg_num_each * self.iter_per_batch - --self.bg_num_total = self.bg_num_each * self.iter_per_batch local fg_windows,bg_windows,opts = self:permuteIdx() --local fg_w,bg_w = self:selectBBoxes(fg_windows,bg_windows) @@ -132,10 +140,10 @@ function BatchProviderROI:getBatch(batches,targets) local batches = batches or {torch.FloatTensor(),torch.FloatTensor()} local targets = targets or torch.IntTensor() - -- batches[1]:resize(self.batch_size,unpack(self.batch_dim)) local im_scales = getImages(self,opts.img_idx,batches[1]) - batches[2]:resize(self.batch_size,unpack(self.batch_dim)) - targets:resize(self.batch_size,self.target_dim) + local rois,labels = self:selectBBoxes(fg_windows,bg_windows,im_scales) + batches[2]:resizeAs(rois):copy(rois) + targets:resizeAs(labels):copy(labels) return batches, targets end diff --git a/ROIPooling.lua b/ROIPooling.lua index 23a5606..9b0cb6f 100644 --- a/ROIPooling.lua +++ b/ROIPooling.lua @@ -5,9 +5,9 @@ function ROIPooling:__init(W,H) self.W = W self.H = H self.pooler = {}--nn.SpatialAdaptiveMaxPooling(W,H) + self.spatial_scale = 1 end --- not for batches for the moment function ROIPooling:updateOutput(input) local data = input[1] local rois = input[2] @@ -16,16 +16,19 @@ function ROIPooling:updateOutput(input) local ss = s:size(1) self.output:resize(num_rois,s[ss-2],self.H,self.W) + if not self._type then self._type = output:type() end + if #self.pooler < num_rois then local diff = num_rois - #self.pooler for i=1,diff do - table.insert(self.pooler,nn.SpatialAdaptiveMaxPooling(self.W,self.H)) + table.insert(self.pooler,nn.SpatialAdaptiveMaxPooling(self.W,self.H):type(self._type)) end end for i=1,num_rois do local roi = rois[i] - local im = data[{{},{roi[2],roi[4]},{roi[1],roi[3]}}] + local im_idx = roi[1] + local im = data[{im_idx,{},{roi[3],roi[5]},{roi[2],roi[4]}}] self.output[i] = self.pooler[i]:forward(im) end return self.output @@ -41,7 +44,8 @@ function ROIPooling:updateGradInput(input,gradOutput) for i=1,num_rois do local roi = rois[i] - local r = {{},{roi[2],roi[3]},{roi[1],roi[3]}} + local im_idx = roi[1] + local r = {im_idx,{},{roi[3],roi[5]},{roi[2],roi[4]}} local im = data[r] local g = self.pooler[i]:backward(im,gradOutput[i]) self.gradInput[r]:add(g) @@ -49,3 +53,12 @@ function ROIPooling:updateGradInput(input,gradOutput) return self.gradInput end + +function ROIPooling:type(type) + parent.type(self,type) + for i=1,#self.pooler do + self.pooler[i]:type(type) + end + self._type = type + return self +end diff --git a/model.lua b/model.lua index 9700f0b..36812e1 100644 --- a/model.lua +++ b/model.lua @@ -21,6 +21,17 @@ if opt.algo == 'RCNN' then elseif opt.algo == 'SPP' then features = model:get(1) classifier = model:get(3) +elseif opt.algo == 'FRCNN' then + local temp = nn.Sequential() + local features = model:get(1) + local classifier = model:get(3) + local prl = nn.ParallelTable() + prl:add(features) + prl:add(nn.Identity()) + temp:add(prl) + temp:add(nnf.ROIPooling(7,7)) + temp:add(nn.View(-1):setNumInputDims(3)) + temp:add(classifier) end -- 2. Create Criterion diff --git a/nnf.lua b/nnf.lua index eee8d36..2cf0592 100644 --- a/nnf.lua +++ b/nnf.lua @@ -11,6 +11,7 @@ torch.include('nnf','BatchProviderROI.lua') --torch.include('nnf','SPP.lua') torch.include('nnf','RCNN.lua') +torch.include('nnf','ROIPooling.lua') torch.include('nnf','Trainer.lua') torch.include('nnf','Tester.lua') From 3bf8968b06cf6e8fc4c4419bf75ab6ecf24f7312 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Sat, 15 Aug 2015 19:49:08 +0200 Subject: [PATCH 06/19] Almost working --- BatchProviderROI.lua | 4 ++-- ROIPooling.lua | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index 9b66e61..5492853 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -138,12 +138,12 @@ function BatchProviderROI:getBatch(batches,targets) --local fg_w,bg_w = self:selectBBoxes(fg_windows,bg_windows) local batches = batches or {torch.FloatTensor(),torch.FloatTensor()} - local targets = targets or torch.IntTensor() + local targets = targets or torch.FloatTensor() local im_scales = getImages(self,opts.img_idx,batches[1]) local rois,labels = self:selectBBoxes(fg_windows,bg_windows,im_scales) batches[2]:resizeAs(rois):copy(rois) - targets:resizeAs(labels):copy(labels) + targets:resize(labels:size()):copy(labels) return batches, targets end diff --git a/ROIPooling.lua b/ROIPooling.lua index 9b0cb6f..a3588b1 100644 --- a/ROIPooling.lua +++ b/ROIPooling.lua @@ -8,14 +8,26 @@ function ROIPooling:__init(W,H) self.spatial_scale = 1 end +function ROIPooling:setSpatialScale(scale) + self.spatial_scale = scale + return self +end + function ROIPooling:updateOutput(input) local data = input[1] local rois = input[2] + local num_rois = rois:size(1) local s = data:size() local ss = s:size(1) self.output:resize(num_rois,s[ss-2],self.H,self.W) + rois[{{},{2,5}}]:add(-1):mul(self.spatial_scale):add(1):round() + rois[{{},2}]:cmin(s[ss]) + rois[{{},3}]:cmin(s[ss-1]) + rois[{{},4}]:cmin(s[ss]) + rois[{{},5}]:cmin(s[ss-1]) + if not self._type then self._type = output:type() end if #self.pooler < num_rois then From 7c9ad2efef113f5cba829326f742bb2f66b586ac Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Sun, 16 Aug 2015 21:14:23 +0200 Subject: [PATCH 07/19] Basic test for FRCNN --- test_frcnn.lua | 107 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 test_frcnn.lua diff --git a/test_frcnn.lua b/test_frcnn.lua new file mode 100644 index 0000000..f5cb124 --- /dev/null +++ b/test_frcnn.lua @@ -0,0 +1,107 @@ +require 'nnf' + +dt = torch.load('pascal_2007_train.t7') +ds = nnf.DataSetPascal{image_set='train', + datadir='/home/francisco/work/datasets/VOCdevkit', + roidbdir='/home/francisco/work/datasets/rcnn/selective_search_data' + } +if false then + ds.roidb = {} + for i=1,ds:size() do + ds.roidb[i] = torch.IntTensor(10,4):random(1,5) + ds.roidb[i][{{},{3,4}}]:add(6) + end +else + ds.roidb = dt.roidb +end + +bp = nnf.BatchProviderROI(ds) +bp:setupData() + +--------------------------------------------------------------------------------------- +-- model +--------------------------------------------------------------------------------------- +do + model = nn.Sequential() + local features = nn.Sequential() + local classifier = nn.Sequential() + + features:add(nn.SpatialConvolutionMM(3,96,11,11,4,4,5,5)) + features:add(nn.ReLU(true)) + features:add(nn.SpatialConvolutionMM(96,128,5,5,2,2,2,2)) + features:add(nn.ReLU(true)) + features:add(nn.SpatialMaxPooling(2,2,2,2)) + + classifier:add(nn.Linear(128*7*7,1024)) + classifier:add(nn.ReLU(true)) + classifier:add(nn.Dropout(0.5)) + classifier:add(nn.Linear(1024,21)) + + local prl = nn.ParallelTable() + prl:add(features) + prl:add(nn.Identity()) + model:add(prl) + model:add(nnf.ROIPooling(7,7):setSpatialScale(1/16)) + model:add(nn.View(-1):setNumInputDims(3)) + model:add(classifier) + +end + +parameters,gradParameters = model:getParameters() + +optimState = {learningRate = 1e-2, weightDecay = 0.0005, momentum = 0.9, + learningRateDecay = 0} + +-------------------------------------------------------------------------- +-- training +-------------------------------------------------------------------------- + +model:float() +model:training() + +criterion = nn.CrossEntropyCriterion():float() + +max_iter = 10 + +function train() + local err = 0 + for i=1,max_iter do + xlua.progress(i,max_iter) + inputs,target = bp:getBatch(inputs,target) + local batchSize = target:size(1) + + local feval = function(x) + if x ~= parameters then + parameters:copy(x) + end + gradParameters:zero() + + local outputs = model:forward(inputs) + + local f = criterion:forward(outputs,target) + local df_do = criterion:backward(outputs,target) + + model:backward(inputs,df_do) + + if normalize then + gradParameters:div(batchSize) + f = f/batchSize + end + + return f,gradParameters + end + + local x,fx = optim.sgd(feval,parameters,optimState) + err = err + fx[1] + end + print('Training error: '..err/max_iter) +end + +train() + +if false then + m = nnf.ROIPooling(50,50):float() + o = m:forward(batches) + g = m:backward(batches,o) +end + From 5819a839fca9f5eb54cf09cefcd425707be71ed7 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Mon, 17 Aug 2015 23:39:36 +0200 Subject: [PATCH 08/19] Fix bug in ROIPooling --- ROIPooling.lua | 10 +++++----- test_frcnn.lua | 8 +++++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ROIPooling.lua b/ROIPooling.lua index a3588b1..0345628 100644 --- a/ROIPooling.lua +++ b/ROIPooling.lua @@ -6,6 +6,7 @@ function ROIPooling:__init(W,H) self.H = H self.pooler = {}--nn.SpatialAdaptiveMaxPooling(W,H) self.spatial_scale = 1 + self.gradInput = {torch.Tensor()} end function ROIPooling:setSpatialScale(scale) @@ -41,7 +42,7 @@ function ROIPooling:updateOutput(input) local roi = rois[i] local im_idx = roi[1] local im = data[{im_idx,{},{roi[3],roi[5]},{roi[2],roi[4]}}] - self.output[i] = self.pooler[i]:forward(im) + self.output[i] = self.pooler[i]:updateOutput(im) end return self.output end @@ -52,18 +53,17 @@ function ROIPooling:updateGradInput(input,gradOutput) local num_rois = rois:size(1) local s = data:size() local ss = s:size(1) - self.gradInput:resizeAs(data):zero() + self.gradInput[1]:resizeAs(data):zero() for i=1,num_rois do local roi = rois[i] local im_idx = roi[1] local r = {im_idx,{},{roi[3],roi[5]},{roi[2],roi[4]}} local im = data[r] - local g = self.pooler[i]:backward(im,gradOutput[i]) - self.gradInput[r]:add(g) + local g = self.pooler[i]:updateGradInput(im,gradOutput[i]) + self.gradInput[1][r]:add(g) end return self.gradInput - end function ROIPooling:type(type) diff --git a/test_frcnn.lua b/test_frcnn.lua index f5cb124..cbca5ae 100644 --- a/test_frcnn.lua +++ b/test_frcnn.lua @@ -15,8 +15,14 @@ else ds.roidb = dt.roidb end +if false then bp = nnf.BatchProviderROI(ds) bp:setupData() +else + bp = nnf.BatchProviderROI(ds) + local temp = torch.load('pascal_2007_train_bp.t7') + bp.bboxes = temp.bboxes +end --------------------------------------------------------------------------------------- -- model @@ -61,7 +67,7 @@ model:training() criterion = nn.CrossEntropyCriterion():float() -max_iter = 10 +max_iter = 20 function train() local err = 0 From bbe3dd20f4071422844780ef4b0946faf06af230 Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Tue, 18 Aug 2015 18:09:54 +0200 Subject: [PATCH 09/19] Improve training script for frcnn --- test_frcnn.lua | 114 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 32 deletions(-) diff --git a/test_frcnn.lua b/test_frcnn.lua index cbca5ae..61455d5 100644 --- a/test_frcnn.lua +++ b/test_frcnn.lua @@ -1,10 +1,18 @@ require 'nnf' dt = torch.load('pascal_2007_train.t7') -ds = nnf.DataSetPascal{image_set='train', - datadir='/home/francisco/work/datasets/VOCdevkit', - roidbdir='/home/francisco/work/datasets/rcnn/selective_search_data' - } +if false then + ds = nnf.DataSetPascal{image_set='train', + datadir='/home/francisco/work/datasets/VOCdevkit', + roidbdir='/home/francisco/work/datasets/rcnn/selective_search_data' + } +else + ds = nnf.DataSetPascal{image_set='train', + datadir='datasets/VOCdevkit', + roidbdir='data/selective_search_data' + } +end + if false then ds.roidb = {} for i=1,ds:size() do @@ -15,9 +23,9 @@ else ds.roidb = dt.roidb end -if false then -bp = nnf.BatchProviderROI(ds) -bp:setupData() +if true then + bp = nnf.BatchProviderROI(ds) + bp:setupData() else bp = nnf.BatchProviderROI(ds) local temp = torch.load('pascal_2007_train_bp.t7') @@ -28,52 +36,89 @@ end -- model --------------------------------------------------------------------------------------- do + model = nn.Sequential() local features = nn.Sequential() local classifier = nn.Sequential() + + if false then + features:add(nn.SpatialConvolutionMM(3,96,11,11,4,4,5,5)) + features:add(nn.ReLU(true)) + features:add(nn.SpatialConvolutionMM(96,128,5,5,2,2,2,2)) + features:add(nn.ReLU(true)) + features:add(nn.SpatialMaxPooling(2,2,2,2)) + + classifier:add(nn.Linear(128*7*7,1024)) + classifier:add(nn.ReLU(true)) + classifier:add(nn.Dropout(0.5)) + classifier:add(nn.Linear(1024,21)) - features:add(nn.SpatialConvolutionMM(3,96,11,11,4,4,5,5)) - features:add(nn.ReLU(true)) - features:add(nn.SpatialConvolutionMM(96,128,5,5,2,2,2,2)) - features:add(nn.ReLU(true)) - features:add(nn.SpatialMaxPooling(2,2,2,2)) + else + require 'loadcaffe' +-- local rcnnfold = '/home/francisco/work/libraries/rcnn/' +-- local base_model = loadcaffe.load( +-- rcnnfold..'model-defs/pascal_finetune_deploy.prototxt', +-- rcnnfold..'data/caffe_nets/finetune_voc_2012_train_iter_70k', +-- 'cudnn') + + local rcnnfold = '/home/francisco/work/libraries/caffe/examples/imagenet/' + local base_model = loadcaffe.load( + rcnnfold..'imagenet_deploy.prototxt', + rcnnfold..'caffe_reference_imagenet_model', + 'cudnn') + + + for i=1,14 do + features:add(base_model:get(i):clone()) + end - classifier:add(nn.Linear(128*7*7,1024)) - classifier:add(nn.ReLU(true)) - classifier:add(nn.Dropout(0.5)) - classifier:add(nn.Linear(1024,21)) + for i=17,22 do + classifier:add(base_model:get(i):clone()) + end + classifier:add(nn.Linear(4096,21):cuda()) + + collectgarbage() + end + collectgarbage() local prl = nn.ParallelTable() prl:add(features) prl:add(nn.Identity()) model:add(prl) - model:add(nnf.ROIPooling(7,7):setSpatialScale(1/16)) + model:add(nnf.ROIPooling(6,6):setSpatialScale(1/16)) + --model:add(inn.ROIPooling(6,6):setSpatialScale(1/16)) model:add(nn.View(-1):setNumInputDims(3)) model:add(classifier) end - +print(model) parameters,gradParameters = model:getParameters() -optimState = {learningRate = 1e-2, weightDecay = 0.0005, momentum = 0.9, +optimState = {learningRate = 1e-3, weightDecay = 0.0005, momentum = 0.9, learningRateDecay = 0} -------------------------------------------------------------------------- -- training -------------------------------------------------------------------------- -model:float() +model:cuda() model:training() -criterion = nn.CrossEntropyCriterion():float() +criterion = nn.CrossEntropyCriterion():cuda() + +display_iter = 20 -max_iter = 20 +inputs = {torch.CudaTensor(),torch.FloatTensor()} +target = torch.CudaTensor() function train() local err = 0 - for i=1,max_iter do - xlua.progress(i,max_iter) - inputs,target = bp:getBatch(inputs,target) + for i=1,display_iter do + xlua.progress(i,display_iter) + inputs0,target0 = bp:getBatch(inputs0,target0) + inputs[1]:resize(inputs0[1]:size()):copy(inputs0[1]) + inputs[2]:resize(inputs0[2]:size()):copy(inputs0[2]) + target:resize(target0:size()):copy(target0) local batchSize = target:size(1) local feval = function(x) @@ -100,14 +145,19 @@ function train() local x,fx = optim.sgd(feval,parameters,optimState) err = err + fx[1] end - print('Training error: '..err/max_iter) + print('Training error: '..err/display_iter) end -train() +stepsize = 30000 -if false then - m = nnf.ROIPooling(50,50):float() - o = m:forward(batches) - g = m:backward(batches,o) -end +num_iter = 3000 +for i=1,num_iter do + print(('Iteration: %d/%d'):format(i,num_iter)) + if i%(stepsize/display_iter) == 0 then + optimState.learningRate = optimState.learningRate/10 + end + + train() + +end From 05d8ad6d501bf008edbd515872672e7211adf473 Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Wed, 19 Aug 2015 21:41:59 +0200 Subject: [PATCH 10/19] Cleaning up and fix in test --- BatchProviderROI.lua | 14 ++------------ test_frcnn.lua | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index 5492853..f6fdf44 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -55,24 +55,16 @@ function BatchProviderROI:permuteIdx() end function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) - --local fg_w = {} - --local bg_w = {} - local rois = {} local labels = {} for im=1,self.imgs_per_batch do - local im_scale = im_scales[im] - --fg_w[im] = {} - --bg_w[im] = {} - local window_idx = torch.randperm(#bg_windows[im]) for i=1,math.min(self.bg_num_each,#bg_windows[im]) do local curr_idx = bg_windows[im][window_idx[i] ][1] local position = bg_windows[im][window_idx[i] ][2] - local dd = self.bboxes[curr_idx][0][position][{{2,5}}]--:totable() + local dd = self.bboxes[curr_idx][0][position][{{2,5}}] dd:add(-1):mul(im_scale):add(1) - --table.insert(bg_w[im],dd) table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) table.insert(labels,self.bboxes[curr_idx][0][position][6]) end @@ -81,16 +73,14 @@ function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) for i=1,math.min(self.fg_num_each,#fg_windows[im]) do local curr_idx = fg_windows[im][window_idx[i] ][1] local position = fg_windows[im][window_idx[i] ][2] - local dd = self.bboxes[curr_idx][1][position][{{2,5}}]--:totable() + local dd = self.bboxes[curr_idx][1][position][{{2,5}}] dd:add(-1):mul(im_scale):add(1) - --table.insert(fg_w[im],dd) table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) table.insert(labels,self.bboxes[curr_idx][1][position][6]) end end rois = torch.FloatTensor(rois) labels = torch.IntTensor(labels) - --return fg_w,bg_w return rois, labels end diff --git a/test_frcnn.lua b/test_frcnn.lua index 61455d5..fb54005 100644 --- a/test_frcnn.lua +++ b/test_frcnn.lua @@ -23,11 +23,16 @@ else ds.roidb = dt.roidb end +local image_transformer= nnf.ImageTransformer{mean_pix={103.939, 116.779, 123.68}, + raw_scale = 255, + swap = {3,2,1}} if true then bp = nnf.BatchProviderROI(ds) + bp.image_transformer = image_transformer bp:setupData() else bp = nnf.BatchProviderROI(ds) + bp.image_transformer = image_transformer local temp = torch.load('pascal_2007_train_bp.t7') bp.bboxes = temp.bboxes end @@ -94,13 +99,17 @@ end print(model) parameters,gradParameters = model:getParameters() -optimState = {learningRate = 1e-3, weightDecay = 0.0005, momentum = 0.9, +optimState = {learningRate = 1e-4, weightDecay = 0.0005, momentum = 0.9, learningRateDecay = 0} -------------------------------------------------------------------------- -- training -------------------------------------------------------------------------- +confusion_matrix = optim.ConfusionMatrix(21) + +savedModel = model:clone('weight','bias','running_mean','running_std') + model:cuda() model:training() @@ -138,6 +147,8 @@ function train() gradParameters:div(batchSize) f = f/batchSize end + + confusion_matrix:batchAdd(outputs,target) return f,gradParameters end @@ -157,7 +168,12 @@ for i=1,num_iter do if i%(stepsize/display_iter) == 0 then optimState.learningRate = optimState.learningRate/10 end + + confusion_matrix:zero() train() - + print(confusion_matrix) + if i%100 == 0 then + torch.save(paths.concat('cachedir','frcnn_t1.t7'),savedModel) + end end From 3e109736f8abe12614b63debc6e0f7ec4402803c Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Thu, 20 Aug 2015 18:35:55 +0200 Subject: [PATCH 11/19] Add gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ae7e697 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +*.swp +cachedir/* From ffaf0f2640a5a4f58bbdc582f7a9a24a0617f186 Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Sat, 22 Aug 2015 15:55:41 +0200 Subject: [PATCH 12/19] Fix unitialized memory in BatchProviderROI --- BatchProviderROI.lua | 21 +++++++------ test_frcnn.lua | 71 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index f6fdf44..2260f3e 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -12,10 +12,8 @@ end -- setup is the same function BatchProviderROI:permuteIdx() - local fg_num_each = self.fg_num_each - local bg_num_each = self.bg_num_each - local fg_num_total = self.fg_num_total - local bg_num_total = self.bg_num_total + --local fg_num_total = self.fg_num_total + --local bg_num_total = self.bg_num_total local total_img = self.dataset:size() local imgs_per_batch = self.imgs_per_batch @@ -26,8 +24,8 @@ function BatchProviderROI:permuteIdx() self._cur = 1 end - local img_idx = self._perm[{{self._cur,self._cur + self.imgs_per_batch - 1}}] - self._cur = self._cur + self.imgs_per_batch + local img_idx = self._perm[{{self._cur,self._cur + self.imgs_per_batch - 1}}] + self._cur = self._cur + self.imgs_per_batch local img_idx_end = imgs_per_batch @@ -55,12 +53,16 @@ function BatchProviderROI:permuteIdx() end function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) + local fg_num_each = self.fg_num_each + local bg_num_each = self.bg_num_each + local rois = {} local labels = {} for im=1,self.imgs_per_batch do local im_scale = im_scales[im] local window_idx = torch.randperm(#bg_windows[im]) - for i=1,math.min(self.bg_num_each,#bg_windows[im]) do + local end_idx = math.min(bg_num_each,#bg_windows[im]) + for i=1,end_idx do local curr_idx = bg_windows[im][window_idx[i] ][1] local position = bg_windows[im][window_idx[i] ][2] local dd = self.bboxes[curr_idx][0][position][{{2,5}}] @@ -70,7 +72,8 @@ function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) end window_idx = torch.randperm(#fg_windows[im]) - for i=1,math.min(self.fg_num_each,#fg_windows[im]) do + local end_idx = math.min(fg_num_each,#fg_windows[im]) + for i=1,end_idx do local curr_idx = fg_windows[im][window_idx[i] ][1] local position = fg_windows[im][window_idx[i] ][2] local dd = self.bboxes[curr_idx][1][position][{{2,5}}] @@ -110,7 +113,7 @@ local function getImages(self,img_ids,images) -- create single tensor with all images, padding with zero for different sizes im_sizes = torch.IntTensor(im_sizes) local max_shape = im_sizes:max(1)[1] - images:resize(num_images,3,max_shape[1],max_shape[2]) + images:resize(num_images,3,max_shape[1],max_shape[2]):zero() for i=1,num_images do images[i][{{},{1,imgs[i]:size(2)},{1,imgs[i]:size(3)}}]:copy(imgs[i]) end diff --git a/test_frcnn.lua b/test_frcnn.lua index fb54005..109c7cd 100644 --- a/test_frcnn.lua +++ b/test_frcnn.lua @@ -1,4 +1,8 @@ require 'nnf' +require 'inn' +require 'cudnn' + +cutorch.setDevice(1) dt = torch.load('pascal_2007_train.t7') if false then @@ -19,16 +23,17 @@ if false then ds.roidb[i] = torch.IntTensor(10,4):random(1,5) ds.roidb[i][{{},{3,4}}]:add(6) end -else +elseif true then ds.roidb = dt.roidb end -local image_transformer= nnf.ImageTransformer{mean_pix={103.939, 116.779, 123.68}, +local image_transformer= nnf.ImageTransformer{mean_pix={102.9801,115.9465,122.7717},--{103.939, 116.779, 123.68}, raw_scale = 255, swap = {3,2,1}} if true then bp = nnf.BatchProviderROI(ds) bp.image_transformer = image_transformer + bp.bg_threshold = {0.1,0.5} bp:setupData() else bp = nnf.BatchProviderROI(ds) @@ -37,6 +42,17 @@ else bp.bboxes = temp.bboxes end + +if false then + local mytest = nnf.ROIPooling(50,50):float() + function do_mytest() + local input0,target0 = bp:getBatch(input0,target0) + local o = mytest:forward(input0) + return input0,target0,o + end + --input0,target0,o = do_mytest() +end + --------------------------------------------------------------------------------------- -- model --------------------------------------------------------------------------------------- @@ -58,7 +74,7 @@ do classifier:add(nn.Dropout(0.5)) classifier:add(nn.Linear(1024,21)) - else + elseif false then require 'loadcaffe' -- local rcnnfold = '/home/francisco/work/libraries/rcnn/' -- local base_model = loadcaffe.load( @@ -83,6 +99,23 @@ do classifier:add(nn.Linear(4096,21):cuda()) collectgarbage() + + else + local fold = 'data/models/imagenet_models/alexnet/' + local m1 = torch.load(fold..'features.t7') + local m2 = torch.load(fold..'top.t7') + + for i=1,14 do + features:add(m1:get(i):clone()) + end + + for i=2,7 do + classifier:add(m2:get(i):clone()) + end + local linear = nn.Linear(4096,21):cuda() + linear.weight:normal(0,0.01) + linear.bias:zero() + classifier:add(linear) end collectgarbage() @@ -90,16 +123,18 @@ do prl:add(features) prl:add(nn.Identity()) model:add(prl) - model:add(nnf.ROIPooling(6,6):setSpatialScale(1/16)) - --model:add(inn.ROIPooling(6,6):setSpatialScale(1/16)) + --model:add(nnf.ROIPooling(6,6):setSpatialScale(1/16)) + model:add(inn.ROIPooling(6,6):setSpatialScale(1/16)) model:add(nn.View(-1):setNumInputDims(3)) model:add(classifier) end print(model) + +model:cuda() parameters,gradParameters = model:getParameters() -optimState = {learningRate = 1e-4, weightDecay = 0.0005, momentum = 0.9, +optimState = {learningRate = 1e-3, weightDecay = 0.0005, momentum = 0.9, learningRateDecay = 0} -------------------------------------------------------------------------- @@ -108,16 +143,20 @@ optimState = {learningRate = 1e-4, weightDecay = 0.0005, momentum = 0.9, confusion_matrix = optim.ConfusionMatrix(21) -savedModel = model:clone('weight','bias','running_mean','running_std') -model:cuda() model:training() +savedModel = model:clone('weight','bias','running_mean','running_std') + criterion = nn.CrossEntropyCriterion():cuda() +--criterion.nll.sizeAverage = false + +--normalize = true display_iter = 20 -inputs = {torch.CudaTensor(),torch.FloatTensor()} +--inputs = {torch.CudaTensor(),torch.FloatTensor()} +inputs = {torch.CudaTensor(),torch.CudaTensor()} target = torch.CudaTensor() function train() @@ -159,9 +198,12 @@ function train() print('Training error: '..err/display_iter) end +epoch_size = math.ceil(ds:size()/bp.imgs_per_batch) stepsize = 30000 +print_step = 10 +num_iter = 40000/display_iter--3000 -num_iter = 3000 +confusion_matrix:zero() for i=1,num_iter do print(('Iteration: %d/%d'):format(i,num_iter)) @@ -169,10 +211,13 @@ for i=1,num_iter do optimState.learningRate = optimState.learningRate/10 end - confusion_matrix:zero() - train() - print(confusion_matrix) + + if i%print_step == 0 then + print(confusion_matrix) + confusion_matrix:zero() + end + if i%100 == 0 then torch.save(paths.concat('cachedir','frcnn_t1.t7'),savedModel) end From 6836ca2a6a5676c8ed84e00d0d2e5e18ae80044f Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Sat, 22 Aug 2015 16:34:08 +0200 Subject: [PATCH 13/19] Fix bug in BatchProviderROI --- BatchProviderROI.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index 2260f3e..e1e1655 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -65,7 +65,7 @@ function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) for i=1,end_idx do local curr_idx = bg_windows[im][window_idx[i] ][1] local position = bg_windows[im][window_idx[i] ][2] - local dd = self.bboxes[curr_idx][0][position][{{2,5}}] + local dd = self.bboxes[curr_idx][0][position][{{2,5}}]:clone() dd:add(-1):mul(im_scale):add(1) table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) table.insert(labels,self.bboxes[curr_idx][0][position][6]) @@ -76,7 +76,7 @@ function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) for i=1,end_idx do local curr_idx = fg_windows[im][window_idx[i] ][1] local position = fg_windows[im][window_idx[i] ][2] - local dd = self.bboxes[curr_idx][1][position][{{2,5}}] + local dd = self.bboxes[curr_idx][1][position][{{2,5}}]:clone() dd:add(-1):mul(im_scale):add(1) table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) table.insert(labels,self.bboxes[curr_idx][1][position][6]) From 4ce9107ab4dca27d67fec2f55807578d9949f87d Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Sun, 23 Aug 2015 13:49:37 +0200 Subject: [PATCH 14/19] Add flip in training and test code --- BatchProviderROI.lua | 30 +++++-- ImageDetect.lua | 88 ++++++++++++++++++++ Tester_FRCNN.lua | 192 +++++++++++++++++++++++++++++++++++++++++++ nnf.lua | 2 + test_frcnn.lua | 28 +++++-- 5 files changed, 325 insertions(+), 15 deletions(-) create mode 100644 ImageDetect.lua create mode 100644 Tester_FRCNN.lua diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index e1e1655..9844ee7 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -46,13 +46,13 @@ function BatchProviderROI:permuteIdx() end end end - - local opts = {img_idx=img_idx,img_idx_end=img_idx_end} + local do_flip = torch.FloatTensor(imgs_per_batch):random(0,1) + local opts = {img_idx=img_idx,img_idx_end=img_idx_end,do_flip=do_flip} return fg_windows,bg_windows,opts end -function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) +function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales,do_flip,im_sizes) local fg_num_each = self.fg_num_each local bg_num_each = self.bg_num_each @@ -62,11 +62,18 @@ function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) local im_scale = im_scales[im] local window_idx = torch.randperm(#bg_windows[im]) local end_idx = math.min(bg_num_each,#bg_windows[im]) + local flip = do_flip[im] == 1 + local im_size = im_sizes[im] for i=1,end_idx do local curr_idx = bg_windows[im][window_idx[i] ][1] local position = bg_windows[im][window_idx[i] ][2] local dd = self.bboxes[curr_idx][0][position][{{2,5}}]:clone() dd:add(-1):mul(im_scale):add(1) + if flip then + local tt = dd[1] + dd[1] = im_size[2]-dd[3] +1 + dd[3] = im_size[2]-tt +1 + end table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) table.insert(labels,self.bboxes[curr_idx][0][position][6]) end @@ -78,6 +85,11 @@ function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) local position = fg_windows[im][window_idx[i] ][2] local dd = self.bboxes[curr_idx][1][position][{{2,5}}]:clone() dd:add(-1):mul(im_scale):add(1) + if flip then + local tt = dd[1] + dd[1] = im_size[2]-dd[3] +1 + dd[3] = im_size[2]-tt +1 + end table.insert(rois,{im,dd[1],dd[2],dd[3],dd[4]}) table.insert(labels,self.bboxes[curr_idx][1][position][6]) end @@ -87,7 +99,7 @@ function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales) return rois, labels end -local function getImages(self,img_ids,images) +local function getImages(self,img_ids,images,do_flip) local dataset = self.dataset local num_images = img_ids:size(1) @@ -98,6 +110,10 @@ local function getImages(self,img_ids,images) for i=1,num_images do local im = dataset:getImage(img_ids[i]) im = self.image_transformer:preprocess(im) + local flip = do_flip[i] == 1 + if flip then + im = image.hflip(im) + end local im_size = im[1]:size() local im_size_min = math.min(im_size[1],im_size[2]) local im_size_max = math.max(im_size[1],im_size[2]) @@ -117,7 +133,7 @@ local function getImages(self,img_ids,images) for i=1,num_images do images[i][{{},{1,imgs[i]:size(2)},{1,imgs[i]:size(3)}}]:copy(imgs[i]) end - return im_scales + return im_scales,im_sizes end @@ -133,8 +149,8 @@ function BatchProviderROI:getBatch(batches,targets) local batches = batches or {torch.FloatTensor(),torch.FloatTensor()} local targets = targets or torch.FloatTensor() - local im_scales = getImages(self,opts.img_idx,batches[1]) - local rois,labels = self:selectBBoxes(fg_windows,bg_windows,im_scales) + local im_scales, im_sizes = getImages(self,opts.img_idx,batches[1],opts.do_flip) + local rois,labels = self:selectBBoxes(fg_windows,bg_windows,im_scales,opts.do_flip, im_sizes) batches[2]:resizeAs(rois):copy(rois) targets:resize(labels:size()):copy(labels) diff --git a/ImageDetect.lua b/ImageDetect.lua new file mode 100644 index 0000000..27bf8e2 --- /dev/null +++ b/ImageDetect.lua @@ -0,0 +1,88 @@ +local ImageDetect = torch.class('nnf.ImageDetect') + +function ImageDetect:__init(model) + self.model = model + self.image_transformer = nnf.ImageTransformer{mean_pix={102.9801,115.9465,122.7717}, + raw_scale = 255, + swap = {3,2,1}} + self.scale = {600} + self.max_size = 1000 + self.sm = nn.SoftMax():cuda() +end + + +local function getImages(self,images,im) + local num_scales = #self.scale + + local imgs = {} + local im_sizes = {} + local im_scales = {} + + im = self.image_transformer:preprocess(im) + + local im_size = im[1]:size() + local im_size_min = math.min(im_size[1],im_size[2]) + local im_size_max = math.max(im_size[1],im_size[2]) + for i=1,num_scales do + local im_scale = self.scale[i]/im_size_min + if torch.round(im_scale*im_size_max) > self.max_size then + im_scale = self.max_size/im_size_max + end + local im_s = {im_size[1]*im_scale,im_size[2]*im_scale} + table.insert(imgs,image.scale(im,im_s[2],im_s[1])) + table.insert(im_sizes,im_s) + table.insert(im_scales,im_scale) + end + -- create single tensor with all images, padding with zero for different sizes + im_sizes = torch.IntTensor(im_sizes) + local max_shape = im_sizes:max(1)[1] + images:resize(num_scales,3,max_shape[1],max_shape[2]):zero() + for i=1,num_scales do + images[i][{{},{1,imgs[i]:size(2)},{1,imgs[i]:size(3)}}]:copy(imgs[i]) + end + return im_scales +end + +local function project_im_rois(im_rois,scales) + local levels + local rois = torch.FloatTensor() + if #scales > 1 then + local scales = torch.FloatTensor(scales) + local widths = im_rois[{{},3}] - im_rois[{{},1}] + 1 + local heights = im_rois[{{},4}] - im_rois[{{}, 2}] + 1 + + local areas = widths * heights + local scaled_areas = areas:view(-1,1) * torch.pow(scales:view(1,-1),2) + local diff_areas = torch.abs(scaled_areas - 224 * 224) + levels = select(2, diff_areas:min(2)) + else + levels = torch.FloatTensor() + rois:resize(im_rois:size(1),5) + rois[{{},1}]:fill(1) + rois[{{},{2,5}}]:copy(im_rois):add(-1):mul(scales[1]):add(1) + end + + return rois + +end + +-- supposes boxes is in [x1,y1,x2,y2] format +function ImageDetect:detect(im,boxes) + local inputs = {torch.FloatTensor(),torch.FloatTensor()} + local im_scales = getImages(self,inputs[1],im) + inputs[2] = project_im_rois(boxes,im_scales) + + local inputs_cuda = {torch.CudaTensor(),torch.CudaTensor()} + inputs_cuda[1]:resize(inputs[1]:size()):copy(inputs[1]) + inputs_cuda[2]:resize(inputs[2]:size()):copy(inputs[2]) + local output0 = self.model:forward(inputs_cuda) + local output = self.sm:forward(output0):float() + --[[ + for i=1,#im_scales do + local dd = boxes:clone() + dd:add(-1):mul(im_scale[i]):add(1) + + end + --]] + return output +end diff --git a/Tester_FRCNN.lua b/Tester_FRCNN.lua new file mode 100644 index 0000000..7e8a139 --- /dev/null +++ b/Tester_FRCNN.lua @@ -0,0 +1,192 @@ +local utils = paths.dofile('utils.lua') +local nms = paths.dofile('nms.lua') + +local keep_top_k = utils.keep_top_k +local VOCevaldet = utils.VOCevaldet + +local Tester = torch.class('nnf.Tester_FRCNN') + +function Tester:__init(module,feat_provider) + self.dataset = feat_provider.dataset + self.module = module + self.feat_provider = feat_provider + + self.feat_dim = {256*50} + self.max_batch_size = 4000 + + self.cachefolder = nil + self.cachename = nil + self.suffix = '' + self.verbose = true +end + +-- improve it ! +function Tester:validate(criterion) + + local tname = paths.concat(self.cachefolder,self.cachename) + local valData + if paths.filep(tname) then + valData = torch.load(tname) + else + -- batch_provider need to be set before + valData = {} + valData.inputs,valData.targets = self.batch_provider:getBatch() + torch.save(tname,valData) + self.batch_provider = nil + end + + local num_batches = valData.inputs:size(1) + local module = self.module + + local err = 0 + local inputs = torch.CudaTensor() + local targets = torch.CudaTensor() + for t=1,num_batches do + xlua.progress(t,num_batches) + + inputs:resize(valData.inputs[t]:size()):copy(valData.inputs[t]) + targets:resize(valData.targets[t]:size()):copy(valData.targets[t]) + + local output = module:forward(inputs) + + err = err + criterion:forward(output,targets) + end + + valData = nil + collectgarbage() + + return err/num_batches +end + +function Tester:test(iteration) + + local dataset = self.dataset + local module = self.module + local feat_provider = self.feat_provider + + local pathfolder = paths.concat(self.cachefolder,'test_iter'..iteration) + paths.mkdir(pathfolder) + + module:evaluate() + dataset:loadROIDB() + + local feats = torch.FloatTensor() + local feats_batched = {} + local feats_cuda = torch.CudaTensor() + + local output = torch.FloatTensor() + + local output_dim = module:get(module:size()) + + local softmax = nn.SoftMax():float() + + local boxes + -- + local aboxes = {} + for i=1,dataset.num_classes do + table.insert(aboxes,{}) + end + + local max_per_set = 5*dataset:size() + local max_per_image = 100 + local thresh = torch.ones(dataset.num_classes):mul(-1.5) + local scored_boxes = torch.FloatTensor() + + local timer = torch.Timer() + local timer2 = torch.Timer() + local timer3 = torch.Timer() + local detec = nnf.ImageDetect(module) + for i=1,dataset:size() do + timer:reset() + io.write(('test: (%s) %5d/%-5d '):format(dataset.dataset_name,i,dataset:size())); + boxes = dataset:getROIBoxes(i):float() + local im = dataset:getImage(i) + local output = detec:detect(im,boxes) + + local add_bg = 0 + if dataset.num_classes ~= output:size(2) then -- if there is no svm + --output = softmax:forward(output) + add_bg = 1 + end + local tt = 0 + local tt2 = timer3:time().real + + timer2:reset() + for j=1,dataset.num_classes do + local scores = output:select(2,j+add_bg) + local idx = torch.range(1,scores:numel()):long() + local idx2 = scores:gt(thresh[j]) + idx = idx[idx2] + scored_boxes:resize(idx:numel(),5) + if scored_boxes:numel() > 0 then + scored_boxes:narrow(2,1,4):index(boxes,1,idx) + scored_boxes:select(2,5):copy(scores[idx2]) + end + local keep = nms(scored_boxes,0.3) + if keep:numel()>0 then + local _,ord = torch.sort(scored_boxes:select(2,5):index(1,keep),true) + ord = ord:narrow(1,1,math.min(ord:numel(),max_per_image)) + keep = keep:index(1,ord) + aboxes[j][i] = scored_boxes:index(1,keep) + else + aboxes[j][i] = torch.FloatTensor() + end + + if i%1000 == 0 then + aboxes[j],thresh[j] = keep_top_k(aboxes[j],max_per_set) + end + + end + + io.write((' prepare feat time: %.3f, forward time: %.3f, select time: %.3fs, total time: %.3fs\n'):format(tt,tt2,timer2:time().real,timer:time().real)); + --collectgarbage() + --mattorch.save(paths.concat(pathfolder,dataset.img_ids[i]..'.mat'),output:double()) + end + + for i = 1,dataset.num_classes do + -- go back through and prune out detections below the found threshold + for j = 1,dataset:size() do + if aboxes[i][j]:numel() > 0 then + local I = aboxes[i][j]:select(2,5):lt(thresh[i]) + local idx = torch.range(1,aboxes[i][j]:size(1)):long() + idx = idx[I] + if idx:numel()>0 then + aboxes[i][j] = aboxes[i][j]:index(1,idx) + end + end + end + save_file = paths.concat(pathfolder, dataset.classes[i].. '_boxes_'.. + dataset.dataset_name..self.suffix) + torch.save(save_file, aboxes) + end + + local res = {} + for i=1,dataset.num_classes do + local cls = dataset.classes[i] + res[i] = VOCevaldet(dataset,aboxes[i],cls) + end + res = torch.Tensor(res) + print('Results:') + -- print class names + io.write('|') + for i = 1, dataset.num_classes do + io.write(('%5s|'):format(dataset.classes[i])) + end + io.write('\n|') + -- print class scores + for i = 1, dataset.num_classes do + local l = #dataset.classes[i] < 5 and 5 or #dataset.classes[i] + local l = res[i] == res[i] and l-5 or l-3 + if l > 0 then + io.write(('%.3f%'..l..'s|'):format(res[i],' ')) + else + io.write(('%.3f|'):format(res[i])) + end + end + io.write('\n') + io.write(('mAP: %.4f\n'):format(res:mean(1)[1])) + + -- clean roidb to free memory + dataset.roidb = nil + return res +end diff --git a/nnf.lua b/nnf.lua index 2cf0592..f4610da 100644 --- a/nnf.lua +++ b/nnf.lua @@ -15,8 +15,10 @@ torch.include('nnf','ROIPooling.lua') torch.include('nnf','Trainer.lua') torch.include('nnf','Tester.lua') +torch.include('nnf','Tester_FRCNN.lua') torch.include('nnf','SVMTrainer.lua') torch.include('nnf','ImageTransformer.lua') +torch.include('nnf','ImageDetect.lua') --return nnf diff --git a/test_frcnn.lua b/test_frcnn.lua index 109c7cd..4db50a9 100644 --- a/test_frcnn.lua +++ b/test_frcnn.lua @@ -2,7 +2,7 @@ require 'nnf' require 'inn' require 'cudnn' -cutorch.setDevice(1) +cutorch.setDevice(2) dt = torch.load('pascal_2007_train.t7') if false then @@ -11,7 +11,7 @@ if false then roidbdir='/home/francisco/work/datasets/rcnn/selective_search_data' } else - ds = nnf.DataSetPascal{image_set='train', + ds = nnf.DataSetPascal{image_set='trainval', datadir='datasets/VOCdevkit', roidbdir='data/selective_search_data' } @@ -23,7 +23,7 @@ if false then ds.roidb[i] = torch.IntTensor(10,4):random(1,5) ds.roidb[i][{{},{3,4}}]:add(6) end -elseif true then +elseif false then ds.roidb = dt.roidb end @@ -123,8 +123,8 @@ do prl:add(features) prl:add(nn.Identity()) model:add(prl) - --model:add(nnf.ROIPooling(6,6):setSpatialScale(1/16)) - model:add(inn.ROIPooling(6,6):setSpatialScale(1/16)) + model:add(nnf.ROIPooling(6,6):setSpatialScale(1/16)) + --model:add(inn.ROIPooling(6,6):setSpatialScale(1/16)) model:add(nn.View(-1):setNumInputDims(3)) model:add(classifier) @@ -155,8 +155,8 @@ criterion = nn.CrossEntropyCriterion():cuda() display_iter = 20 ---inputs = {torch.CudaTensor(),torch.FloatTensor()} -inputs = {torch.CudaTensor(),torch.CudaTensor()} +inputs = {torch.CudaTensor(),torch.FloatTensor()} +--inputs = {torch.CudaTensor(),torch.CudaTensor()} target = torch.CudaTensor() function train() @@ -219,6 +219,18 @@ for i=1,num_iter do end if i%100 == 0 then - torch.save(paths.concat('cachedir','frcnn_t1.t7'),savedModel) + torch.save(paths.concat('cachedir','frcnn_t2.t7'),savedModel) end end + +-- test +dsv = nnf.DataSetPascal{image_set='test', + datadir='datasets/VOCdevkit', + roidbdir='data/selective_search_data' + } + + +local fpv = {dataset=dsv} +tester = nnf.Tester_FRCNN(model,fpv) +tester.cachefolder = 'cachedir/frcnn_t2' +tester:test(num_iter) From aac63f6ed268921a76eeb0439989f5f9c9f1fdb2 Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Sun, 23 Aug 2015 17:20:40 +0200 Subject: [PATCH 15/19] Force rois to be in cpu GPU single element access is much slower --- ROIPooling.lua | 10 ++++++++++ Tester_FRCNN.lua | 1 + 2 files changed, 11 insertions(+) diff --git a/ROIPooling.lua b/ROIPooling.lua index 0345628..af12402 100644 --- a/ROIPooling.lua +++ b/ROIPooling.lua @@ -29,6 +29,13 @@ function ROIPooling:updateOutput(input) rois[{{},4}]:cmin(s[ss]) rois[{{},5}]:cmin(s[ss-1]) + -- element access is faster if not a cuda tensor + if rois:type() == 'torch.CudaTensor' then + self._rois = self._rois or torch.FloatTensor() + self._rois:resize(rois:size()):copy(rois) + rois = self._rois + end + if not self._type then self._type = output:type() end if #self.pooler < num_rois then @@ -50,6 +57,9 @@ end function ROIPooling:updateGradInput(input,gradOutput) local data = input[1] local rois = input[2] + if rois:type() == 'torch.CudaTensor' then + rois = self._rois + end local num_rois = rois:size(1) local s = data:size() local ss = s:size(1) diff --git a/Tester_FRCNN.lua b/Tester_FRCNN.lua index 7e8a139..3a541e0 100644 --- a/Tester_FRCNN.lua +++ b/Tester_FRCNN.lua @@ -101,6 +101,7 @@ function Tester:test(iteration) io.write(('test: (%s) %5d/%-5d '):format(dataset.dataset_name,i,dataset:size())); boxes = dataset:getROIBoxes(i):float() local im = dataset:getImage(i) + timer3:reset() local output = detec:detect(im,boxes) local add_bg = 0 From 7b8b8e7a352ea2944f0f05540d307536eabc63f5 Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Sun, 30 Aug 2015 17:15:39 +0200 Subject: [PATCH 16/19] Fix wrong batch-size --- BatchProviderROI.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index 9844ee7..1e40bcf 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -53,8 +53,8 @@ function BatchProviderROI:permuteIdx() end function BatchProviderROI:selectBBoxes(fg_windows,bg_windows,im_scales,do_flip,im_sizes) - local fg_num_each = self.fg_num_each - local bg_num_each = self.bg_num_each + local fg_num_each = torch.round(self.fg_num_each/self.imgs_per_batch) + local bg_num_each = torch.round(self.bg_num_each/self.imgs_per_batch) local rois = {} local labels = {} @@ -121,7 +121,7 @@ local function getImages(self,img_ids,images,do_flip) if torch.round(im_scale*im_size_max) > self.max_size then im_scale = self.max_size/im_size_max end - local im_s = {im_size[1]*im_scale,im_size[2]*im_scale} + local im_s = {torch.round(im_size[1]*im_scale),torch.round(im_size[2]*im_scale)} table.insert(imgs,image.scale(im,im_s[2],im_s[1])) table.insert(im_sizes,im_s) table.insert(im_scales,im_scale) From 4a46bd0beb8b2db65e1caffe62b36399696b203b Mon Sep 17 00:00:00 2001 From: fsuzanomassa Date: Mon, 31 Aug 2015 23:04:41 +0200 Subject: [PATCH 17/19] Modifications in the training procedure Trying to mimic exactly Caffe's SGD --- test_frcnn.lua | 76 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/test_frcnn.lua b/test_frcnn.lua index 4db50a9..24bb23b 100644 --- a/test_frcnn.lua +++ b/test_frcnn.lua @@ -1,6 +1,7 @@ require 'nnf' require 'inn' require 'cudnn' +require 'gnuplot' cutorch.setDevice(2) @@ -92,7 +93,6 @@ do for i=1,14 do features:add(base_model:get(i):clone()) end - for i=17,22 do classifier:add(base_model:get(i):clone()) end @@ -108,7 +108,11 @@ do for i=1,14 do features:add(m1:get(i):clone()) end - + features:get(3).padW = 1 + features:get(3).padH = 1 + features:get(7).padW = 1 + features:get(7).padH = 1 + for i=2,7 do classifier:add(m2:get(i):clone()) end @@ -123,8 +127,8 @@ do prl:add(features) prl:add(nn.Identity()) model:add(prl) - model:add(nnf.ROIPooling(6,6):setSpatialScale(1/16)) - --model:add(inn.ROIPooling(6,6):setSpatialScale(1/16)) + --model:add(nnf.ROIPooling(6,6):setSpatialScale(1/16)) + model:add(inn.ROIPooling(6,6):setSpatialScale(1/16)) model:add(nn.View(-1):setNumInputDims(3)) model:add(classifier) @@ -134,8 +138,22 @@ print(model) model:cuda() parameters,gradParameters = model:getParameters() -optimState = {learningRate = 1e-3, weightDecay = 0.0005, momentum = 0.9, - learningRateDecay = 0} +parameters2,gradParameters2 = model:parameters() + +lr = {0,0,1,2,1,2,1,2,1,2,1,2,1,2,1,2} +wd = {0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0} + +local function updateGPlrwd(clr) + local clr = clr or 1 + for i,p in pairs(gradParameters2) do + p:add(wd[i]*0.0005,parameters2[i]) + p:mul(lr[i]*clr) + end +end + +optimState = {learningRate = 1,--1e-3, + weightDecay = 0.000, momentum = 0.9, + learningRateDecay = 0, dampening=0} -------------------------------------------------------------------------- -- training @@ -155,10 +173,12 @@ criterion = nn.CrossEntropyCriterion():cuda() display_iter = 20 -inputs = {torch.CudaTensor(),torch.FloatTensor()} ---inputs = {torch.CudaTensor(),torch.CudaTensor()} +--inputs = {torch.CudaTensor(),torch.FloatTensor()} +inputs = {torch.CudaTensor(),torch.CudaTensor()} target = torch.CudaTensor() +learningRate = 1e-3 + function train() local err = 0 for i=1,display_iter do @@ -182,6 +202,10 @@ function train() model:backward(inputs,df_do) + -- mimic different learning rates per layer + -- without the cost of having a huge tensor + updateGPlrwd(learningRate) + if normalize then gradParameters:div(batchSize) f = f/batchSize @@ -196,30 +220,52 @@ function train() err = err + fx[1] end print('Training error: '..err/display_iter) + return err/display_iter end epoch_size = math.ceil(ds:size()/bp.imgs_per_batch) -stepsize = 30000 +stepsize = 30000--30000 print_step = 10 -num_iter = 40000/display_iter--3000 +num_iter = 40000--40000 +num_iter = num_iter/display_iter--3000 confusion_matrix:zero() +train_err = {} +exp_name = 'frcnn_t11' +paths.mkdir(paths.concat('cachedir',exp_name)) +--logger = optim.Logger(paths.concat('cachedir',exp_name,'train_err.log')) +train_acc = {} for i=1,num_iter do - print(('Iteration: %d/%d'):format(i,num_iter)) + if i%(stepsize/display_iter) == 0 then - optimState.learningRate = optimState.learningRate/10 + --optimState.learningRate = optimState.learningRate/10 + learningRate = learningRate/10 end - train() + --print(('Iteration: %d/%d, lr: %.5f'):format(i,num_iter,optimState.learningRate)) + print(('Iteration: %d/%d, lr: %.5f'):format(i,num_iter,learningRate)) + + local t_err = train() + table.insert(train_err,t_err) + if i%print_step == 0 then print(confusion_matrix) + table.insert(train_acc,confusion_matrix.averageUnionValid*100) + gnuplot.epsfigure(paths.concat('cachedir',exp_name,'train_err.eps')) + gnuplot.plot('train',torch.Tensor(train_acc),'-') + gnuplot.xlabel('Iterations (200 batch update)') + gnuplot.ylabel('Training accuracy') + gnuplot.grid('on') + gnuplot.plotflush() + gnuplot.closeall() + confusion_matrix:zero() end if i%100 == 0 then - torch.save(paths.concat('cachedir','frcnn_t2.t7'),savedModel) + torch.save(paths.concat('cachedir',exp_name..'.t7'),savedModel) end end @@ -232,5 +278,5 @@ dsv = nnf.DataSetPascal{image_set='test', local fpv = {dataset=dsv} tester = nnf.Tester_FRCNN(model,fpv) -tester.cachefolder = 'cachedir/frcnn_t2' +tester.cachefolder = 'cachedir/'..exp_name tester:test(num_iter) From 4d0d46a6eb1c68a62923cce35239da2e421b5c30 Mon Sep 17 00:00:00 2001 From: Sergey Zagoruyko Date: Tue, 22 Sep 2015 12:04:57 -0400 Subject: [PATCH 18/19] fix bug in nms --- nms.lua | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/nms.lua b/nms.lua index ab12f63..2663b76 100644 --- a/nms.lua +++ b/nms.lua @@ -39,14 +39,22 @@ local function nms(boxes, overlap) I = I[{{1,last-1}}] xx1:index(x1,1,I) - xx1:clamp(0,x1[i]) yy1:index(y1,1,I) - yy1:clamp(0,y1[i]) xx2:index(x2,1,I) - xx2:clamp(x2[i],math.huge) yy2:index(y2,1,I) - yy2:clamp(y2[i],math.huge) - + + -- this code is wrong + -- xx1:clamp(0,x1[i]) + -- yy1:clamp(0,y1[i]) + -- xx2:clamp(x2[i],math.huge) + -- yy2:clamp(y2[i],math.huge) + + -- this code is right + xx1:clamp(x1[i],math.huge) + yy1:clamp(y1[i],math.huge) + xx2:clamp(0,x2[i]) + yy2:clamp(0,y2[i]) + w:resizeAs(xx2):zero() w:map2(xx2,xx1,function(xx,xxx2,xxx1) return math.max(xxx2-xxx1+1,0) end) h:resizeAs(yy2):zero() From 154313c1133cb1accb6932d9fb24a6da11330b51 Mon Sep 17 00:00:00 2001 From: Tingfan Wu Date: Wed, 23 Sep 2015 05:34:14 -0500 Subject: [PATCH 19/19] Fix BatchProviderROI incorrect super constructor invokation so the BatchProviderROI inherited fields can be properly saved by torch.save --- BatchProviderROI.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BatchProviderROI.lua b/BatchProviderROI.lua index 1e40bcf..38fa310 100644 --- a/BatchProviderROI.lua +++ b/BatchProviderROI.lua @@ -2,7 +2,7 @@ local BatchProviderROI, parent = torch.class('nnf.BatchProviderROI','nnf.BatchPr function BatchProviderROI:__init(dataset) local fp = {dataset=dataset} - parent:__init(fp) + parent.__init(self, fp) self.imgs_per_batch = 2 self.scale = 600 self.max_size = 1000