Pfthb

I have the following image:

from which I want to extract the boxes inside which a triangle is shown. I did it using simple vertical and horizontal line detection, which takes a total of 10 seconds on my machine.

Now, since this has to be done a lot of times, efficiency is of great concern. So, is there any method in openCV (or anywhere else) that can extract these boxes efficiently? Here an efficient method that takes less time than the brute force approach would suffice.

Any help is appreciated!

My approach:

I've made a function searchHorizontalLinesX that is used to search lines of particular lengths. It does so by going through every pixel once. My image has the dimensions: 2479 x 3504 = 8686416 pixels. (The image attached is of low quality, since the upload limit is 2MB. The image I am using is of more than 16 MB)

So, I input the width of the whole box (L1), and the width of box numbered 5(L2). Now, the function returns lines that have L1 length and L2 length. So, all the ones having L1 lengths can be used to extract boxes 1,2,3,4,6,7,8,9 and 10.

Once the lines are found, I search for the lines that have a difference of H between them, where H is the height of each box, for each box. Using these I get the boxes.

For the boxes that aren't the whole column that was extracted then I try to find a line that goes from the top to the bottom and then extract the image on the relevant side of the line.

def searchHorizontalLinesX(im,lengths,pg,gap=10):
 """
 im: PIL image object of the file
 lengths: list of lengths of lines to recognize
 pg: Page number of the page to take under consideration
 """
 im.seek(pg)
 dimX = im.size[0]
 dimY = im.size[1]
 pix = im.load()
 line = False 
 n = len(lengths) 
 linez = [ for i in range(0,n)]
 for j in range(0,dimY):
 for i in range(0,dimX):
 if(line):
 if(pix[i,j][0]!=0 or pix[i,j][1]!=255):
 line = False
 end = (i-1,j) 
 for num,l in enumerate(lengths):
 if(end[0]-start[0]>(l-gap) and end[0]-start[0]<(l+gap)):
 linez[num].append([start,end])
 elif(start[1]==j-1 and i==0):#End to end line
 end = (im.size[0],start[1])
 line = False
 for i,l in enumerate(lengths):
 if(end[0]-start[0]>(l-gap) and end[0]-start[0]<(l+gap)):
 linez[num].append([start,end])
 else:
 if(pix[i,j][0]==0 or pix[i,j][1]==255):
 start = (i,j) 
 line = True
 return linez

def searchVerticalLines(im,length,pg,gap=5):
 """
 im: PIL image object of the file
 length: Length of box
 dimX: Width of the page
 dimY: Height of the page
 pg: Page number of the page to take under consideration
 """
 im.seek(pg)
 dimX = im.size[0] 
 dimY = im.size[1]
 pix = im.load()
 line = False
 linez = 
 start = (0,0) 
 end = (0,0) 
 for i in range(0,dimX):
 for j in range(0,dimY):
 if(line):
 if(pix[i,j][0]!=0 or pix[i,j][1]!=255):
 line = False
 end = (i,j-1)
 if(end[1]-start[1]>(length-gap) and end[1]-start[1]<(length+gap)):
 linez.append([start,end])
 elif(start[0]==i-1 and j==0):
 line = False
 end = (start[0],im.size[1])
 if(end[1]-start[1]>(length-gap) and end[1]-start[1]<(length+gap)):
 linez.append([start,end])

 else:
 if(pix[i,j][0]==0 or pix[i,j][1]==255):
 start = (i,j)
 line = True 
 return linez

def grouping(hor,width):
 """
 Groups similar horizontal lines together.
 hor: List of lines in the format [[(start_x,start_y),(end_x,end_y)],...]
 width: What width can a line have. Takes all the lines inside this list into a group
 Returns: A list of groups. Each group is a list of lines. Each line has the format [(start_x,start_y),(end_x,end_y)]
 """
 horSet = set()
 for i in hor:
 horSet.add((i[0],i[1]))
 group = 
 count = 0 
 for i in hor:
 if (i[0],i[1]) in horSet:
 print("Visiting :"+str(i))
 for j in hor:
 if (j[0],j[1]) in horSet:
 if(i[0][0]==j[0][0] and i[1][0]==j[1][0] and abs(j[0][1]-i[0][1])<width and abs(j[1][1]-i[1][1])<width):
 if(len(group)<=count):
 group.append([j])
 else:
 group[count].append(j)
 horSet.remove((j[0],j[1]))
 count+=1
 return group
def groupingH(ver,width,heightDifference = 2):
 """
 Groups similar vertical lines together.
 hor: List of lines in the format [[(start_x,start_y),(end_x,end_y)],...]
 width: What width can a line have. Takes all the lines inside this list into a group
 Returns: A list of groups. Each group is a list of lines. Each line has the format [(start_x,start_y),(end_x,end_y)]
 """
 verSet = set()
 for i in ver:
 verSet.add((i[0],i[1]))
 group = 
 count = 0 
 for i in ver:
 if (i[0],i[1]) in verSet:
 for j in ver:
 if (abs(i[0][1]-j[0][1])< heightDifference and abs(i[1][1]-j[1][1])< heightDifference and abs(j[0][0]-i[0][0])<width and abs(j[1][0]-i[1][0])<width):
 if(len(group)<=count):
 group.append([j])
 else:
 group[count].append(j)
 verSet.remove((j[0],j[1]))
 count+=1
 return group




def boxing(groupList,boxHeight,gap=5,lw=4):
 """
 Given a groupList, it makes a list of coordinates of boxes that have a boxHeight height+-5
 """
 boxCoord = 
 groupSet = set()
 for i in range(len(groupList)):
 groupSet.add(i)
 for i,g in enumerate(groupList):
 if i in groupSet:
 print("On :"+str(i))
 for j,g1 in enumerate(groupList):
 if j in groupSet:
 print(" Checking :"+str(j))
 if(g[0][0][0]==g1[0][0][0] and g[0][1][0]==g1[0][1][0] and abs(g[len(g)-1][0][1]-g1[0][0][1])>=boxHeight-gap and abs(g[len(g)-1][0][1]-g1[0][0][1])<boxHeight+gap):
 groupSet.remove(j)
 print(" removing :"+str(j))
 boxCoord.append((g[0][0][0]+lw,min(g[len(g)-1][0][1],g1[0][0][1])+1,g[0][1][0]-lw,max(g[len(g)-1][0][1],g1[0][0][1])-1))
 break
 groupSet.remove(i)
 return boxCoord

##############################
def getFrontPageBoxes(im,pg,coords,margin=10):
 """
 coords[0]: List of lengths of horizontal lines to be recognized: 0th for box1, 1st for box2+3 (and 4,5,6,7,8,9,10) 
 coords[1]: Height of box 1
 coords[2]: Height of box 2,3
 coords[3]: Height of box 4,5,6,7
 coords[4]: Height of box 8,10
 coords[5]: Height of box 9
 """
 # Box 1: Police station... et cetera.
 t = 
 t1 = t.append(time())
 linez = searchHorizontalLinesX(im,coords[0],0) #0 for box 1 <-- Takes time
 t2 = t.append(time())
 groupForBox1 = grouping(linez[0],4)
 leftUpperCorner = groupForBox1[0][len(groupForBox1[0])-1][0]
 t3 = t.append(time())
 verLinez = searchVerticalLines(im,coords[1],0,10) # <-- Takes time
 t4 = t.append(time())
 rightLowerCorner = verLinez[0][1]
 box1 = im.crop((leftUpperCorner[0]+margin,leftUpperCorner[1]+margin,linez[0][0][1][0]-margin,rightLowerCorner[1]-margin))
 # Box 2+3: Polling station name and address
 groupForBox2 = grouping(linez[1],4)
 bxesFor2 = boxing(groupForBox2,coords[2])
 box2Uncropped = im.crop(bxesFor2[len(bxesFor2)-1]) # The last one is the one we want
 t5 = t.append(time())
 verLinesFor2 = searchVerticalLines(box2Uncropped,box2Uncropped.size[1],0,10) # <-- Takes time
 t6 = t.append(time())
 group2ForBox2 = groupingH(verLinesFor2,4)
 leftUpperCorner = (0,0)
 rightLowerCorner = group2ForBox2[0][0][1]
 box2 = box2Uncropped.crop((leftUpperCorner[0]+margin,leftUpperCorner[1]+margin,rightLowerCorner[0]-margin,rightLowerCorner[1]-margin))
 # Box 4,5,6,7:
 groupForBox4 = groupForBox2
 bxesFor4 = boxing(groupForBox4,coords[3])
 box4Uncropped = im.crop(bxesFor4[0])
 t7 = t.append(time())
 verLinesFor4 = searchVerticalLines(box4Uncropped,box4Uncropped.size[1],0,10) # <-- Takes time
 t8 = t.append(time())
 group2ForBox4 = groupingH(verLinesFor4,4)
 #There must be 5 groups.

 # Error raising mechanism needed here
 maleBox = box4Uncropped.crop((group2ForBox4[1][len(group2ForBox4[1])-1][0][0]+margin,0+margin,group2ForBox4[2][0][0][0]-margin,box4Uncropped.size[1]-margin))
 femaleBox = box4Uncropped.crop((group2ForBox4[2][len(group2ForBox4[2])-1][0][0]+margin,0+margin,group2ForBox4[3][0][0][0]-margin,box4Uncropped.size[1]-margin))
 third_gender = box4Uncropped.crop((group2ForBox4[3][len(group2ForBox4[3])-1][0][0]+margin,0+margin,group2ForBox4[4][0][0][0]-margin,box4Uncropped.size[1]-margin))
 total = box4Uncropped.crop((group2ForBox4[4][len(group2ForBox4[4])-1][0][0]+margin,0+margin,box4Uncropped.size[0]-margin,box4Uncropped.size[1]-margin))

 # Box 8, 10:
 groupForBox8 = groupForBox2
 bxesFor8 = boxing(groupForBox8,coords[4])
 box8Uncropped = im.crop(bxesFor8[0])
 t9 = t.append(time())
 verLinesFor8 = searchVerticalLines(box8Uncropped,box8Uncropped.size[1],0,10) # <--Takes time
 t10 = t.append(time())
 group2ForBox8 = groupingH(verLinesFor8,4)
 box10 = box8Uncropped.crop((group2ForBox8[0][len(group2ForBox8)-1][0][0]+margin,0+margin,box8Uncropped.size[0]-margin,box8Uncropped.size[1]-margin))
 box8 = box8Uncropped.crop((0+margin,0+margin,group2ForBox8[0][0][0][0]-margin,box8Uncropped.size[1]-margin))

 # Box 9:
 groupForBox9 = groupForBox2
 bxesFor9= boxing(groupForBox9,coords[5])
 box9 = im.crop((bxesFor9[0][0]+margin,bxesFor9[0][1]+margin,bxesFor9[0][2]-margin,bxesFor9[0][3]-margin))
 for i in range(0,len(t)-1):
 print('Time taken :'+str(t[i+1]-t[i]))
 return [box1,box2,maleBox,femaleBox,third_gender,total,box8,box9,box10]
def processBoxes(bxLst):
 opList = 
 for i,box in enumerate(bxLst):
 print("box :"+str(i))
 opList.append(pt.image_to_string(box,lang='hin+eng',config='--psm 6'))
 return opList

def getFrontPageInfo(im,pg,coords):
 """
 coords[0]: List of lengths of horizontal lines to be recognized: 0th for box1, 1st for box2+3, 
 2nd for boxes 4,5,6,7, 3rd for boxes 8 and 10, 4th for box 9
 coords[1]: List of lengths of vertical lines to be recognized: 0th for box1
 coords[2]: Height of box 2,3
 coords[3]: Height of box 4,5,6,7
 coords[4]: Height of box 8,10
 coords[5]: Height of box 9
 """
 boxList = getFrontPageBoxes(im,pg,coords)
 return processBoxes(boxList)

Look at my code with comment for more details. I just make a sample of only 1 box. Apply the same method to others.

import cv2

img = cv2.imread("1.png")


imgray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
ret,thresh = cv2.threshold(imgray,127,255,0)
cv2.bitwise_not(thresh,thresh)

im2, contours, hierarchy = cv2.findContours(thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)

#find the document in the image
max_area = 0
max_contour = None
ind = 0
for i,c in enumerate(contours):
 area = cv2.contourArea(c)
 if area>max_area:
 max_area = area
 max_contour = c
 ind = i
cv2.drawContours(img, contours, ind ,(0,0,255), 2)

#extract the document
rect = cv2.boundingRect(max_contour)
roi = img[rect[1]:rect[1]+rect[3],rect[0]:rect[0]+rect[2]]

(h,w) = roi.shape[:2]

#create a mask (the mask of box 1)
mask1 = (0,0,w*0.88,h*0.062) #the parameter 0.88 and 0.062 were found base on the format of the document
cv2.rectangle(roi,(mask1[0],mask1[1]),(int(mask1[2]),int(mask1[3])),(0,255,0),2)


cv2.imshow("img",img)
cv2.imshow("ROI",roi)
cv2.waitKey()
cv2.destroyAllWindows()