是否必须校准相机才能与 StereoBM 一起使用？
Posted
技术标签:
【中文标题】是否必须校准相机才能与 StereoBM 一起使用？【英文标题】：Do cameras have to be calibrated in order to be used with StereoBM? 【发布时间】：2017-10-11 03:57:43 【问题描述】：在我的情况下，OpenCV StereoBM Depth Map 返回的数据没有意义，无论参数调整如何。
我正在研究一个涉及 OpenCV 并使用立体视觉生成深度图的设计项目。我目前能够成功加载我的两个网络摄像头并使用 StereoBM 生成深度图。但是，结果数据目前没有用，如下面的屏幕截图所示。所以我创建了一个小型 python 应用程序，它可以帮助我调整 StereoBM 参数，但没有帮助。
我的问题是必须校准相机才能与 StereoBM 功能一起使用吗？
如果没有，有哪些替代方法可以帮助我改进结果（即提高分辨率、使用 StereoSBGM 等）
代码
import cv2
import time
import numpy as np
from Tkinter import *
oldVal = 15
def oddVals(n):
global oldVal
n = int(n)
if not n % 2:
window_size.set(n+1 if n > oldVal else n-1)
oldVal = window_size.get()
minDispValues = [16,32,48,64]
def minDispCallback(n):
n = int(n)
newvalue = min(minDispValues, key=lambda x:abs(x-float(n)))
min_disp.set(newvalue)
# Display the sliders to control the stereo vision
master = Tk()
master.title("StereoBM Settings");
min_disp = Scale(master, from_=16, to=64, command=minDispCallback, length=600, orient=HORIZONTAL, label="Minimum Disparities")
min_disp.pack()
min_disp.set(16)
window_size = Scale(master, from_=5, to=255, command=oddVals, length=600, orient=HORIZONTAL, label="Window Size")
window_size.pack()
window_size.set(15)
Disp12MaxDiff = Scale(master, from_=5, to=30, length=600, orient=HORIZONTAL, label="Max Difference")
Disp12MaxDiff.pack()
Disp12MaxDiff.set(0)
UniquenessRatio = Scale(master, from_=0, to=30, length=600, orient=HORIZONTAL, label="Uniqueness Ratio")
UniquenessRatio.pack()
UniquenessRatio.set(15)
SpeckleRange = Scale(master, from_=0, to=60, length=600, orient=HORIZONTAL, label="Speckle Range")
SpeckleRange.pack()
SpeckleRange.set(34)
SpeckleWindowSize = Scale(master, from_=60, to=150, length=600, orient=HORIZONTAL, label="Speckle Window Size")
SpeckleWindowSize.pack()
SpeckleWindowSize.set(100)
master.update()
vcLeft = cv2.VideoCapture(0) # Load video campture for the left camera
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420);
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340);
vcLeft.set(3,640) # Set camera width
vcLeft.set(4,480) # Set camera height
vcRight = cv2.VideoCapture(1) # Load video capture for the right camera
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420);
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340);
firstTime = time.time() # First time log
totalFramesPassed = 0 # Number of frames passed
if vcLeft.isOpened() and vcRight.isOpened():
rvalLeft, frameLeft = vcLeft.read()
rvalRight, frameRight = vcRight.read()
else:
rvalLeft = False
rvalRight = False
while rvalLeft and rvalRight: # If the cameras are opened
rvalLeft, frameLeft = vcLeft.read()
rvalRight, frameRight = vcRight.read()
cv2.putText(frameLeft, "FPS : " + str(totalFramesPassed / (time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10)
cv2.imshow("Left Camera", frameLeft)
cv2.putText(frameRight, "FPS : " + str(totalFramesPassed / (time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10)
cv2.imshow("Right Camera", frameRight)
frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)
frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)
num_disp = 112 - min_disp.get()
stereo = cv2.StereoBM_create(numDisparities = num_disp, blockSize = window_size.get())
stereo.setMinDisparity(min_disp.get())
stereo.setNumDisparities(num_disp)
stereo.setBlockSize(window_size.get())
stereo.setDisp12MaxDiff(Disp12MaxDiff.get())
stereo.setUniquenessRatio(UniquenessRatio.get())
stereo.setSpeckleRange(SpeckleRange.get())
stereo.setSpeckleWindowSize(SpeckleWindowSize.get())
disparity = stereo.compute(frameLeftNew, frameRightNew).astype(np.float32) / 16.0
disp_map = (disparity - min_disp.get())/num_disp
cv2.imshow("Disparity", disp_map)
master.update() # Update the slider options
key = cv2.waitKey(20)
totalFramesPassed = totalFramesPassed + 1 # One frame passed, increment
if key == 27:
break
vcLeft.release()
vcRight.release()
【问题讨论】：
校准相机 - 尤其是不失真会有所帮助。您是否尝试过将 StereoBM 与更多随机场景（文本、随机点等）一起使用？也许这是对应的问题。我会尝试实现简单的相关算法，看看相关性有多好。 @KamilSzelag 感谢您的建议。我将大胆地尝试校准左右图像，以便算法能够更好地工作。我尝试将 StereoBM 与其他场景一起使用，但我会检查我的场景中的相关性是否良好。我希望校准是这里的主要问题，因为我目前的结果，即使场景中有很多物体，也不可靠。再次感谢。 【参考方案1】：正如 StereoBM opencv stereoBM doc 的 opencv 文档中所述，这两个图像需要是“校正立体对”。
这意味着在计算视差之前，您需要校正两个摄像头。
查看stereo_match，您可以在其中了解如何在计算视差之前校正两个摄像头。
当您使用立体BM 计算视差时，您正在查看两个图像中平行核线的对应关系。 这意味着图像应该以这样的距离对齐，即两个图像中的相同行对应于空间中的相同行。整改过程会解决这个问题。
更多信息请查看Rectification with opencv
【讨论】：
【参考方案2】：我发现我们需要纠正该对才能使用 StereoBM 功能。此外，我发现虽然 StereoSGBM 功能更耗费资源，但它给了我更优化的结果。
如果将来有人需要校准他们的相机，您可以使用此代码来帮助您：
# Imports
import cv2
import numpy as np
# Constants
leftCameraNumber = 2 # Number for left camera
rightCameraNumber = 1 # Number for right camera
numberOfChessRows = 6
numberOfChessColumns = 8
chessSquareSize = 30 # Length of square in millimeters
numberOfChessColumns = numberOfChessColumns - 1 # Update to reflect how many corners are inside the chess board
numberOfChessRows = numberOfChessRows - 1
objp = np.zeros((numberOfChessColumns*numberOfChessRows,3), np.float32)
objp[:,:2] = np.mgrid[0:numberOfChessRows,0:numberOfChessColumns].T.reshape(-1,2)*chessSquareSize
objectPoints = []
leftImagePoints = []
rightImagePoints = []
parameterCriteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# Code
print("Press \"n\" when you're done caputing checkerboards.")
vcLeft = cv2.VideoCapture(leftCameraNumber) # Load video campture for the left camera
vcLeft.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2);
vcLeft.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2);
vcRight = cv2.VideoCapture(rightCameraNumber) # Load video capture for the right camera
vcRight.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2);
vcRight.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2);
if vcLeft.isOpened() and vcRight.isOpened():
rvalLeft, frameLeft = vcLeft.read()
rvalRight, frameRight = vcRight.read()
else:
rvalLeft = False
rvalRight = False
# Number of succesful recognitions
checkerboardRecognitions = 0
while rvalLeft and rvalRight: # If the cameras are opened
vcLeft.grab();
vcRight.grab();
rvalLeft, frameLeft = vcLeft.retrieve()
rvalRight, frameRight = vcRight.retrieve()
frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)
frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)
foundPatternLeft, cornersLeft = cv2.findChessboardCorners(frameLeftNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK)
foundPatternRight, cornersRight = cv2.findChessboardCorners(frameRightNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK)
if foundPatternLeft and foundPatternRight: # If found corners in this frame
# Process the images and display the count of checkboards in our array
checkerboardRecognitions = checkerboardRecognitions + 1
print("Checker board recognitions: " + str(checkerboardRecognitions))
objectPoints.append(objp)
exactCornersLeft = cv2.cornerSubPix(frameLeftNew, cornersLeft, (11, 11), (-1, -1), parameterCriteria);
leftImagePoints.append(exactCornersLeft)
exactCornersRight = cv2.cornerSubPix(frameRightNew, cornersRight, (11, 11), (-1, -1), parameterCriteria);
rightImagePoints.append(exactCornersRight)
frameLeft = cv2.drawChessboardCorners(frameLeft, (numberOfChessRows, numberOfChessColumns), (exactCornersLeft), True);
frameRight = cv2.drawChessboardCorners(frameRight, (numberOfChessRows, numberOfChessColumns), (exactCornersRight), True);
# Display current webcams regardless if board was found or not
cv2.imshow("Left Camera", frameLeft)
cv2.imshow("Right Camera", frameRight)
key = cv2.waitKey(250) # Give the frame some time
if key == ord('n'):
break
cameraMatrixLeft = np.zeros( (3,3) )
cameraMatrixRight = np.zeros( (3,3) )
distortionLeft = np.zeros( (8,1) )
distortionRight = np.zeros( (8,1) )
height, width = frameLeft.shape[:2]
rms, leftMatrix, leftDistortion, rightMatrix, rightDistortion, R, T, E, F = cv2.stereoCalibrate(objectPoints, leftImagePoints, rightImagePoints, cameraMatrixLeft, distortionLeft, cameraMatrixRight, distortionRight, (width, height),parameterCriteria, flags=0)
arr1 = np.arange(8).reshape(2, 4)
arr2 = np.arange(10).reshape(2, 5)
np.savez('camera_calibration.npz', leftMatrix=leftMatrix, leftDistortion=leftDistortion, rightMatrix=rightMatrix, rightDistortion=rightDistortion, R=R, T=T, E=E, F=F)
print("Calibration Settings Saved to File!")
print("RMS:")
print(rms)
print("Left Matrix:")
print(leftMatrix)
print("Left Distortion:")
print(leftDistortion)
print("Right Matrix:")
print(rightMatrix)
print("Right Distortion:")
print(rightDistortion)
print("R:")
print(R)
print("T:")
print(T)
print("E:")
print(E)
print("F:")
print(F)
leftRectTransform, rightRectTransform, leftProjMatrix, rightProjMatrix, _, _, _ = cv2.stereoRectify(leftMatrix, leftDistortion, rightMatrix, rightDistortion, (width, height), R, T, alpha=-1);
leftMapX, leftMapY = cv2.initUndistortRectifyMap(leftMatrix, leftDistortion, leftRectTransform, leftProjMatrix, (width, height), cv2.CV_32FC1);
rightMapX, rightMapY = cv2.initUndistortRectifyMap(rightMatrix, rightDistortion, rightRectTransform, rightProjMatrix, (width, height), cv2.CV_32FC1);
minimumDisparities = 0
maximumDisparities = 128
stereo = cv2.StereoSGBM_create(minimumDisparities, maximumDisparities, 18)
while True: # If the cameras are opened
vcLeft.grab();
vcRight.grab();
rvalLeft, frameLeft = vcLeft.retrieve()
rvalRight, frameRight = vcRight.retrieve()
frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)
frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)
leftRectified = cv2.remap(frameLeftNew, leftMapX, leftMapY, cv2.INTER_LINEAR);
rightRectified = cv2.remap(frameRightNew, rightMapX, rightMapY, cv2.INTER_LINEAR);
disparity = stereo.compute(leftRectified, rightRectified)
cv2.filterSpeckles(disparity, 0, 6000, maximumDisparities);
cv2.imshow("Normalized Disparity", (disparity/16.0 - minimumDisparities)/maximumDisparities);
cv2.imshow("Left Camera", leftRectified)
cv2.imshow("Right Camera", rightRectified)
key = cv2.waitKey(10) # Give the frame some time
if key == 27:
break
print("Finished!")
【讨论】：
以上是关于是否必须校准相机才能与 StereoBM 一起使用？的主要内容，如果未能解决你的问题，请参考以下文章