StereoBMで使用するには、カメラのキャリブレーションが必要ですか？

Question

私の状況では、OpenCV StereoBM Depth Mapによって返されたデータは、パラメータチューニングに関係なく意味を持ちません。

私は、OpenCVを使用し、ステレオビジョンを使用して奥行きマップを生成する設計プロジェクトについて研究しています。私は現在、私のWebカメラを両方読み込んで、StereoBMを使って奥行きマップを生成することに成功しています。しかし、私のスクリーンショットが以下に示すように、結果データは現時点では有用ではありません。だから私は助けていないStereoBMパラメータを調整するのに役立つ小さなPythonアプリケーションを作った。

私の質問StereoBM機能で使用するには、カメラをキャリブレーションする必要がありますか？

、私は私の結果を改善するためにいくつかの選択肢が（つまりなど、StereoSBGMを使用し、解像度を上げる）どのようなものでない場合

コード

import cv2 
import time 
import numpy as np 
from Tkinter import * 

oldVal = 15 
def oddVals(n): 
     global oldVal 
     n = int(n) 
     if not n % 2: 
       window_size.set(n+1 if n > oldVal else n-1) 
       oldVal = window_size.get() 

minDispValues = [16,32,48,64] 
def minDispCallback(n): 
     n = int(n) 
     newvalue = min(minDispValues, key=lambda x:abs(x-float(n))) 
     min_disp.set(newvalue) 

# Display the sliders to control the stereo vision 
master = Tk() 

master.title("StereoBM Settings"); 

min_disp = Scale(master, from_=16, to=64, command=minDispCallback, length=600, orient=HORIZONTAL, label="Minimum Disparities") 
min_disp.pack() 
min_disp.set(16) 

window_size = Scale(master, from_=5, to=255, command=oddVals, length=600, orient=HORIZONTAL, label="Window Size") 
window_size.pack() 
window_size.set(15) 

Disp12MaxDiff = Scale(master, from_=5, to=30, length=600, orient=HORIZONTAL, label="Max Difference") 
Disp12MaxDiff.pack() 
Disp12MaxDiff.set(0) 

UniquenessRatio = Scale(master, from_=0, to=30, length=600, orient=HORIZONTAL, label="Uniqueness Ratio") 
UniquenessRatio.pack() 
UniquenessRatio.set(15) 

SpeckleRange = Scale(master, from_=0, to=60, length=600, orient=HORIZONTAL, label="Speckle Range") 
SpeckleRange.pack() 
SpeckleRange.set(34) 

SpeckleWindowSize = Scale(master, from_=60, to=150, length=600, orient=HORIZONTAL, label="Speckle Window Size") 
SpeckleWindowSize.pack() 
SpeckleWindowSize.set(100) 

master.update() 

vcLeft = cv2.VideoCapture(0) # Load video campture for the left camera 
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420); 
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340); 
vcLeft.set(3,640) # Set camera width 
vcLeft.set(4,480) # Set camera height 

vcRight = cv2.VideoCapture(1) # Load video capture for the right camera 
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420); 
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340); 

firstTime = time.time() # First time log 

totalFramesPassed = 0 # Number of frames passed 

if vcLeft.isOpened() and vcRight.isOpened(): 
     rvalLeft, frameLeft = vcLeft.read() 
     rvalRight, frameRight = vcRight.read() 

else: 
     rvalLeft = False 
     rvalRight = False 

while rvalLeft and rvalRight: # If the cameras are opened 

     rvalLeft, frameLeft = vcLeft.read() 

     rvalRight, frameRight = vcRight.read() 

     cv2.putText(frameLeft, "FPS : " + str(totalFramesPassed/(time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10) 

     cv2.imshow("Left Camera", frameLeft) 

     cv2.putText(frameRight, "FPS : " + str(totalFramesPassed/(time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10) 

     cv2.imshow("Right Camera", frameRight) 

     frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY) 

     frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY) 

     num_disp = 112 - min_disp.get() 

     stereo = cv2.StereoBM_create(numDisparities = num_disp, blockSize = window_size.get()) 

     stereo.setMinDisparity(min_disp.get()) 

     stereo.setNumDisparities(num_disp) 

     stereo.setBlockSize(window_size.get()) 

     stereo.setDisp12MaxDiff(Disp12MaxDiff.get()) 

     stereo.setUniquenessRatio(UniquenessRatio.get()) 

     stereo.setSpeckleRange(SpeckleRange.get()) 

     stereo.setSpeckleWindowSize(SpeckleWindowSize.get()) 

     disparity = stereo.compute(frameLeftNew, frameRightNew).astype(np.float32)/16.0 

     disp_map = (disparity - min_disp.get())/num_disp 

     cv2.imshow("Disparity", disp_map) 

     master.update() # Update the slider options 

     key = cv2.waitKey(20) 

     totalFramesPassed = totalFramesPassed + 1 # One frame passed, increment 

     if key == 27: 

       break 


vcLeft.release() 

vcRight.release() 

Answer 1

で述べたようにStereoBM opencv stereoBM docのopencvのドキュメントでは、2つのイメージは「整流されたステレオペア」である必要があります。

これは、視差を計算する前に、2台のカメラを修正する必要があることを意味します。

視差を計算する前に、2台のカメラを修正する方法がわかるstereo_matchをご覧ください。

stereoBMを使用して視差を計算するときは、両方の画像の平行なエピポーラ線の対応が見えます。 これは、両方のイメージの同じ行が同じ行のスペースに対応するようにイメージが整列することが予想されることを意味します。整流プロセスがそれを処理します。詳細については

は、私たちはStereoBM機能を使用するためにペアを是正する必要があることが判明Rectification with opencv

Answer 2

を見てください。さらに、リソース集中型ではあるが、StereoSGBM機能により最適な結果が得られることが分かった。 - 役立つ特に歪みのない

# Imports 
import cv2 
import numpy as np 

# Constants 
leftCameraNumber = 2 # Number for left camera 
rightCameraNumber = 1 # Number for right camera 

numberOfChessRows = 6 
numberOfChessColumns = 8 
chessSquareSize = 30 # Length of square in millimeters 

numberOfChessColumns = numberOfChessColumns - 1 # Update to reflect how many corners are inside the chess board 
numberOfChessRows = numberOfChessRows - 1 

objp = np.zeros((numberOfChessColumns*numberOfChessRows,3), np.float32) 
objp[:,:2] = np.mgrid[0:numberOfChessRows,0:numberOfChessColumns].T.reshape(-1,2)*chessSquareSize 

objectPoints = [] 
leftImagePoints = [] 
rightImagePoints = [] 

parameterCriteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001) 

# Code 
print("Press \"n\" when you're done caputing checkerboards.") 

vcLeft = cv2.VideoCapture(leftCameraNumber) # Load video campture for the left camera 
vcLeft.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2); 
vcLeft.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2); 

vcRight = cv2.VideoCapture(rightCameraNumber) # Load video capture for the right camera 
vcRight.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2); 
vcRight.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2); 

if vcLeft.isOpened() and vcRight.isOpened(): 
    rvalLeft, frameLeft = vcLeft.read() 
    rvalRight, frameRight = vcRight.read() 

else: 
    rvalLeft = False 
    rvalRight = False 

# Number of succesful recognitions 
checkerboardRecognitions = 0 

while rvalLeft and rvalRight: # If the cameras are opened 

    vcLeft.grab(); 

    vcRight.grab(); 

    rvalLeft, frameLeft = vcLeft.retrieve() 

    rvalRight, frameRight = vcRight.retrieve() 

    frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY) 

    frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY) 

    foundPatternLeft, cornersLeft = cv2.findChessboardCorners(frameLeftNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK) 

    foundPatternRight, cornersRight = cv2.findChessboardCorners(frameRightNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK) 


    if foundPatternLeft and foundPatternRight: # If found corners in this frame 

     # Process the images and display the count of checkboards in our array 
     checkerboardRecognitions = checkerboardRecognitions + 1 
     print("Checker board recognitions: " + str(checkerboardRecognitions)) 

     objectPoints.append(objp) 

     exactCornersLeft = cv2.cornerSubPix(frameLeftNew, cornersLeft, (11, 11), (-1, -1), parameterCriteria); 
     leftImagePoints.append(exactCornersLeft) 

     exactCornersRight = cv2.cornerSubPix(frameRightNew, cornersRight, (11, 11), (-1, -1), parameterCriteria); 
     rightImagePoints.append(exactCornersRight) 

     frameLeft = cv2.drawChessboardCorners(frameLeft, (numberOfChessRows, numberOfChessColumns), (exactCornersLeft), True); 

     frameRight = cv2.drawChessboardCorners(frameRight, (numberOfChessRows, numberOfChessColumns), (exactCornersRight), True); 


    # Display current webcams regardless if board was found or not 
    cv2.imshow("Left Camera", frameLeft) 

    cv2.imshow("Right Camera", frameRight) 


    key = cv2.waitKey(250) # Give the frame some time 

    if key == ord('n'): 

     break 

cameraMatrixLeft = np.zeros((3,3)) 
cameraMatrixRight = np.zeros((3,3)) 
distortionLeft = np.zeros((8,1)) 
distortionRight = np.zeros((8,1)) 
height, width = frameLeft.shape[:2] 

rms, leftMatrix, leftDistortion, rightMatrix, rightDistortion, R, T, E, F = cv2.stereoCalibrate(objectPoints, leftImagePoints, rightImagePoints, cameraMatrixLeft, distortionLeft, cameraMatrixRight, distortionRight, (width, height),parameterCriteria, flags=0) 

arr1 = np.arange(8).reshape(2, 4) 
arr2 = np.arange(10).reshape(2, 5) 
np.savez('camera_calibration.npz', leftMatrix=leftMatrix, leftDistortion=leftDistortion, rightMatrix=rightMatrix, rightDistortion=rightDistortion, R=R, T=T, E=E, F=F) 
print("Calibration Settings Saved to File!") 

print("RMS:") 
print(rms) 
print("Left Matrix:") 
print(leftMatrix) 
print("Left Distortion:") 
print(leftDistortion) 
print("Right Matrix:") 
print(rightMatrix) 
print("Right Distortion:") 
print(rightDistortion) 
print("R:") 
print(R) 
print("T:") 
print(T) 
print("E:") 
print(E) 
print("F:") 
print(F) 


leftRectTransform, rightRectTransform, leftProjMatrix, rightProjMatrix, _, _, _ = cv2.stereoRectify(leftMatrix, leftDistortion, rightMatrix, rightDistortion, (width, height), R, T, alpha=-1); 
leftMapX, leftMapY = cv2.initUndistortRectifyMap(leftMatrix, leftDistortion, leftRectTransform, leftProjMatrix, (width, height), cv2.CV_32FC1); 
rightMapX, rightMapY = cv2.initUndistortRectifyMap(rightMatrix, rightDistortion, rightRectTransform, rightProjMatrix, (width, height), cv2.CV_32FC1); 

minimumDisparities = 0 
maximumDisparities = 128 

stereo = cv2.StereoSGBM_create(minimumDisparities, maximumDisparities, 18) 

while True: # If the cameras are opened 
    vcLeft.grab(); 

    vcRight.grab(); 

    rvalLeft, frameLeft = vcLeft.retrieve() 

    rvalRight, frameRight = vcRight.retrieve() 

    frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY) 

    frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY) 

    leftRectified = cv2.remap(frameLeftNew, leftMapX, leftMapY, cv2.INTER_LINEAR); 

    rightRectified = cv2.remap(frameRightNew, rightMapX, rightMapY, cv2.INTER_LINEAR); 

    disparity = stereo.compute(leftRectified, rightRectified) 

    cv2.filterSpeckles(disparity, 0, 6000, maximumDisparities); 

    cv2.imshow("Normalized Disparity", (disparity/16.0 - minimumDisparities)/maximumDisparities); 

    cv2.imshow("Left Camera", leftRectified) 

    cv2.imshow("Right Camera", rightRectified) 


    key = cv2.waitKey(10) # Give the frame some time 

    if key == 27: 

     break 

print("Finished!")