画像のクラスタリングや分類などのタスクでは、通常、画像を数値の特徴ベクトルに変換します。さて、画像全体の特徴ベクトルを計算する代わりに、画像のセグメント（矩形セグメントに制約されていない）のフィーチャを生成したいと思います。たとえば、SLICアルゴリズム（skimage.segmentation.slic）を使用すると、イメージをスーパーピクセルに分割できます。ここで、セクション5.3（画像セグメント（スーパーピクセル）の特徴ベクトルの計算
）のGold（Stephen、et al）に記載されているように、各セグメントのフィーチャ（領域サイズ、位置、色、形状およびテクスチャのフィーチャ）を生成したいと考えています。 "相対的な位置を事前に持つ複数クラスのセグメンテーション"国際コンピュータジャーナル80.3（2008）：300-316。
イメージとセグメントのマスクを指定すると、これらの機能を生成するのに役立つ既存のライブラリがPythonにありますか？私はskimageを使ってこれをすることができますか？
私はそのようなライブラリについて認識していません。しかし、以前は機能を自分で計算する必要がありました。下にいくつかのコードスニペットがあります。コードはPythonではありませんが、それはあなたの役に立つかもしれません。私はスーパーボクセルを実験しました。したがって、そこにPCLリファレンスがあるかもしれません。
あなたの機能を自分で実装を開始した場合、（例の表1の両方に）いくつかのアイデアのために、以下の資料を見て：ヘッダファイルからではないすべての定義が実際に実装されている
Derek Hoiem, Andrew N. Stein, Alexei A. Efros, Martial Hebert:
Recovering Occlusion Boundaries from a Single Image. ICCV 2007: 1-8
Joseph Tighe, Svetlana Lazebnik:
Superparsing - Scalable Nonparametric Image Parsing with Superpixels. International Journal of Computer Vision 101(2): 329-349 (2013)
注意。しかし、彼らはインスピレーションとして役立つかもしれません。
ヘッダー：
#ifndef SUPERPIXELFEATURES_H
#define SUPERPIXELFEATURES_H
#include <opencv2/opencv.hpp>
#include <pcl/point_cloud.h>
#include <pcl/point_types.h>
#include <Eigen/Dense>
#include <string>
namespace features {
/**
* Class SuperpixelFeatures represents a set of features computed for
* each superpixel in a given image.
*/
class SuperpixelFeatures {
public:
/**
* Construct superpixel features form only an image.
*
* @param image
* @param labels
*/
SuperpixelFeatures(const cv::Mat &image, int** labels);
/**
* Construct superpixel features from the image and its depth and
* a given superpixel segmentation.
*
* @param image
* @param depth
* @param labels
*/
SuperpixelFeatures(const cv::Mat &image, const cv::Mat &depth, int** labels);
/**
* Constructu superpixel features form the image and a point cloud and
* a given superpixel segmentation.
*
* @param image
* @param pointCloud
*/
SuperpixelFeatures(const cv::Mat &image, pcl::PointCloud<pcl::PointXYZ>::Ptr pointCloud, int** labels);
/**
* Destructor.
*/
~SuperpixelFeatures();
/**
* Add maximum color in each channel to the features.
*
* @return
*/
Eigen::Vector2i addMaximumColor();
/**
* Add minimum color in each channel to the features.
*
* @return
*/
Eigen::Vector2i addMinimumColor();
/**
* Add mean color to the features.
*
* @return
*/
Eigen::Vector2i addMeanBGRColor();
/**
* Add mean position to the features.
*
* @return
*/
Eigen::Vector2i addMean3DPosition();
/**
* Add mean position (pixel coordinates) to the features.
*
* @return
*/
Eigen::Vector2i addMean2DPosition();
/**
* Add the surface normal (mean normal) to the features.
*
* @return
*/
Eigen::Vector2i addMeanNormal();
/**
* Add a 3D bounding box of the superpixel to the features.
*
* @return
*/
Eigen::Vector2i addBoundingBox();
/**
* Add the compactness of the superpixel in its 2D sens to the features.
*
* @return
*/
Eigen::Vector2i addCompactness();
/**
* Add the area in pixels to the features.
*
* @return
*/
Eigen::Vector2i addArea();
/**
* Add the color covariance matrix to the features.
*
* @return
*/
Eigen::Vector2i addColorCovariance();
/**
* Add the position covariance matrix to the features.
* @return
*/
Eigen::Vector2i addPositionCovariance();
/**
* Add point-ness, curve-ness and surface-ness to the features.
*
* @return
*/
Eigen::Vector2i addSuperpixelStatistics();
/**
* Add a color histogram of the given number of bins to the features.
*
* @param bins
* @return
*/
Eigen::Vector2i addColorHistogram(int bins);
/**
* Add the ground truth label to the features.
*
* @param labels
* @return
*/
Eigen::Vector2i addGroundTruth(int** labels);
/**
* Get the dimension of the computed features.
*
* @return
*/
int getFeatureDimension() const;
/**
* Get the total number of superpixels.
*
* @return
*/
int getNumberOfSuperpixels() const;
/**
* Get pointer to comptued features.
*
* @return
*/
Eigen::MatrixXd* getFeatures() const;
protected:
void appendFeatures(Eigen::MatrixXd features);
cv::Mat* image;
int height;
int width;
int** labels;
int numberOfSuperpixels;
pcl::PointCloud<pcl::PointXYZ>::Ptr pointCloud;
bool pointCloudAvailable;
Eigen::MatrixXd* features;
};
}
出典：
#include <pcl/features/normal_3d.h>
#include <pcl/features/integral_image_normal.h>
#include "Tools.h"
#include "SuperpixelFeatures.h"
SuperpixelFeatures::SuperpixelFeatures(const cv::Mat &image, int** labels) {
this->image = new cv::Mat();
int channels = image.channels();
assert(channels == 1 || channels == 3);
if (channels == 1) {
image.convertTo(*this->image, CV_8UC1);
}
else if (channels == 3) {
image.convertTo(*this->image, CV_8UC3);
cv::cvtColor(*this->image, *this->image, SEEDS_REVISED_OPENCV_BGR2Lab, 3);
}
this->height = image.rows;
this->width = image.cols;
this->pointCloudAvailable = false;
// Copy labels.
this->labels = new int*[this->height];
for (int i = 0; i < this->height; ++i) {
this->labels[i] = new int[this->width];
for (int j = 0; j < this->width; ++j) {
this->labels[i][j] = labels[i][j];
}
}
this->numberOfSuperpixels = seeds_revised::tools::Integrity::countSuperpixels(this->labels, this->height, this->width);
seeds_revised::tools::Integrity::relabel(this->labels, this->height, this->width);
this->features = new Eigen::MatrixXd(this->numberOfSuperpixels, 1);
// Initialize first column with labels.
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
(*this->features)(label, 0) = label;
}
}
SuperpixelFeatures::SuperpixelFeatures(const cv::Mat &image, pcl::PointCloud<pcl::PointXYZ>::Ptr pointCloud, int** labels) {
assert(image.rows == (int) pointCloud->height);
assert(image.cols == (int) pointCloud->width);
this->image = new cv::Mat();
int channels = image.channels();
assert(channels == 1 || channels == 3);
if (channels == 1) {
image.convertTo(*this->image, CV_8UC1);
}
else if (channels == 3) {
image.convertTo(*this->image, CV_8UC3);
cv::cvtColor(*this->image, *this->image, SEEDS_REVISED_OPENCV_BGR2Lab, 3);
}
this->pointCloud = pointCloud;
this->height = pointCloud->height;
this->width = pointCloud->width;
this->pointCloudAvailable = true;
// Copy labels.
this->labels = new int*[this->height];
for (int i = 0; i < this->height; ++i) {
this->labels[i] = new int[this->width];
for (int j = 0; j < this->width; ++j) {
this->labels[i][j] = labels[i][j];
}
}
this->numberOfSuperpixels = seeds_revised::tools::Integrity::countSuperpixels(this->labels, this->height, this->width);
seeds_revised::tools::Integrity::relabel(this->labels, this->height, this->width);
this->features = new Eigen::MatrixXd(this->numberOfSuperpixels, 1);
// Initialize first column with labels.
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
(*this->features)(label, 0) = label;
}
}
SuperpixelFeatures::~SuperpixelFeatures() {
delete this->image;
for (int i = 0; i < this->height; ++i) {
delete[] this->labels[i];
}
delete[] this->labels;
}
Eigen::Vector2i SuperpixelFeatures::addMeanBGRColor() {
int cols = this->features->cols();
this->features->resize(this->numberOfSuperpixels, cols + 3);
double meanB = 0;
double meanG = 0;
double meanR = 0;
int count = 0;
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
meanB = 0;
meanG = 0;
meanR = 0;
count = 0;
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
if (this->labels[i][j] == label) {
meanB += this->image->at<cv::Vec3b>(i, j)[0];
meanG += this->image->at<cv::Vec3b>(i, j)[1];
meanR += this->image->at<cv::Vec3b>(i, j)[2];
++count;
}
}
}
(*this->features)(label, cols) = meanB/count;
(*this->features)(label, cols + 1) = meanG/count;
(*this->features)(label, cols + 2) = meanR/count;
}
return Eigen::Vector2i(cols, cols + 2);
}
Eigen::Vector2i SuperpixelFeatures::addMean3DPosition() {
assert(this->pointCloudAvailable);
int cols = this->features->cols();
this->features->resize(this->numberOfSuperpixels, cols + 3);
double meanX = 0;
double meanY = 0;
double meanZ = 0;
int count = 0;
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
meanX = 0;
meanY = 0;
meanZ = 0;
count = 0;
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
if (this->labels[i][j] == label) {
meanX += (*this->pointCloud)(j, i).x;
meanY += (*this->pointCloud)(j, i).y;
meanZ += (*this->pointCloud)(j, i).z;
++count;
}
}
}
(*this->features)(label, cols) = meanX/count;
(*this->features)(label, cols + 1) = meanY/count;
(*this->features)(label, cols + 2) = meanZ/count;
}
return Eigen::Vector2i(cols, cols + 2);
}
Eigen::Vector2i SuperpixelFeatures::addMean2DPosition() {
int cols = this->features->cols();
this->features->resize(this->numberOfSuperpixels, cols + 2);
double meanX = 0;
double meanY = 0;
int count = 0;
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
meanX = 0;
meanY = 0;
count = 0;
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
if (this->labels[i][j] == label) {
meanX += j;
meanY += i;
++count;
}
}
}
(*this->features)(label, cols) = meanX/count;
(*this->features)(label, cols + 1) = meanY/count;
}
return Eigen::Vector2i(cols, cols + 1);
}
Eigen::Vector2i SuperpixelFeatures::addMeanNormal() {
int cols = this->features->cols();
this->features->resize(this->numberOfSuperpixels, cols + 3);
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
std::vector<int> indices;
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
if (this->labels[i][j] == label) {
indices.push_back(i*cols + j);
}
}
}
Eigen::Vector4f superpixelCentroid;
Eigen::Matrix3f superpixelCovariance;
Eigen::Vector3f superpixelNormal;
pcl::compute3DCentroid(*pointCloud, indices, superpixelCentroid);
pcl::computeCovarianceMatrix(*pointCloud, indices, superpixelCentroid, superpixelCovariance);
Eigen::SelfAdjointEigenSolver<Eigen::Matrix3f> superpixelEigenValues(superpixelCovariance);
superpixelNormal = superpixelEigenValues.eigenvectors().col(0);
(*this->features)(label, cols) = superpixelNormal(0);
(*this->features)(label, cols + 1) = superpixelNormal(1);
(*this->features)(label, cols + 2) = superpixelNormal(2);
}
return Eigen::Vector2i(cols, cols + 2);
}
Eigen::Vector2i SuperpixelFeatures::addArea() {
int cols = this->features->cols();
this->features->resize(this->numberOfSuperpixels, cols + 1);
int area = 0;
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
area = 0;
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
if (this->labels[i][j] == label) {
++area;
}
}
}
(*this->features)(label, cols) = area;
}
return Eigen::Vector2i(cols, cols);
}
Eigen::Vector2i SuperpixelFeatures::addSuperpixelStatistics() {
assert(this->pointCloudAvailable);
int cols = this->features->cols();
this->features->resize(this->numberOfSuperpixels, cols + 3);
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
std::vector<int> indices;
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
if (this->labels[i][j] == label) {
indices.push_back(i*cols + j);
}
}
}
Eigen::Vector4f superpixelCentroid;
Eigen::Matrix3f superpixelCovariance;
Eigen::Vector3f superpixelNormal;
pcl::compute3DCentroid(*pointCloud, indices, superpixelCentroid);
pcl::computeCovarianceMatrix(*pointCloud, indices, superpixelCentroid, superpixelCovariance);
Eigen::SelfAdjointEigenSolver<Eigen::Matrix3f> superpixelEigenValues(superpixelCovariance);
// Point-ness:
(*this->features)(label, cols) = superpixelEigenValues.eigenvalues()(0);
(*this->features)(label, cols + 1) = superpixelEigenValues.eigenvalues()(2) - superpixelEigenValues.eigenvalues()(1);
(*this->features)(label, cols + 2) = superpixelEigenValues.eigenvalues()(1) - superpixelEigenValues.eigenvalues()(0);
}
return Eigen::Vector2i(cols, cols + 2);
}
Eigen::Vector2i SuperpixelFeatures::addColorHistogram(int bins) {
assert(bins > 0 && bins < 10);
int histogramSize = std::pow(bins, 3);
int cols = this->features->cols();
this->features->resize(this->numberOfSuperpixels, cols + histogramSize);
int* normalization = new int[this->numberOfSuperpixels];
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
normalization[label] = 0;
for (int k = 0; k < histogramSize; ++k) {
(*this->features)(label, cols + k) = 0;
}
}
int denominator = ceil(256./((double) bins));
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
int bin = this->image->at<cv::Vec3b>(i, j)[0]/denominator + bins*(this->image->at<cv::Vec3b>(i, j)[1]/denominator) + bins*bins*(this->image->at<cv::Vec3b>(i, j)[2]/denominator);
++(*this->features)(this->labels[i][j], cols + bin);
++normalization[this->labels[i][j]];
}
}
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
for (int k = 0; k < histogramSize; ++k) {
(*this->features)(label, cols + k) /= normalization[label];
}
}
return Eigen::Vector2i(cols, cols + histogramSize);
}
Eigen::Vector2i SuperpixelFeatures::addGroundTruth(int** labels) {
int numberOfLabels = 0;
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
if (labels[i][j] > numberOfLabels) {
numberOfLabels = labels[i][j];
}
}
}
// Remember that zero may be a label as well.
numberOfLabels = numberOfLabels + 1;
Eigen::MatrixXi intersection(this->numberOfSuperpixels, numberOfLabels);
for (int i = 0; i < this->height; ++i) {
for (int j = 0; j < this->width; ++j) {
assert(this->labels[i][j] < this->numberOfSuperpixels);
assert(labels[i][j] < numberOfLabels);
++intersection(this->labels[i][j], labels[i][j]);
}
}
for (int label = 0; label < this->numberOfSuperpixels; ++label) {
int maxIntersection = 0;
int maxGTLabel = 0;
for (int gtLabel = 0; gtLabel < numberOfLabels; ++gtLabel) {
if (intersection(label, gtLabel) > maxIntersection) {
maxIntersection = intersection(label, gtLabel);
maxGTLabel = gtLabel;
}
}
(*this->features)(label, 0) = maxGTLabel;
}
return Eigen::Vector2i(0, 0);
}
int SuperpixelFeatures::getFeatureDimension() const {
return this->features->cols();
}
Eigen::MatrixXd* SuperpixelFeatures::getFeatures() const {
return this->features;
}