Given a directory of superpixel segmentations and a directory of ground truth segmentations, this class is used to generate a CSV file of statistics of different metrics. More...

#include <evaluation_summary.h>

Classes
struct	EvaluationMetrics
	Available metrics, see evaluation.h. More...

struct	EvaluationStatistics
	Available statistics that may be computed for each metric. More...

struct	SuperpixelVisualizations
	Available superpixel visualizations, see visualization.h. More...

Public Member Functions
	EvaluationSummary (boost::filesystem::path sp_directory, boost::filesystem::path gt_directory, boost::filesystem::path img_directory)
	Create an evaluation summary for the given directory. More...

	EvaluationSummary (boost::filesystem::path sp_directory, boost::filesystem::path gt_directory, boost::filesystem::path img_directory, EvaluationMetrics evaluation_metrics, EvaluationStatistics evaluation_statistics)
	Constructor. More...

	EvaluationSummary (boost::filesystem::path sp_directory, boost::filesystem::path gt_directory, boost::filesystem::path img_directory, EvaluationMetrics evaluation_metrics, EvaluationStatistics evaluation_statistics, SuperpixelVisualizations superpixel_visualizations)
	Constructor. More...

void	computeSummary (int &gt_max)
	Summarize the results; this is the main entrance point. More...

void	setAppendFile (const boost::filesystem::path &append_file)
	Add CSV file to append CSV output to. More...

void	setEvaluationStatistics (const EvaluationStatistics &evaluation_statistics)
	Set the evaluation statistics to compute. More...

void	setEvaluationMetrics (const EvaluationMetrics &evaluation_metrics)
	Set the evaluation metrics to compute. More...

void	setSuperpixelVisualizations (const SuperpixelVisualizations &superpixel_visualizations)
	Set the superpixel visualizations to compute. More...

void	setComputeCorrelation (bool compute_correlation)
	Set whether to compute correlation. More...

void	getEvaluationStatistics (EvaluationSummary::EvaluationStatistics &evaluation_statistics)
	Get the evaluation statistics used. More...

void	getEvaluationMetrics (EvaluationSummary::EvaluationMetrics &evaluation_metrics)
	Get the evaluation metrics used. More...

void	getSuperpixelVisualizations (EvaluationSummary::SuperpixelVisualizations &superpixel_visualizations)
	Get the visualizations used. More...

bool	getComputeCorrelation ()
	Get whether correlation is computed. More...

Protected Member Functions
int	countMetrics ()
	Count number of metrics used. More...

void	evaluateHeader (std::stringstream &output, std::vector< std::string > &metric_order)
	Add header to output. More...

void	evaluate (const cv::Mat &sp_segmentation, const cv::Mat &gt_segmentation, const cv::Mat &image, cv::Mat &data, std::stringstream &output)
	Actually do the evaluation. More...

void	visualize (const cv::Mat &sp_segmentation, const cv::Mat &gt_segmentation, const cv::Mat &image, std::string name, int t=0)
	Visualize given segmentation. More...

void	correlate (const cv::Mat &mat_results, const std::vector< std::string > &metric_order, cv::Mat &mat_correlation, std::string &csv_correlation)
	Compute correlation between all metrics. More...

int	countStatistics (int gt_max)
	Count the number of statistics to be used. More...

void	validateStatistics ()
	Validate and correct statistics. More...

void	summaryHeader (const std::vector< int > &gt, std::stringstream &output)
	Add the summary header to output. More...

void	summarize (const std::vector< int > &gt, const cv::Mat &data, int j, cv::Mat &mat_summary, std::stringstream &output)
	Summarize a specific column of the evaluation data (corresponding to a particular metric). More...

float	computeMean (const std::vector< int > gt, const cv::Mat &data, int j, std::vector< float > &mean, float &mean_min, float &mean_max)
	Compute mean statistic. More...

float	comuteMedian (const std::vector< int > gt, const cv::Mat &data, int j, std::vector< float > &median, float &median_min, float &median_max)
	Compute median statistic. More...

void	comuteMedianFirstAndThirdQuartile (const std::vector< int > gt, const cv::Mat &data, int j, std::vector< float > &median, std::vector< float > &first_q, std::vector< float > &thrid_q, float &median_min, float &first_q_min, float &third_q_min, float &median_max, float &first_q_max, float &second_q_max)
	Compute median, first quartile and third quartile. More...

void	comuteMinMax (const std::vector< int > gt, const cv::Mat &data, int j, std::vector< float > &min, std::vector< float > &max, float &min_min, float &max_min, float &min_max, float &max_max)
	Compute min/max over all ground truth segmentations. More...

float	computeStandardDeviation (const std::vector< int > gt, const cv::Mat &data, int j, const std::vector< float > &mean, std::vector< float > &std, float &std_min, float &std_max)
	Compute standard deviation over all ground truth segmentations. More...

Protected Attributes
EvaluationMetrics	evaluation_metrics
	Evaluation metrics to use. More...

EvaluationStatistics	evaluation_statistics
	Evaluation statistics to use. More...

SuperpixelVisualizations	superpixel_visualizations
	Visualizations to compute. More...

bool	compute_correlation
	Whether to compute correlation. More...

boost::filesystem::path	sp_directory
	Directory of superpixel segmentations. More...

boost::filesystem::path	gt_directory
	Directory of ground truth segmentations. More...

boost::filesystem::path	img_directory
	Directory of images. More...

boost::filesystem::path	results_file
	Path to results file. More...

boost::filesystem::path	correlation_file
	Path to correlation file. More...

boost::filesystem::path	summary_file
	Path to summary file. More...

boost::filesystem::path	vis_directory
	Path to visualization directory. More...

boost::filesystem::path	append_file
	Path to file to append summary to. More...

Detailed Description

Given a directory of superpixel segmentations and a directory of ground truth segmentations, this class is used to generate a CSV file of statistics of different metrics.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Author: David Stutz

Constructor & Destructor Documentation

EvaluationSummary::EvaluationSummary	(	boost::filesystem::path	sp_directory,
		boost::filesystem::path	gt_directory,
		boost::filesystem::path	img_directory
	)

Create an evaluation summary for the given directory.

The directory is assumed to contain several superpixel segmentations as CSV files. The files are assumed to be named according to the ground truth files, see below.

In addition, a directory containing ground truth segmentations as CSV files needs to be provided.

Parameters

[in]	sp_directory	directory containing superpixel segmentations as CSV files
[in]	gt_directory	directory containing ground truth segmentations as CSV files
[in]	img_directory	directory containing the corresponding images as JPEG or PNG

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

EvaluationSummary::EvaluationSummary	(	boost::filesystem::path	sp_directory,
		boost::filesystem::path	gt_directory,
		boost::filesystem::path	img_directory,
		EvaluationMetrics	evaluation_metrics,
		EvaluationStatistics	evaluation_statistics
	)

Constructor.

Parameters

[in]	sp_directory	directory containing superpixel segmentations as CSV files
[in]	gt_directory	directory containing ground truth segmentations as CSV files
[in]	img_directory	directory containing the corresponding images as JPEG or PNG
[in]	evaluation_metrics	evaluation metrics to use
[in]	evaluation_statistics	evaluation statistics to compute

EvaluationSummary::EvaluationSummary	(	boost::filesystem::path	sp_directory,
		boost::filesystem::path	gt_directory,
		boost::filesystem::path	img_directory,
		EvaluationMetrics	evaluation_metrics,
		EvaluationStatistics	evaluation_statistics,
		SuperpixelVisualizations	superpixel_visualizations
	)

Constructor.

Parameters

[in]	sp_directory	directory containing superpixel segmentations as CSV files
[in]	gt_directory	directory containing ground truth segmentations as CSV files
[in]	img_directory	directory containing the corresponding images as JPEG or PNG
[in]	evaluation_metrics	evaluation metrics to use
[in]	evaluation_statistics	evaluation statistics to compute
[in]	superpixel_visualizations	visualizations to create

Member Function Documentation

float EvaluationSummary::computeMean	(	const std::vector< int >	gt,
		const cv::Mat &	data,
		int	j,
		std::vector< float > &	mean,
		float &	mean_min,
		float &	mean_max
	)

protected

Compute mean statistic.

Parameters

[in]	gt	ground truth indices to determine the number of ground truth segmentations
[in]	data	data to summarize
[in]	j	the column
[out]	mean	computed means as vector for each ground truth segmentation
[out]	mean_min	minimum of means in case of multiple ground truth segmentations per image
[out]	mean_max	maximum of means in case of multiple ground truth segmentations per image

float EvaluationSummary::computeStandardDeviation	(	const std::vector< int >	gt,
		const cv::Mat &	data,
		int	j,
		const std::vector< float > &	mean,
		std::vector< float > &	std,
		float &	std_min,
		float &	std_max
	)

protected

Compute standard deviation over all ground truth segmentations.

Parameters

[in]	gt	ground truth indices to determine the number of ground truth segmentations
[in]	data	data to summarize
[in]	j	the column
[out]	mean	means as vector for each ground truth segmentation
[out]	std	std as vector for each ground truth segmentation
[out]	std_min	minimum of std over all ground truth segmentations
[out]	std_max	maximum of std over all ground truth segmentations

void EvaluationSummary::computeSummary ( int & gt_max )

Summarize the results; this is the main entrance point.

Parameters

[out] gt_max the maxmimum number of ground truth used, for BSDS 5 for all other 1

float EvaluationSummary::comuteMedian	(	const std::vector< int >	gt,
		const cv::Mat &	data,
		int	j,
		std::vector< float > &	median,
		float &	median_min,
		float &	median_max
	)

protected

Compute median statistic.

Parameters

[in]	gt	ground truth indices to determine the number of ground truth segmentations
[in]	data	data to summarize
[in]	j	the column
[out]	median	computed medians as vector for each ground truth segmentation
[out]	median_min	minimum of medians in case of multiple ground truth segmentations per image
[out]	median_max	maximum of medians in case of multiple ground truth segmentations per image

void EvaluationSummary::comuteMedianFirstAndThirdQuartile	(	const std::vector< int >	gt,
		const cv::Mat &	data,
		int	j,
		std::vector< float > &	median,
		std::vector< float > &	first_q,
		std::vector< float > &	thrid_q,
		float &	median_min,
		float &	first_q_min,
		float &	third_q_min,
		float &	median_max,
		float &	first_q_max,
		float &	second_q_max
	)

protected

Compute median, first quartile and third quartile.

Parameters

[in]	gt	ground truth indices to determine the number of ground truth segmentations
[in]	data	data to summarize
[in]	j	the column
[out]	median	computed medians as vector for each ground truth segmentation
[out]	first_q	computed first quantiles as vector for each ground truth segmentation
[out]	thrid_q	computed third quantiles as vector for each ground truth segmentation
[out]	median_min	minimum of medians in case of multiple ground truth segmentations per image
[out]	first_q_min	minimum of first quantiles in case of multiple ground truth segmentations per image
[out]	third_q_min	minimum of quantiles in case of multiple ground truth segmentations per image
[out]	median_max	maximum of medians in case of multiple ground truth segmentations per image
[out]	first_q_max	maximum of first quantiles in case of multiple ground truth segmentations per image
[out]	second_q_max	maximum of quantiles in case of multiple ground truth segmentations per image

void EvaluationSummary::comuteMinMax	(	const std::vector< int >	gt,
		const cv::Mat &	data,
		int	j,
		std::vector< float > &	min,
		std::vector< float > &	max,
		float &	min_min,
		float &	max_min,
		float &	min_max,
		float &	max_max
	)

protected

Compute min/max over all ground truth segmentations.

Parameters

[in]	gt	ground truth indices to determine the number of ground truth segmentations
[in]	data	data to summarize
[in]	j	the column
[out]	min	minimum as vector for each ground truth segmentation
[out]	max	maximum as vector for each ground truth segmentation
[out]	min_min	minimum of minimum over all ground truth segmentations
[out]	max_min	maximum of minimum over all ground truth segmentations
[out]	min_max	minimum of maximum over all ground truth segmentations
[out]	max_max	maximum of maximum over all ground truth segmentations

void EvaluationSummary::correlate	(	const cv::Mat &	mat_results,
		const std::vector< std::string > &	metric_order,
		cv::Mat &	mat_correlation,
		std::string &	csv_correlation
	)

protected

Compute correlation between all metrics.

Parameters

[in]	mat_results	matrix containing results in order to compute correlatiom
[in]	metric_order	order of metrix used in mat_results
[out]	mat_correlation	correlation matrix
[out]	csv_correlation	correlation matrix as CSV string

int EvaluationSummary::countMetrics ( )

protected

Count number of metrics used.

Returns: number of metrics to compute

int EvaluationSummary::countStatistics ( int gt_max )

protected

Count the number of statistics to be used.

Parameters

[in] gt_max maximum number of ground truth segmentations to consider

Returns: number of statistics

void EvaluationSummary::evaluate	(	const cv::Mat &	sp_segmentation,
		const cv::Mat &	gt_segmentation,
		const cv::Mat &	image,
		cv::Mat &	data,
		std::stringstream &	output
	)

protected

Actually do the evaluation.

Parameters

[in]	sp_segmentation	superpixel labels as int image
[in]	gt_segmentation	ground truth segmentation as int image
[in]	image	corresponding image
[in]	data	data matrix to append results to
[in]	output	CSV file stream to append results to

void EvaluationSummary::evaluateHeader	(	std::stringstream &	output,
		std::vector< std::string > &	metric_order
	)

protected

Add header to output.

Parameters

[in]	output	the header of the CSV file to create
[out]	metric_order	the order of the metrics

bool EvaluationSummary::getComputeCorrelation ( )

Get whether correlation is computed.

Returns: whether to compute correlation

void EvaluationSummary::getEvaluationMetrics ( EvaluationSummary::EvaluationMetrics & evaluation_metrics )

Get the evaluation metrics used.

Parameters

[out] evaluation_metrics evaluation metrics used

void EvaluationSummary::getEvaluationStatistics ( EvaluationSummary::EvaluationStatistics & evaluation_statistics )

Get the evaluation statistics used.

Parameters

[out] evaluation_statistics evaluation statistics used

void EvaluationSummary::getSuperpixelVisualizations ( EvaluationSummary::SuperpixelVisualizations & superpixel_visualizations )

Get the visualizations used.

Parameters

[out] superpixel_visualizations visualizations used

void EvaluationSummary::setAppendFile ( const boost::filesystem::path & append_file )

Add CSV file to append CSV output to.

Parameters

[in] append_file path to CSV file to append to

void EvaluationSummary::setComputeCorrelation ( bool compute_correlation )

Set whether to compute correlation.

Parameters

[in] compute_correlation whether to compute correlation matrix

void EvaluationSummary::setEvaluationMetrics ( const EvaluationMetrics & evaluation_metrics )

Set the evaluation metrics to compute.

Parameters

[in] evaluation_metrics evaluation metrics to compute

void EvaluationSummary::setEvaluationStatistics ( const EvaluationStatistics & evaluation_statistics )

Set the evaluation statistics to compute.

Parameters

[in] evaluation_statistics evaluation statistics to compute

void EvaluationSummary::setSuperpixelVisualizations ( const SuperpixelVisualizations & superpixel_visualizations )

Set the superpixel visualizations to compute.

Parameters

[in] superpixel_visualizations visualizations to compute

void EvaluationSummary::summarize	(	const std::vector< int > &	gt,
		const cv::Mat &	data,
		int	j,
		cv::Mat &	mat_summary,
		std::stringstream &	output
	)

protected

Summarize a specific column of the evaluation data (corresponding to a particular metric).

Parameters

[in]	gt	ground truth indices to determine the number of ground truth segmentations
[in]	data	data to summarize
[in]	j	the column
[out]	mat_sumamry	summary as matrix
[out]	output	summary as CSV string

void EvaluationSummary::summaryHeader	(	const std::vector< int > &	gt,
		std::stringstream &	output
	)

protected

Add the summary header to output.

Parameters

[in] gt ground truth indices to determine the number of ground truth segmentations

void EvaluationSummary::validateStatistics ( )

protected

Validate and correct statistics.

void EvaluationSummary::visualize	(	const cv::Mat &	sp_segmentation,
		const cv::Mat &	gt_segmentation,
		const cv::Mat &	image,
		std::string	name,
		int	t = `0`
	)

protected

Visualize given segmentation.

Parameters

[in]	sp_segmentation	superpixel labels as int image
[in]	gt_segmentation	ground truth segmentation as int image
[in]	image	corresponding image
[in]	name	name or identifier of image to visualize
[in]	t	ground truth number

Member Data Documentation

boost::filesystem::path EvaluationSummary::append_file

protected

Path to file to append summary to.

bool EvaluationSummary::compute_correlation

protected

Whether to compute correlation.

boost::filesystem::path EvaluationSummary::correlation_file

protected

Path to correlation file.

EvaluationMetrics EvaluationSummary::evaluation_metrics

protected

Evaluation metrics to use.

EvaluationStatistics EvaluationSummary::evaluation_statistics

protected

Evaluation statistics to use.

boost::filesystem::path EvaluationSummary::gt_directory

protected

Directory of ground truth segmentations.

boost::filesystem::path EvaluationSummary::img_directory

protected

Directory of images.

boost::filesystem::path EvaluationSummary::results_file

protected

Path to results file.

boost::filesystem::path EvaluationSummary::sp_directory

protected

Directory of superpixel segmentations.

boost::filesystem::path EvaluationSummary::summary_file

protected

Path to summary file.

SuperpixelVisualizations EvaluationSummary::superpixel_visualizations

protected

Visualizations to compute.

boost::filesystem::path EvaluationSummary::vis_directory

protected

Path to visualization directory.

The documentation for this class was generated from the following files:

lib_eval/evaluation_summary.h
lib_eval/evaluation_summary.cpp

Classes

Public Member Functions

Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation