Skip to content

serialization for mcc #3906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: 4.x
Choose a base branch
from
Open

serialization for mcc #3906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions modules/mcc/include/opencv2/mcc/ccm.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,8 @@ Produce a ColorCorrectionModel instance for inference
class CV_EXPORTS_W ColorCorrectionModel
{
public:
CV_WRAP ColorCorrectionModel();

/** @brief Color Correction Model

Supported list of color cards:
Expand Down Expand Up @@ -507,11 +509,17 @@ class CV_EXPORTS_W ColorCorrectionModel
*/
CV_WRAP Mat infer(const Mat& img, bool islinear = false);

CV_WRAP void write(cv::FileStorage& fs) const;
CV_WRAP void read(const cv::FileNode& node);

class Impl;
private:
std::shared_ptr<Impl> p;
};

CV_EXPORTS void write(cv::FileStorage& fs, const std::string&, const ColorCorrectionModel& ccm);
CV_EXPORTS void read(const cv::FileNode& node, ColorCorrectionModel& ccm, const ColorCorrectionModel& default_value = ColorCorrectionModel());

//! @} ccm
} // namespace ccm
} // namespace cv
Expand Down
158 changes: 158 additions & 0 deletions modules/mcc/src/ccm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,28 @@

#include "opencv2/mcc/ccm.hpp"
#include "linearize.hpp"

namespace cv {
namespace ccm {
class ColorCorrectionModel::Impl
{
public:
Mat src;

std::shared_ptr<Color> dst = std::make_shared<Color>();
// Track initialization parameters for serialization
Mat dst_colors;
COLOR_SPACE dst_cs_enum;
Mat dst_colored;
CONST_COLOR dst_constcolor;
bool dst_use_constcolor;

Mat dist;

RGBBase_& cs;
// Track initialization parameters for serialization
COLOR_SPACE cs_enum;

Mat mask;

// RGBl of detected data and the reference
Expand Down Expand Up @@ -138,6 +151,7 @@ class ColorCorrectionModel::Impl

ColorCorrectionModel::Impl::Impl()
: cs(*GetCS::getInstance().get_rgb(COLOR_SPACE_sRGB))
, cs_enum(COLOR_SPACE_sRGB)
, ccm_type(CCM_3x3)
, distance(DISTANCE_CIE2000)
, linear_type(LINEARIZATION_GAMMA)
Expand Down Expand Up @@ -282,6 +296,10 @@ void ColorCorrectionModel::Impl::fitting(void)
loss = pow((res / masked_len), 0.5);
}

ColorCorrectionModel::ColorCorrectionModel()
: p(std::make_shared<Impl>())
{}

Mat ColorCorrectionModel::infer(const Mat& img, bool islinear)
{
if (!p->ccm.data)
Expand All @@ -300,14 +318,24 @@ Mat ColorCorrectionModel::infer(const Mat& img, bool islinear)

void ColorCorrectionModel::Impl::getColor(CONST_COLOR constcolor)
{
dst_use_constcolor = true;
dst_constcolor = constcolor;
dst = (GetColor::getColor(constcolor));
}
void ColorCorrectionModel::Impl::getColor(Mat colors_, COLOR_SPACE ref_cs_)
{
dst_use_constcolor = false;
dst_colors = colors_;
dst_cs_enum = ref_cs_;
dst_colored = Mat();
dst.reset(new Color(colors_, *GetCS::getInstance().get_cs(ref_cs_)));
}
void ColorCorrectionModel::Impl::getColor(Mat colors_, COLOR_SPACE cs_, Mat colored_)
{
dst_use_constcolor = false;
dst_colors = colors_;
dst_cs_enum = cs_;
dst_colored = colored_;
dst.reset(new Color(colors_, *GetCS::getInstance().get_cs(cs_), colored_));
}
ColorCorrectionModel::ColorCorrectionModel(const Mat& src_, CONST_COLOR constcolor)
Expand All @@ -331,6 +359,7 @@ ColorCorrectionModel::ColorCorrectionModel(const Mat& src_, Mat colors_, COLOR_S

void ColorCorrectionModel::setColorSpace(COLOR_SPACE cs_)
{
p->cs_enum = cs_;
p->cs = *GetCS::getInstance().get_rgb(cs_);
}
void ColorCorrectionModel::setCCM_TYPE(CCM_TYPE ccm_type_)
Expand Down Expand Up @@ -433,5 +462,134 @@ Mat ColorCorrectionModel::getMask() const{
Mat ColorCorrectionModel::getWeights() const{
return p->weights;
}

void ColorCorrectionModel::write(FileStorage& fs) const
{
fs << "{"
<< "ccm" << p->ccm
<< "loss" << p->loss
<< "src" << p->src
<< "dist" << p->dist
<< "cs_enum" << p->cs_enum
<< "src_rgbl" << p->src_rgbl
<< "dst_rgbl" << p->dst_rgbl
<< "mask" << p->mask
<< "ccm_type" << p->ccm_type
<< "shape" << p->shape
<< "linear" << *p->linear
<< "distance" << p->distance
<< "linear_type" << p->linear_type
<< "weights" << p->weights
<< "weights_list" << p->weights_list
<< "ccm0" << p->ccm0
<< "gamma" << p->gamma
<< "deg" << p->deg
<< "saturated_threshold" << p->saturated_threshold
<< "initial_method_type" << p->initial_method_type
<< "weights_coeff" << p->weights_coeff
<< "masked_len" << p->masked_len
<< "max_count" << p->max_count
<< "epsilon" << p->epsilon
<< "dst_use_constcolor" << p->dst_use_constcolor;

Comment on lines +466 to +494
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the contribution. The code looks good overall, but I feel we don’t need to write/save this many details.
I think keeping [ccm, loss, colorchart, ccm_type, gamma, deg, saturated-threshold, linear-type, distance-type] should be sufficient.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense. My reasoning for serializing everything was (1) reloading the model in another process should enable all interfaces on the class to function as if it were the original and (2) if state inside the class is non-essential, then should it be stored in the class in the first place? Are some of the fields only cached state for the others?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(1) For inference, only p->ccm, p->linear, p->cs, and p->shape (which can also be derived from ccm_type) are actually needed. You can see this directly in the infer() function in ccm.cpp.
(2) Most of the other fields are indeed just cached values to speed up CCM computation and aren't necessary for inference.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You don't want to support use cases like a model being saved, loaded elsewhere, and then accessors like getWeights() being called?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

CCM calculation is not really something similar to a deep learning model where we want to save all the parameters and resume training.

The application is to just save the matrix and some other metadata for informative purpose and then use it for inference in an application

@asmorkalov what are your thoughts? Your comments are welcomed

if (p->dst_use_constcolor) {
fs << "dst_constcolor" << p->dst_constcolor;
} else {
fs << "dst_colors" << p->dst_colors
<< "dst_cs_enum" << p->dst_cs_enum
<< "dst_colored" << p->dst_colored;
}
fs << "}";
}


void ColorCorrectionModel::read(const FileNode& node)
{
node["ccm"] >> p->ccm;
node["loss"] >> p->loss;
node["src"] >> p->src;
node["dist"] >> p->dist;
node["src_rgbl"] >> p->src_rgbl;
node["dst_rgbl"] >> p->dst_rgbl;
node["mask"] >> p->mask;
node["ccm_type"] >> p->ccm_type;
node["shape"] >> p->shape;
node["distance"] >> p->distance;
node["gamma"] >> p->gamma;
node["deg"] >> p->deg;
node["saturated_threshold"] >> p->saturated_threshold;
node["initial_method_type"] >> p->initial_method_type;
node["weights_coeff"] >> p->weights_coeff;
node["weights"] >> p->weights;
node["weights_list"] >> p->weights_list;
node["ccm0"] >> p->ccm0;
node["masked_len"] >> p->masked_len;
node["max_count"] >> p->max_count;
node["epsilon"] >> p->epsilon;

COLOR_SPACE cs_enum;
node["cs_enum"] >> cs_enum;
setColorSpace(cs_enum);

bool dst_use_constcolor;
node["dst_use_constcolor"] >> dst_use_constcolor;
if (dst_use_constcolor) {
CONST_COLOR dst_constcolor;
node["dst_constcolor"] >> dst_constcolor;
p->getColor(dst_constcolor);
} else {
Mat dst_colors;
node["dst_colors"] >> dst_colors;
COLOR_SPACE dst_cs_enum;
node["dst_cs_enum"] >> dst_cs_enum;
Mat dst_colored;
node["dst_colored"] >> dst_colored;
if (dst_colored.empty()) {
p->getColor(dst_colors, dst_cs_enum);
} else {
p->getColor(dst_colors, dst_cs_enum, dst_colored);
}
}

node["linear_type"] >> p->linear_type;
switch (p->linear_type) {
case cv::ccm::LINEARIZATION_GAMMA:
p->linear = std::shared_ptr<Linear>(new LinearGamma());
break;
case cv::ccm::LINEARIZATION_COLORPOLYFIT:
p->linear = std::shared_ptr<Linear>(new LinearColor<Polyfit>());
break;
case cv::ccm::LINEARIZATION_IDENTITY:
p->linear = std::shared_ptr<Linear>(new LinearIdentity());
break;
case cv::ccm::LINEARIZATION_COLORLOGPOLYFIT:
p->linear = std::shared_ptr<Linear>(new LinearColor<LogPolyfit>());
break;
case cv::ccm::LINEARIZATION_GRAYPOLYFIT:
p->linear = std::shared_ptr<Linear>(new LinearGray<Polyfit>());
break;
case cv::ccm::LINEARIZATION_GRAYLOGPOLYFIT:
p->linear = std::shared_ptr<Linear>(new LinearGray<LogPolyfit>());
break;
default:
CV_Error(Error::StsBadArg, "Wrong linear_type!");
break;
}
node["linear"] >> *p->linear;
}

void write(FileStorage& fs, const std::string&, const cv::ccm::ColorCorrectionModel& ccm)
{
ccm.write(fs);
}

void read(const cv::FileNode& node, cv::ccm::ColorCorrectionModel& ccm, const cv::ccm::ColorCorrectionModel& default_value)
{
if (node.empty())
ccm = default_value;
else
ccm.read(node);
}

}
} // namespace cv::ccm
Loading
Loading