diff --git a/cscore/src/main/native/cpp/Frame.cpp b/cscore/src/main/native/cpp/Frame.cpp index 98b458a8e1..eb8a5c0fd0 100644 --- a/cscore/src/main/native/cpp/Frame.cpp +++ b/cscore/src/main/native/cpp/Frame.cpp @@ -7,6 +7,8 @@ #include "Frame.h" +#include + #include #include #include @@ -54,7 +56,8 @@ Image* Frame::GetNearestImage(int width, int height) const { } Image* Frame::GetNearestImage(int width, int height, - VideoMode::PixelFormat pixelFormat) const { + VideoMode::PixelFormat pixelFormat, + int jpegQuality) const { if (!m_impl) return nullptr; std::lock_guard lock(m_impl->mutex); Image* found = nullptr; @@ -67,9 +70,10 @@ Image* Frame::GetNearestImage(int width, int height, // image processing to come, so it's worth spending a little extra time // looking for the most efficient conversion. - // 1) Same width, height, and pixelFormat (e.g. exactly what we want) + // 1) Same width, height, pixelFormat, and (possibly) JPEG quality + // (e.g. exactly what we want) for (auto i : m_impl->images) { - if (i->Is(width, height, pixelFormat)) return i; + if (i->Is(width, height, pixelFormat, jpegQuality)) return i; } // 2) Same width, height, different (but non-JPEG) pixelFormat (color conv) @@ -120,10 +124,18 @@ Image* Frame::GetNearestImage(int width, int height, } if (found) return found; - // 5) Same width, height, JPEG pixelFormat (decompression) + // 5) Same width, height, JPEG pixelFormat (decompression). As there may be + // multiple JPEG images, find the highest quality one. for (auto i : m_impl->images) { - if (i->Is(width, height, VideoMode::kMJPEG)) return i; + if (i->Is(width, height, VideoMode::kMJPEG) && + (!found || i->jpegQuality > found->jpegQuality)) { + found = i; + // consider one without a quality setting to be the highest quality + // (e.g. directly from the camera) + if (i->jpegQuality == -1) break; + } } + if (found) return found; // 6) Different width, height, JPEG pixelFormat (decompression) // 6a) Smallest image at least width/height in size @@ -146,9 +158,11 @@ Image* Frame::GetNearestImage(int width, int height, return m_impl->images.empty() ? nullptr : m_impl->images[0]; } -Image* Frame::Convert(Image* image, VideoMode::PixelFormat pixelFormat, - int jpegQuality) { - if (!image || image->pixelFormat == pixelFormat) return image; +Image* Frame::ConvertImpl(Image* image, VideoMode::PixelFormat pixelFormat, + int requiredJpegQuality, int defaultJpegQuality) { + if (!image || + image->Is(image->width, image->height, pixelFormat, requiredJpegQuality)) + return image; Image* cur = image; // If the source image is a JPEG, we need to decode it before we can do @@ -160,7 +174,7 @@ Image* Frame::Convert(Image* image, VideoMode::PixelFormat pixelFormat, if (pixelFormat == VideoMode::kBGR) return cur; } - // Color convert; if ultimate destination is JPEG, we need to convert to BGR + // Color convert switch (pixelFormat) { case VideoMode::kRGB565: // If source is YUYV or Gray, need to convert to BGR first @@ -208,7 +222,7 @@ Image* Frame::Convert(Image* image, VideoMode::PixelFormat pixelFormat, if (pixelFormat == VideoMode::kBGR) return ConvertGrayToBGR(cur); else - return ConvertGrayToMJPEG(cur, jpegQuality); + return ConvertGrayToMJPEG(cur, defaultJpegQuality); } break; case VideoMode::kYUYV: @@ -218,7 +232,7 @@ Image* Frame::Convert(Image* image, VideoMode::PixelFormat pixelFormat, // Compress if destination is JPEG if (pixelFormat == VideoMode::kMJPEG) - cur = ConvertBGRToMJPEG(cur, jpegQuality); + cur = ConvertBGRToMJPEG(cur, defaultJpegQuality); return cur; } @@ -427,12 +441,14 @@ Image* Frame::ConvertGrayToMJPEG(Image* image, int quality) { return rv; } -Image* Frame::GetImage(int width, int height, - VideoMode::PixelFormat pixelFormat, int jpegQuality) { +Image* Frame::GetImageImpl(int width, int height, + VideoMode::PixelFormat pixelFormat, + int requiredJpegQuality, int defaultJpegQuality) { if (!m_impl) return nullptr; std::lock_guard lock(m_impl->mutex); - Image* cur = GetNearestImage(width, height, pixelFormat); - if (!cur || cur->Is(width, height, pixelFormat)) return cur; + Image* cur = GetNearestImage(width, height, pixelFormat, requiredJpegQuality); + if (!cur || cur->Is(width, height, pixelFormat, requiredJpegQuality)) + return cur; DEBUG4("converting image from " << cur->width << "x" << cur->height << " type " << cur->pixelFormat @@ -440,8 +456,8 @@ Image* Frame::GetImage(int width, int height, // If the source image is a JPEG, we need to decode it before we can do // anything else with it. Note that if the destination format is JPEG, we - // still need to do this (unless the width/height were the same, in which - // case we already returned the existing JPEG above). + // still need to do this (unless the width/height/compression were the same, + // in which case we already returned the existing JPEG above). if (cur->pixelFormat == VideoMode::kMJPEG) cur = ConvertMJPEGToBGR(cur); // Resize @@ -461,7 +477,7 @@ Image* Frame::GetImage(int width, int height, } // Convert to output format - return Convert(cur, pixelFormat, jpegQuality); + return ConvertImpl(cur, pixelFormat, requiredJpegQuality, defaultJpegQuality); } bool Frame::GetCv(cv::Mat& image, int width, int height) { diff --git a/cscore/src/main/native/cpp/Frame.h b/cscore/src/main/native/cpp/Frame.h index 9d447b8ac8..ed4e67dac4 100644 --- a/cscore/src/main/native/cpp/Frame.h +++ b/cscore/src/main/native/cpp/Frame.h @@ -98,6 +98,13 @@ class Frame { return m_impl->images[0]->pixelFormat; } + int GetOriginalJpegQuality() const { + if (!m_impl) return 0; + std::lock_guard lock(m_impl->mutex); + if (m_impl->images.empty()) return 0; + return m_impl->images[0]->jpegQuality; + } + Image* GetExistingImage(size_t i = 0) const { if (!m_impl) return nullptr; std::lock_guard lock(m_impl->mutex); @@ -124,12 +131,31 @@ class Frame { return nullptr; } + Image* GetExistingImage(int width, int height, + VideoMode::PixelFormat pixelFormat, + int jpegQuality) const { + if (!m_impl) return nullptr; + std::lock_guard lock(m_impl->mutex); + for (auto i : m_impl->images) { + if (i->Is(width, height, pixelFormat, jpegQuality)) return i; + } + return nullptr; + } + Image* GetNearestImage(int width, int height) const; Image* GetNearestImage(int width, int height, - VideoMode::PixelFormat pixelFormat) const; + VideoMode::PixelFormat pixelFormat, + int jpegQuality = -1) const; - Image* Convert(Image* image, VideoMode::PixelFormat pixelFormat, - int jpegQuality = 80); + Image* Convert(Image* image, VideoMode::PixelFormat pixelFormat) { + if (pixelFormat == VideoMode::kMJPEG) return nullptr; + return ConvertImpl(image, pixelFormat, -1, 80); + } + Image* ConvertToMJPEG(Image* image, int requiredQuality, + int defaultQuality = 80) { + return ConvertImpl(image, VideoMode::kMJPEG, requiredQuality, + defaultQuality); + } Image* ConvertMJPEGToBGR(Image* image); Image* ConvertMJPEGToGray(Image* image); Image* ConvertYUYVToBGR(Image* image); @@ -140,8 +166,15 @@ class Frame { Image* ConvertBGRToMJPEG(Image* image, int quality); Image* ConvertGrayToMJPEG(Image* image, int quality); - Image* GetImage(int width, int height, VideoMode::PixelFormat pixelFormat, - int jpegQuality = 80); + Image* GetImage(int width, int height, VideoMode::PixelFormat pixelFormat) { + if (pixelFormat == VideoMode::kMJPEG) return nullptr; + return GetImageImpl(width, height, pixelFormat, -1, 80); + } + Image* GetImageMJPEG(int width, int height, int requiredQuality, + int defaultQuality = 80) { + return GetImageImpl(width, height, VideoMode::kMJPEG, requiredQuality, + defaultQuality); + } bool GetCv(cv::Mat& image) { return GetCv(image, GetOriginalWidth(), GetOriginalHeight()); @@ -149,6 +182,10 @@ class Frame { bool GetCv(cv::Mat& image, int width, int height); private: + Image* ConvertImpl(Image* image, VideoMode::PixelFormat pixelFormat, + int requiredJpegQuality, int defaultJpegQuality); + Image* GetImageImpl(int width, int height, VideoMode::PixelFormat pixelFormat, + int requiredJpegQuality, int defaultJpegQuality); void DecRef() { if (m_impl && --(m_impl->refcount) == 0) ReleaseFrame(); } diff --git a/cscore/src/main/native/cpp/Image.h b/cscore/src/main/native/cpp/Image.h index 604d05cf8c..6305d93061 100644 --- a/cscore/src/main/native/cpp/Image.h +++ b/cscore/src/main/native/cpp/Image.h @@ -79,6 +79,14 @@ class Image { bool Is(int width_, int height_, VideoMode::PixelFormat pixelFormat_) { return width == width_ && height == height_ && pixelFormat == pixelFormat_; } + bool Is(int width_, int height_, VideoMode::PixelFormat pixelFormat_, + int jpegQuality_) { + // Consider +/-5 on JPEG quality to be "close enough" + return width == width_ && height == height_ && + pixelFormat == pixelFormat_ && + (pixelFormat != VideoMode::kMJPEG || jpegQuality_ == -1 || + (jpegQuality != -1 && std::abs(jpegQuality - jpegQuality_) <= 5)); + } bool IsLarger(int width_, int height_) { return width >= width_ && height >= height_; } @@ -95,6 +103,7 @@ class Image { VideoMode::PixelFormat pixelFormat{VideoMode::kUnknown}; int width{0}; int height{0}; + int jpegQuality{-1}; }; } // namespace cs diff --git a/cscore/src/main/native/cpp/MjpegServerImpl.cpp b/cscore/src/main/native/cpp/MjpegServerImpl.cpp index cf0278db61..6b8230b983 100644 --- a/cscore/src/main/native/cpp/MjpegServerImpl.cpp +++ b/cscore/src/main/native/cpp/MjpegServerImpl.cpp @@ -114,7 +114,7 @@ class MjpegServerImpl::ConnThread : public wpi::SafeThread { int m_width{0}; int m_height{0}; - int m_compression{80}; + int m_compression{-1}; int m_fps{0}; }; @@ -635,8 +635,8 @@ void MjpegServerImpl::ConnThread::SendStream(wpi::raw_socket_ostream& os) { int width = m_width != 0 ? m_width : frame.GetOriginalWidth(); int height = m_height != 0 ? m_height : frame.GetOriginalHeight(); - Image* image = - frame.GetImage(width, height, VideoMode::kMJPEG, m_compression); + Image* image = frame.GetImageMJPEG( + width, height, m_compression, m_compression == -1 ? 80 : m_compression); if (!image) { // Shouldn't happen, but just in case... std::this_thread::sleep_for(std::chrono::milliseconds(20)); @@ -695,7 +695,7 @@ void MjpegServerImpl::ConnThread::ProcessRequest() { // Reset per-request settings m_width = 0; m_height = 0; - m_compression = 80; + m_compression = -1; m_fps = 0; // Read the request string from the stream