From 841902e02c6bbb4c95bee722ef511d90f993bb64 Mon Sep 17 00:00:00 2001 From: Christian Feldmann Date: Thu, 11 Aug 2022 18:35:28 +0200 Subject: [PATCH 01/11] Refactoring --- .../playlistitem/playlistItemDifference.cpp | 95 +++++++++---------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/YUViewLib/src/playlistitem/playlistItemDifference.cpp b/YUViewLib/src/playlistitem/playlistItemDifference.cpp index 18e95d750..c51f9c188 100644 --- a/YUViewLib/src/playlistitem/playlistItemDifference.cpp +++ b/YUViewLib/src/playlistitem/playlistItemDifference.cpp @@ -44,15 +44,15 @@ #define DEBUG_DIFF(fmt, ...) ((void)0) #endif -#define DIFFERENCE_INFO_TEXT \ - "Please drop two video item's onto this difference item to calculate the difference." +constexpr auto DIFFERENCE_INFO_TEXT = + "Please drop two video item's onto this difference item to calculate the difference."; playlistItemDifference::playlistItemDifference() : playlistItemContainer("Difference Item") { - setIcon(0, functionsGui::convertIcon(":img_difference.png")); + this->setIcon(0, functionsGui::convertIcon(":img_difference.png")); // Enable dropping for difference objects. The user can drop the two items to calculate the // difference from. - setFlags(flags() | Qt::ItemIsDropEnabled); + this->setFlags(this->flags() | Qt::ItemIsDropEnabled); this->prop.propertiesWidgetTitle = "Difference Properties"; @@ -61,10 +61,10 @@ playlistItemDifference::playlistItemDifference() : playlistItemContainer("Differ this->frameLimitsMax = false; this->infoText = DIFFERENCE_INFO_TEXT; - connect(&difference, - &video::videoHandlerDifference::signalHandlerChanged, - this, - &playlistItemDifference::SignalItemChanged); + this->connect(&this->difference, + &video::videoHandlerDifference::signalHandlerChanged, + this, + &playlistItemDifference::SignalItemChanged); } /* For a difference item, the info list is just a list of the names of the @@ -74,20 +74,18 @@ InfoData playlistItemDifference::getInfo() const { InfoData info("Difference Info"); - if (childCount() >= 1) - info.items.append(InfoItem(QString("File 1"), getChildPlaylistItem(0)->properties().name)); - if (childCount() >= 2) - info.items.append(InfoItem(QString("File 2"), getChildPlaylistItem(1)->properties().name)); + if (this->childCount() >= 1) + info.items.append( + InfoItem(QString("File 1"), this->getChildPlaylistItem(0)->properties().name)); + if (this->childCount() >= 2) + info.items.append( + InfoItem(QString("File 2"), this->getChildPlaylistItem(1)->properties().name)); // Report the position of the first difference in coding order - difference.reportFirstDifferencePosition(info.items); + this->difference.reportFirstDifferencePosition(info.items); - // Report MSE - for (int i = 0; i < difference.differenceInfoList.length(); i++) - { - InfoItem p = difference.differenceInfoList[i]; - info.items.append(p); - } + for (const auto &item : this->difference.differenceInfoList) + info.items.append(item); return info; } @@ -99,35 +97,35 @@ void playlistItemDifference::drawItem(QPainter *painter, { DEBUG_DIFF("playlistItemDifference::drawItem frameIdx %d %s", frameIdx, - childLlistUpdateRequired ? "childLlistUpdateRequired" : ""); - if (childLlistUpdateRequired) + this->childLlistUpdateRequired ? "childLlistUpdateRequired" : ""); + if (this->childLlistUpdateRequired) { // Update the 'childList' and connect the signals/slots - updateChildList(); + this->updateChildList(); // Update the items in the difference item video::FrameHandler *childVideo0 = nullptr; video::FrameHandler *childVideo1 = nullptr; if (childCount() >= 1) - childVideo0 = getChildPlaylistItem(0)->getFrameHandler(); + childVideo0 = this->getChildPlaylistItem(0)->getFrameHandler(); if (childCount() >= 2) - childVideo1 = getChildPlaylistItem(1)->getFrameHandler(); + childVideo1 = this->getChildPlaylistItem(1)->getFrameHandler(); - difference.setInputVideos(childVideo0, childVideo1); + this->difference.setInputVideos(childVideo0, childVideo1); - if (childCount() > 2) - infoText = "More than two items are not supported.\n" DIFFERENCE_INFO_TEXT; + if (this->childCount() > 2) + infoText = "More than two items are not supported.\n" + QString(DIFFERENCE_INFO_TEXT); else infoText = DIFFERENCE_INFO_TEXT; } - if (childCount() != 2 || !difference.inputsValid()) + if (this->childCount() != 2 || !this->difference.inputsValid()) // Draw the emptyText playlistItem::drawItem(painter, -1, zoomFactor, drawRawData); else { // draw the videoHandler - difference.drawDifferenceFrame(painter, frameIdx, zoomFactor, drawRawData); + this->difference.drawDifferenceFrame(painter, frameIdx, zoomFactor, drawRawData); } } @@ -138,13 +136,11 @@ ItemLoadingState playlistItemDifference::needsLoading(int frameIdx, bool loadRaw QSize playlistItemDifference::getSize() const { - if (!difference.inputsValid()) - { + if (!this->difference.inputsValid()) // Return the size of the empty text. return playlistItemContainer::getSize(); - } - auto s = difference.getFrameSize(); + auto s = this->difference.getFrameSize(); return QSize(s.width, s.height); } @@ -152,12 +148,12 @@ void playlistItemDifference::createPropertiesWidget() { Q_ASSERT_X(!this->propertiesWidget, "createPropertiesWidget", "Properties widget already exists"); - preparePropertiesWidget(QStringLiteral("playlistItemDifference")); + this->preparePropertiesWidget(QStringLiteral("playlistItemDifference")); // On the top level everything is layout vertically - QVBoxLayout *vAllLaout = new QVBoxLayout(propertiesWidget.data()); + auto vAllLaout = new QVBoxLayout(this->propertiesWidget.data()); - QFrame *line = new QFrame; + auto line = new QFrame; line->setObjectName(QStringLiteral("line")); line->setFrameShape(QFrame::HLine); line->setFrameShadow(QFrame::Sunken); @@ -187,7 +183,7 @@ void playlistItemDifference::savePlaylist(QDomElement &root, const QDir &playlis playlistItemDifference * playlistItemDifference::newPlaylistItemDifference(const YUViewDomElement &root) { - playlistItemDifference *newDiff = new playlistItemDifference(); + auto newDiff = new playlistItemDifference(); // Load properties from the parent classes newDiff->difference.loadPlaylist(root); @@ -211,7 +207,7 @@ ValuePairListSets playlistItemDifference::getPixelValues(const QPoint &pixelPos, { newSet.append("Item B", getChildPlaylistItem(1)->getFrameHandler()->getPixelValues(pixelPos, frameIdx)); - newSet.append("Diff (A-B)", difference.getPixelValues(pixelPos, frameIdx, nullptr)); + newSet.append("Diff (A-B)", this->difference.getPixelValues(pixelPos, frameIdx, nullptr)); } return newSet; @@ -222,18 +218,18 @@ void playlistItemDifference::loadFrame(int frameIdx, bool loadRawData, bool emitSignals) { - if (childCount() != 2 || !difference.inputsValid()) + if (this->childCount() != 2 || !this->difference.inputsValid()) return; - auto state = difference.needsLoading(frameIdx, loadRawData); + const auto state = this->difference.needsLoading(frameIdx, loadRawData); if (state == ItemLoadingState::LoadingNeeded) { // Load the requested current frame DEBUG_DIFF("playlistItemDifference::loadFrame loading difference for frame %d", frameIdx); - isDifferenceLoading = true; + this->isDifferenceLoading = true; // Since every playlist item can have it's own relative indexing, we need two frame indices - difference.loadFrameDifference(frameIdx); - isDifferenceLoading = false; + this->difference.loadFrameDifference(frameIdx); + this->isDifferenceLoading = false; if (emitSignals) emit SignalItemChanged(true, RECACHE_NONE); } @@ -248,16 +244,19 @@ void playlistItemDifference::loadFrame(int frameIdx, DEBUG_DIFF("playlistItemDifference::loadFrame loading difference into double buffer %d %s", nextFrameIdx, playing ? "(playing)" : ""); - isDifferenceLoadingToDoubleBuffer = true; - difference.loadFrameDifference(frameIdx, true); - isDifferenceLoadingToDoubleBuffer = false; + this->isDifferenceLoadingToDoubleBuffer = true; + this->difference.loadFrameDifference(frameIdx, true); + this->isDifferenceLoadingToDoubleBuffer = false; if (emitSignals) emit signalItemDoubleBufferLoaded(); } } } -bool playlistItemDifference::isLoading() const { return this->isDifferenceLoading; } +bool playlistItemDifference::isLoading() const +{ + return this->isDifferenceLoading; +} bool playlistItemDifference::isLoadingDoubleBuffer() const { @@ -268,6 +267,6 @@ void playlistItemDifference::childChanged(bool redraw, recacheIndicator recache) { // One of the child items changed and needs to redraw. This means that the difference is out of // date and has to be recalculated. - difference.invalidateAllBuffers(); + this->difference.invalidateAllBuffers(); playlistItemContainer::childChanged(redraw, recache); } \ No newline at end of file From 5aa96176eb104cc4be7c26367c1b2bf60df98150 Mon Sep 17 00:00:00 2001 From: Christian Feldmann Date: Thu, 11 Aug 2022 23:38:31 +0200 Subject: [PATCH 02/11] A bit refactoing of the videoDiff --- .../src/video/videoHandlerDifference.cpp | 538 +++++++++--------- YUViewLib/src/video/videoHandlerDifference.h | 17 - 2 files changed, 263 insertions(+), 292 deletions(-) diff --git a/YUViewLib/src/video/videoHandlerDifference.cpp b/YUViewLib/src/video/videoHandlerDifference.cpp index 3e618ef76..35ca5c4e8 100644 --- a/YUViewLib/src/video/videoHandlerDifference.cpp +++ b/YUViewLib/src/video/videoHandlerDifference.cpp @@ -49,6 +49,198 @@ namespace video #define DEBUG_VIDEO(fmt, ...) ((void)0) #endif +namespace +{ + +struct PositionAndPartIndex +{ + PositionAndPartIndex(Position position, int partIndex) + { + this->position = position; + this->partIndex = partIndex; + }; + Position position{}; + int partIndex; +}; + +std::optional findFirstDifferenceInHEVCBlock(Position pos, + int blockSize, + const QImage &diffImg, + bool markDifference) +{ + const auto frameSize = diffImg.size(); + if (pos.x >= int(frameSize.width()) || pos.y >= int(frameSize.height())) + return {}; + + if (blockSize == 4) + { + // Check for a difference + for (auto subX = pos.x; subX < pos.x + 4; subX++) + { + for (auto subY = pos.y; subY < pos.y + 4; subY++) + { + auto rgb = diffImg.pixel(QPoint(subX, subY)); + + if (markDifference) + { + // Black means no difference + if (qRed(rgb) != 0 || qGreen(rgb) != 0 || qBlue(rgb) != 0) + return PositionAndPartIndex(pos, 0); + } + else + { + // TODO: Double check if this is always true + // Do other values also convert to RGB(130,130,130) ? + // What about 10 bit input material + auto red = qRed(rgb); + auto green = qGreen(rgb); + auto blue = qBlue(rgb); + + if (red != 130 || green != 130 || blue != 130) + return PositionAndPartIndex(pos, 0); + } + } + } + } + else + { + // Walk further into the hierarchy + const int bs = blockSize / 2; + const int nr4x4BlocksPerPart = (bs / 4) * (bs / 4); + if (auto diff = findFirstDifferenceInHEVCBlock(pos, bs, diffImg, markDifference)) + return diff; + if (auto diff = findFirstDifferenceInHEVCBlock( + Position(pos.x + bs, pos.y), bs, diffImg, markDifference)) + return PositionAndPartIndex(diff->position, diff->partIndex + nr4x4BlocksPerPart); + if (auto diff = findFirstDifferenceInHEVCBlock( + Position(pos.x, pos.y + bs), bs, diffImg, markDifference)) + return PositionAndPartIndex(diff->position, diff->partIndex + nr4x4BlocksPerPart * 2); + if (auto diff = findFirstDifferenceInHEVCBlock( + Position(pos.x + bs, pos.y + bs), bs, diffImg, markDifference)) + return PositionAndPartIndex(diff->position, diff->partIndex + nr4x4BlocksPerPart * 3); + } + return {}; +} + +inline int +getValueFromSource(const unsigned char *src, const int idx, const int bps, const bool bigEndian) +{ + if (bps > 8) + // Read two bytes in the right order + return (bigEndian) ? src[idx * 2] << 8 | src[idx * 2 + 1] + : src[idx * 2] | src[idx * 2 + 1] << 8; + else + // Just read one byte + return src[idx]; +} + +std::optional +findFirstDifferenceInHEVCBlockYUV(Position pos, + int blockSize, + const QByteArray & diffYUV, + const Size & frameSize, + const yuv::PixelFormatYUV &diffYUVFormat) +{ + if (pos.x >= int(frameSize.width) || pos.y >= int(frameSize.height)) + return {}; + + // The items can be of different size (we then calculate the difference of the top left aligned + // part) + const auto w_in = frameSize.width; + const auto h_in = frameSize.height; + + // Get subsampling modes (they are identical for both inputs and the output) + const auto subH = diffYUVFormat.getSubsamplingHor(); + const auto subV = diffYUVFormat.getSubsamplingVer(); + + const bool bigEndian = diffYUVFormat.isBigEndian(); + const auto bps_in = diffYUVFormat.getBitsPerSample(); + const auto diffZero = 128 << (bps_in - 8); + + // Get pointers to the inputs + const auto componentSizeLuma_In = w_in * h_in; + const auto componentSizeChroma_In = (w_in / subH) * (h_in / subV); + const auto nrBytesLumaPlane_In = bps_in > 8 ? 2 * componentSizeLuma_In : componentSizeLuma_In; + const auto nrBytesChromaPlane_In = + bps_in > 8 ? 2 * componentSizeChroma_In : componentSizeChroma_In; + + // Calculate Luma sample difference + const int stride_in = bps_in > 8 ? w_in * 2 : w_in; // How many bytes to the next y line? + const int strideC_in = + w_in / subH * (bps_in > 8 ? 2 : 1); // How many bytes to the next U/V y line + + if (blockSize == 4) + { + auto srcY1 = (unsigned char *)diffYUV.data(); + auto srcU1 = (diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUV || + diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUVA) + ? srcY1 + nrBytesLumaPlane_In + : srcY1 + nrBytesLumaPlane_In + nrBytesChromaPlane_In; + auto srcV1 = (diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUV || + diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUVA) + ? srcY1 + nrBytesLumaPlane_In + nrBytesChromaPlane_In + : srcY1 + nrBytesLumaPlane_In; + + // adjust source pointer according to block position + srcY1 += pos.y * stride_in; + srcU1 += pos.y / subV * strideC_in; + srcV1 += pos.y / subV * strideC_in; + + // Check for a difference + for (int subY = pos.y; subY < pos.y + 4; subY++) + { + + for (int subX = pos.x; subX < pos.x + 4; subX++) + { + + auto val1 = getValueFromSource(srcY1, subX, bps_in, bigEndian); + if (val1 != diffZero) + return PositionAndPartIndex(pos, 0); + + // is this a position at which we have a chroma sample? + if (subX % subH == 0 && subY % subV == 0 && subX * subV < w_in) + { + auto valU1 = getValueFromSource(srcU1, subX, bps_in, bigEndian); + auto valV1 = getValueFromSource(srcV1, subX, bps_in, bigEndian); + if (valU1 != diffZero || valV1 != diffZero) + return PositionAndPartIndex(pos, 0); + } + } + + // Goto the next y line + srcY1 += stride_in; + + // is this a position at which we have a chroma line? + if (subY % subV == 0) + { + // Goto the next y line + srcU1 += strideC_in; + srcV1 += strideC_in; + } + } + } + else + { + // Walk further into the hierarchy + const int bs = blockSize / 2; + const int nr4x4BlocksPerPart = (bs / 4) * (bs / 4); + if (auto diff = findFirstDifferenceInHEVCBlockYUV(pos, bs, diffYUV, frameSize, diffYUVFormat)) + return diff; + if (auto diff = findFirstDifferenceInHEVCBlockYUV( + Position(pos.x + bs, pos.y), bs, diffYUV, frameSize, diffYUVFormat)) + PositionAndPartIndex(diff->position, diff->partIndex + nr4x4BlocksPerPart); + if (auto diff = findFirstDifferenceInHEVCBlockYUV( + Position(pos.x, pos.y + bs), bs, diffYUV, frameSize, diffYUVFormat)) + PositionAndPartIndex(diff->position, diff->partIndex + nr4x4BlocksPerPart * 2); + if (auto diff = findFirstDifferenceInHEVCBlockYUV( + Position(pos.x + bs, pos.y + bs), bs, diffYUV, frameSize, diffYUVFormat)) + PositionAndPartIndex(diff->position, diff->partIndex + nr4x4BlocksPerPart * 3); + } + return {}; +} + +} // namespace + videoHandlerDifference::videoHandlerDifference() : videoHandler() { } @@ -58,28 +250,28 @@ void videoHandlerDifference::drawDifferenceFrame(QPainter *painter, double zoomFactor, bool drawRawValues) { - if (!inputsValid()) + if (!this->inputsValid()) return; // Check if the frameIdx changed and if we have to load a new frame - if (frameIdx != currentImageIndex) + if (frameIdx != this->currentImageIndex) { // The current buffer is out of date. Update it. // Check the double buffer - if (frameIdx == doubleBufferImageFrameIndex) + if (frameIdx == this->doubleBufferImageFrameIndex) { - currentImage = doubleBufferImage; - currentImageIndex = frameIdx; + this->currentImage = this->doubleBufferImage; + this->currentImageIndex = frameIdx; DEBUG_VIDEO("videoHandler::drawFrame %d loaded from double buffer", frameIdx); } else { - QMutexLocker lock(&imageCacheAccess); - if (cacheValid && imageCache.contains(frameIdx)) + QMutexLocker lock(&this->imageCacheAccess); + if (this->cacheValid && this->imageCache.contains(frameIdx)) { - currentImage = imageCache[frameIdx]; - currentImageIndex = frameIdx; + this->currentImage = this->imageCache[frameIdx]; + this->currentImageIndex = frameIdx; DEBUG_VIDEO("videoHandler::drawFrame %d loaded from cache", frameIdx); } } @@ -91,14 +283,15 @@ void videoHandlerDifference::drawDifferenceFrame(QPainter *painter, videoRect.moveCenter(QPoint(0, 0)); // Draw the current image (currentImage) - currentImageSetMutex.lock(); - painter->drawImage(videoRect, currentImage); - currentImageSetMutex.unlock(); + { + QMutexLocker lock(&this->currentImageSetMutex); + painter->drawImage(videoRect, currentImage); + } if (drawRawValues && zoomFactor >= SPLITVIEW_DRAW_VALUES_ZOOMFACTOR) { // Draw the pixel values onto the pixels - inputVideo[0]->drawPixelValues( + this->inputVideo[0]->drawPixelValues( painter, frameIdx, videoRect, zoomFactor, inputVideo[1], this->markDifference); } } @@ -106,42 +299,41 @@ void videoHandlerDifference::drawDifferenceFrame(QPainter *painter, void videoHandlerDifference::loadFrameDifference(int frameIndex, bool) { // Calculate the difference between the inputVideos - if (!inputsValid()) + if (!this->inputsValid()) return; - differenceInfoList.clear(); + this->differenceInfoList.clear(); // Check if the second item is a video and the first one is not. In that case, // make sure that the right frame is loaded for the video item. - videoHandler *video0 = dynamic_cast(inputVideo[0].data()); - videoHandler *video1 = dynamic_cast(inputVideo[1].data()); + auto video0 = dynamic_cast(this->inputVideo[0].data()); + auto video1 = dynamic_cast(this->inputVideo[1].data()); if (video0 == nullptr && video1 != nullptr && video1->getCurrentImageIndex() != frameIndex) video1->loadFrame(frameIndex); // Calculate the difference - QImage newFrame = inputVideo[0]->calculateDifference(inputVideo[1], - frameIndex, - frameIndex, - differenceInfoList, - amplificationFactor, - markDifference); + auto newFrame = this->inputVideo[0]->calculateDifference(this->inputVideo[1], + frameIndex, + frameIndex, + this->differenceInfoList, + this->amplificationFactor, + this->markDifference); if (!newFrame.isNull()) { // The new difference frame is ready - currentImageIndex = frameIndex; - currentImageSetMutex.lock(); - currentImage = newFrame; - currentImageSetMutex.unlock(); + this->currentImageIndex = frameIndex; + QMutexLocker lock(&this->currentImageSetMutex); + this->currentImage = newFrame; } } bool videoHandlerDifference::inputsValid() const { - if (inputVideo[0].isNull() || inputVideo[1].isNull()) + if (this->inputVideo[0].isNull() || this->inputVideo[1].isNull()) return false; - if (!inputVideo[0]->isFormatValid() || !inputVideo[1]->isFormatValid()) + if (!this->inputVideo[0]->isFormatValid() || !this->inputVideo[1]->isFormatValid()) return false; return true; @@ -149,22 +341,22 @@ bool videoHandlerDifference::inputsValid() const void videoHandlerDifference::setInputVideos(FrameHandler *childVideo0, FrameHandler *childVideo1) { - if (inputVideo[0] != childVideo0 || inputVideo[1] != childVideo1) + if (this->inputVideo[0] != childVideo0 || inputVideo[1] != childVideo1) { // Something changed - inputVideo[0] = childVideo0; - inputVideo[1] = childVideo1; + this->inputVideo[0] = childVideo0; + this->inputVideo[1] = childVideo1; if (inputsValid()) { // We have two valid video "children" // Get the frame size of the difference (min in x and y direction), and set it. - auto size0 = inputVideo[0]->getFrameSize(); - auto size1 = inputVideo[1]->getFrameSize(); + auto size0 = this->inputVideo[0]->getFrameSize(); + auto size1 = this->inputVideo[1]->getFrameSize(); auto diffSize = Size(std::min(size0.width, size1.width), std::min(size0.height, size1.height)); - setFrameSize(diffSize); + this->setFrameSize(diffSize); } // If something changed, we might need a redraw @@ -177,10 +369,10 @@ QStringPairList videoHandlerDifference::getPixelValues(const QPoint &pixelPos, FrameHandler *, const int frameIdx1) { - if (!inputsValid()) + if (!this->inputsValid()) return QStringPairList(); - return inputVideo[0]->getPixelValues(pixelPos, frameIdx, inputVideo[1], frameIdx1); + return this->inputVideo[0]->getPixelValues(pixelPos, frameIdx, this->inputVideo[1], frameIdx1); } void videoHandlerDifference::setFormatFromSizeAndName( @@ -221,90 +413,91 @@ QLayout *videoHandlerDifference::createDifferenceHandlerControls() void videoHandlerDifference::slotDifferenceControlChanged() { // The control that caused the slot to be called - QObject *sender = QObject::sender(); + auto sender = QObject::sender(); if (sender == ui.markDifferenceCheckBox) { - markDifference = ui.markDifferenceCheckBox->isChecked(); + this->markDifference = ui.markDifferenceCheckBox->isChecked(); // Set the current frame in the buffer to be invalid and emit the signal that something has // changed - currentImageIndex = -1; + this->currentImageIndex = -1; emit signalHandlerChanged(true, RECACHE_NONE); } else if (sender == ui.codingOrderComboBox) { - codingOrder = (CodingOrder)ui.codingOrderComboBox->currentIndex(); + this->codingOrder = (CodingOrder)ui.codingOrderComboBox->currentIndex(); // The calculation of the first difference in coding order changed but no redraw is necessary emit signalHandlerChanged(false, RECACHE_NONE); } else if (sender == ui.amplificationFactorSpinBox) { - amplificationFactor = ui.amplificationFactorSpinBox->value(); + this->amplificationFactor = ui.amplificationFactorSpinBox->value(); // Set the current frame in the buffer to be invalid and emit the signal that something has // changed - currentImageIndex = -1; + this->currentImageIndex = -1; emit signalHandlerChanged(true, RECACHE_NONE); } } void videoHandlerDifference::reportFirstDifferencePosition(QList &infoList) const { - if (!inputsValid()) + if (!this->inputsValid()) return; - if (functions::clipToUnsigned(currentImage.width()) != frameSize.width || - functions::clipToUnsigned(currentImage.height()) != frameSize.height) + if (functions::clipToUnsigned(this->currentImage.width()) != this->frameSize.width || + functions::clipToUnsigned(this->currentImage.height()) != this->frameSize.height) return; - if (codingOrder == CodingOrder::HEVC) + if (this->codingOrder == CodingOrder::HEVC) { // Assume the following: // - The picture is split into LCUs of 64x64 pixels which are scanned in raster scan // - Each LCU is scanned in a hierarchical tree until the smallest unit size (4x4 pixels) is // reached This is exactly what we are going to do here now - int widthLCU = (frameSize.width + 63) / 64; // Round up - int heightLCU = (frameSize.height + 63) / 64; + int widthLCU = (this->frameSize.width + 63) / 64; // Round up + int heightLCU = (this->frameSize.height + 63) / 64; for (int y = 0; y < heightLCU; y++) { for (int x = 0; x < widthLCU; x++) { - // Now take the tree approach - int firstX, firstY, partIndex = 0; + // Go into LCUs in a tree fashion - auto videoYUV0 = dynamic_cast(inputVideo[0].data()); - if (videoYUV0 != NULL && videoYUV0->isDiffReady()) + auto videoYUV0 = dynamic_cast(this->inputVideo[0].data()); + if (videoYUV0 != nullptr && videoYUV0->isDiffReady()) { // find first difference using YUV instead of QImage. The latter does not work for 10bit // videos and very small differences, since it only supports 8bit - if (hierarchicalPositionYUV(x * 64, - y * 64, - 64, - firstX, - firstY, - partIndex, - videoYUV0->getDiffYUV(), - videoYUV0->getDiffYUVFormat())) + if (auto posAndIndex = findFirstDifferenceInHEVCBlockYUV(Position(x * 64, y * 64), + 64, + videoYUV0->getDiffYUV(), + this->frameSize, + videoYUV0->getDiffYUVFormat())) { - // We found a difference in this block infoList.append(InfoItem("First diff LCU", QString::number(y * widthLCU + x))); - infoList.append(InfoItem("First diff X,Y", QString("%1,%2").arg(firstX).arg(firstY))); - infoList.append(InfoItem("First diff partIndex", QString::number(partIndex))); + infoList.append(InfoItem( + "First diff X,Y", + QString("%1,%2").arg(posAndIndex->position.x).arg(posAndIndex->position.y))); + infoList.append( + InfoItem("First diff partIndex", QString::number(posAndIndex->partIndex))); return; } } else { - if (hierarchicalPosition(x * 64, y * 64, 64, firstX, firstY, partIndex, currentImage)) + if (auto posAndIndex = findFirstDifferenceInHEVCBlock( + Position(x * 64, y * 64), 64, currentImage, this->markDifference)) { - // We found a difference in this block infoList.append(InfoItem("First diff LCU", QString::number(y * widthLCU + x))); - infoList.append(InfoItem("First diff X,Y", QString("%1,%2").arg(firstX).arg(firstY))); - infoList.append(InfoItem("First diff partIndex", QString::number(partIndex))); + infoList.append(InfoItem( + "First diff X,Y", + QString("%1,%2").arg(posAndIndex->position.x).arg(posAndIndex->position.y))); + infoList.append( + InfoItem("First diff partIndex", QString::number(posAndIndex->partIndex))); return; } } @@ -340,220 +533,15 @@ void videoHandlerDifference::loadPlaylist(const YUViewDomElement &element) ItemLoadingState videoHandlerDifference::needsLoadingRawValues(int frameIndex) { - if (auto video = dynamic_cast(inputVideo[0].data())) + if (auto video = dynamic_cast(this->inputVideo[0].data())) if (video->needsLoadingRawValues(frameIndex) == ItemLoadingState::LoadingNeeded) return ItemLoadingState::LoadingNeeded; - if (auto video = dynamic_cast(inputVideo[1].data())) + if (auto video = dynamic_cast(this->inputVideo[1].data())) if (video->needsLoadingRawValues(frameIndex) == ItemLoadingState::LoadingNeeded) return ItemLoadingState::LoadingNeeded; return ItemLoadingState::LoadingNotNeeded; } -bool videoHandlerDifference::hierarchicalPosition(int x, - int y, - int blockSize, - int & firstX, - int & firstY, - int & partIndex, - const QImage &diffImg) const -{ - if (x >= int(frameSize.width) || y >= int(frameSize.height)) - // This block is entirely outside of the picture - return false; - - if (blockSize == 4) - { - // Check for a difference - for (int subX = x; subX < x + 4; subX++) - { - for (int subY = y; subY < y + 4; subY++) - { - QRgb rgb = diffImg.pixel(QPoint(subX, subY)); - - if (markDifference) - { - // Black means no difference - if (qRed(rgb) != 0 || qGreen(rgb) != 0 || qBlue(rgb) != 0) - { - // First difference found - firstX = x; - firstY = y; - return true; - } - } - else - { - // TODO: Double check if this is always true - // Do other values also convert to RGB(130,130,130) ? - // What about ten bit input material - int red = qRed(rgb); - int green = qGreen(rgb); - int blue = qBlue(rgb); - - if (red != 130 || green != 130 || blue != 130) - { - // First difference found - firstX = x; - firstY = y; - return true; - } - } - } - } - - // No difference found in this block. Count the number of 4x4 blocks scanned (that is the - // partIndex) - partIndex++; - } - else - { - // Walk further into the hierarchy - const int b2 = blockSize / 2; - if (hierarchicalPosition(x, y, b2, firstX, firstY, partIndex, diffImg)) - return true; - if (hierarchicalPosition(x + b2, y, b2, firstX, firstY, partIndex, diffImg)) - return true; - if (hierarchicalPosition(x, y + b2, b2, firstX, firstY, partIndex, diffImg)) - return true; - if (hierarchicalPosition(x + b2, y + b2, b2, firstX, firstY, partIndex, diffImg)) - return true; - } - return false; -} - -inline int -getValueFromSource(const unsigned char *src, const int idx, const int bps, const bool bigEndian) -{ - if (bps > 8) - // Read two bytes in the right order - return (bigEndian) ? src[idx * 2] << 8 | src[idx * 2 + 1] - : src[idx * 2] | src[idx * 2 + 1] << 8; - else - // Just read one byte - return src[idx]; -} - -bool videoHandlerDifference::hierarchicalPositionYUV(int x, - int y, - int blockSize, - int & firstX, - int & firstY, - int & partIndex, - const QByteArray & diffYUV, - const yuv::PixelFormatYUV &diffYUVFormat) const -{ - if (x >= int(frameSize.width) || y >= int(frameSize.height)) - // This block is entirely outside of the picture - return false; - - // The items can be of different size (we then calculate the difference of the top left aligned - // part) - const int w_in = frameSize.width; - const int h_in = frameSize.height; - - // Get subsampling modes (they are identical for both inputs and the output) - const int subH = diffYUVFormat.getSubsamplingHor(); - const int subV = diffYUVFormat.getSubsamplingVer(); - - // Get the endianness of the inputs - const bool bigEndian = diffYUVFormat.isBigEndian(); - - // Get/Set the bit depth of the input - const int bps_in = diffYUVFormat.getBitsPerSample(); - const int diffZero = 128 << (bps_in - 8); - - // Get pointers to the inputs - const int componentSizeLuma_In = w_in * h_in; - const int componentSizeChroma_In = (w_in / subH) * (h_in / subV); - const int nrBytesLumaPlane_In = bps_in > 8 ? 2 * componentSizeLuma_In : componentSizeLuma_In; - const int nrBytesChromaPlane_In = - bps_in > 8 ? 2 * componentSizeChroma_In : componentSizeChroma_In; - // Current item - - // Calculate Luma sample difference - const int stride_in = bps_in > 8 ? w_in * 2 : w_in; // How many bytes to the next y line? - const int strideC_in = - w_in / subH * (bps_in > 8 ? 2 : 1); // How many bytes to the next U/V y line - - if (blockSize == 4) - { - const unsigned char *srcY1 = (unsigned char *)diffYUV.data(); - const unsigned char *srcU1 = (diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUV || - diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUVA) - ? srcY1 + nrBytesLumaPlane_In - : srcY1 + nrBytesLumaPlane_In + nrBytesChromaPlane_In; - const unsigned char *srcV1 = (diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUV || - diffYUVFormat.getPlaneOrder() == yuv::PlaneOrder::YUVA) - ? srcY1 + nrBytesLumaPlane_In + nrBytesChromaPlane_In - : srcY1 + nrBytesLumaPlane_In; - - // adjust source pointer according to block position - srcY1 += y * stride_in; - srcU1 += y / subV * strideC_in; - srcV1 += y / subV * strideC_in; - - // Check for a difference - for (int subY = y; subY < y + 4; subY++) - { - - for (int subX = x; subX < x + 4; subX++) - { - - int val1 = getValueFromSource(srcY1, subX, bps_in, bigEndian); - if (val1 != diffZero) - { - firstX = x; - firstY = y; - return true; - } - - // is this a position at which we have a chroma sample? - if (subX % subH == 0 && subY % subV == 0 && subX * subV < w_in) - { - int valU1 = getValueFromSource(srcU1, subX, bps_in, bigEndian); - int valV1 = getValueFromSource(srcV1, subX, bps_in, bigEndian); - if (valU1 != diffZero || valV1 != diffZero) - { - firstX = x * subV; - firstY = y; - return true; - } - } - } - - // Goto the next y line - srcY1 += stride_in; - - // is this a position at which we have a chroma line? - if (subY % subV == 0) - { - // Goto the next y line - srcU1 += strideC_in; - srcV1 += strideC_in; - } - } - - // No difference found in this block. Count the number of 4x4 blocks scanned (that is the - // partIndex) - partIndex++; - } - else - { - // Walk further into the hierarchy - const int b2 = blockSize / 2; - if (hierarchicalPositionYUV(x, y, b2, firstX, firstY, partIndex, diffYUV, diffYUVFormat)) - return true; - if (hierarchicalPositionYUV(x + b2, y, b2, firstX, firstY, partIndex, diffYUV, diffYUVFormat)) - return true; - if (hierarchicalPositionYUV(x, y + b2, b2, firstX, firstY, partIndex, diffYUV, diffYUVFormat)) - return true; - if (hierarchicalPositionYUV( - x + b2, y + b2, b2, firstX, firstY, partIndex, diffYUV, diffYUVFormat)) - return true; - } - return false; -} - } // namespace video diff --git a/YUViewLib/src/video/videoHandlerDifference.h b/YUViewLib/src/video/videoHandlerDifference.h index 3b259d082..9f91b8eab 100644 --- a/YUViewLib/src/video/videoHandlerDifference.h +++ b/YUViewLib/src/video/videoHandlerDifference.h @@ -106,23 +106,6 @@ private slots: // The two videos that the difference will be calculated from QPointer inputVideo[2]; - // Recursively scan the LCU - bool hierarchicalPosition(int x, - int y, - int blockSize, - int & firstX, - int & firstY, - int & partIndex, - const QImage &diffImg) const; - bool hierarchicalPositionYUV(int x, - int y, - int blockSize, - int & firstX, - int & firstY, - int & partIndex, - const QByteArray & diffYUV, - const yuv::PixelFormatYUV &diffYUVFormat) const; - SafeUi ui; }; From 0beb729c200cd6ac8e6dfa8fe02d0b55b9767ab5 Mon Sep 17 00:00:00 2001 From: Christian Feldmann Date: Fri, 12 Aug 2022 11:34:11 +0200 Subject: [PATCH 03/11] Add a Position struct --- YUViewLib/src/common/Typedef.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/YUViewLib/src/common/Typedef.h b/YUViewLib/src/common/Typedef.h index 9bbf63f78..5cf04937f 100644 --- a/YUViewLib/src/common/Typedef.h +++ b/YUViewLib/src/common/Typedef.h @@ -274,6 +274,21 @@ template struct Range } }; +struct Position +{ + Position() = default; + Position(int x, int y) + { + this->x = x; + this->y = y; + } + + int x{}; + int y{}; + + bool operator!=(const Position &other) const { return this->x != other.x || this->y != other.y; } +}; + struct Ratio { int num{}; From a5905d4556c3e8d56fdd798fd5a429616575c252 Mon Sep 17 00:00:00 2001 From: Christian Feldmann Date: Fri, 12 Aug 2022 11:34:22 +0200 Subject: [PATCH 04/11] The bugfix --- YUViewLib/src/video/videoHandlerDifference.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/YUViewLib/src/video/videoHandlerDifference.cpp b/YUViewLib/src/video/videoHandlerDifference.cpp index 35ca5c4e8..063030b1b 100644 --- a/YUViewLib/src/video/videoHandlerDifference.cpp +++ b/YUViewLib/src/video/videoHandlerDifference.cpp @@ -198,7 +198,7 @@ findFirstDifferenceInHEVCBlockYUV(Position pos, return PositionAndPartIndex(pos, 0); // is this a position at which we have a chroma sample? - if (subX % subH == 0 && subY % subV == 0 && subX * subV < w_in) + if (subX % subH == 0 && subY % subV == 0 && subX * subV < int(w_in)) { auto valU1 = getValueFromSource(srcU1, subX, bps_in, bigEndian); auto valV1 = getValueFromSource(srcV1, subX, bps_in, bigEndian); @@ -292,7 +292,7 @@ void videoHandlerDifference::drawDifferenceFrame(QPainter *painter, { // Draw the pixel values onto the pixels this->inputVideo[0]->drawPixelValues( - painter, frameIdx, videoRect, zoomFactor, inputVideo[1], this->markDifference); + painter, frameIdx, videoRect, zoomFactor, inputVideo[1], this->markDifference, frameIdx); } } @@ -370,7 +370,7 @@ QStringPairList videoHandlerDifference::getPixelValues(const QPoint &pixelPos, const int frameIdx1) { if (!this->inputsValid()) - return QStringPairList(); + return {}; return this->inputVideo[0]->getPixelValues(pixelPos, frameIdx, this->inputVideo[1], frameIdx1); } From 5572d5e0e17984c3accd0b3608e4349c873429e9 Mon Sep 17 00:00:00 2001 From: Christian Feldmann Date: Fri, 12 Aug 2022 12:26:49 +0200 Subject: [PATCH 05/11] Start adding a test --- YUViewLib/src/common/Error.h | 41 ++++++++++++ YUViewUnitTest/helper/YUVGenerator.cpp | 91 ++++++++++++++++++++++++++ YUViewUnitTest/helper/YUVGenerator.h | 45 +++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 YUViewLib/src/common/Error.h create mode 100644 YUViewUnitTest/helper/YUVGenerator.cpp create mode 100644 YUViewUnitTest/helper/YUVGenerator.h diff --git a/YUViewLib/src/common/Error.h b/YUViewLib/src/common/Error.h new file mode 100644 index 000000000..8c884a417 --- /dev/null +++ b/YUViewLib/src/common/Error.h @@ -0,0 +1,41 @@ +/* This file is part of YUView - The YUV player with advanced analytics toolset + * + * Copyright (C) 2015 Institut für Nachrichtentechnik, RWTH Aachen University, GERMANY + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations including + * the two. + * + * You must obey the GNU General Public License in all respects for all + * of the code used other than OpenSSL. If you modify file(s) with this + * exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do + * so, delete this exception statement from your version. If you delete + * this exception statement from all source files in the program, then + * also delete it here. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include + +class NotImplemented : public std::logic_error +{ +public: + NotImplemented() : std::logic_error("Function not yet implemented"){}; +}; diff --git a/YUViewUnitTest/helper/YUVGenerator.cpp b/YUViewUnitTest/helper/YUVGenerator.cpp new file mode 100644 index 000000000..b706fe818 --- /dev/null +++ b/YUViewUnitTest/helper/YUVGenerator.cpp @@ -0,0 +1,91 @@ +/* This file is part of YUView - The YUV player with advanced analytics toolset + * + * Copyright (C) 2015 Institut für Nachrichtentechnik, RWTH Aachen University, GERMANY + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations including + * the two. + * + * You must obey the GNU General Public License in all respects for all + * of the code used other than OpenSSL. If you modify file(s) with this + * exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do + * so, delete this exception statement from your version. If you delete + * this exception statement from all source files in the program, then + * also delete it here. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "YUVGenerator.h" + +#include + +namespace helper +{ + +namespace +{ + +template QByteArray generatePlane(Size planeSize) +{ + typedef typename std::conditional::type InValueType; + static_assert(bitDepth > 0 && bitDepth <= 16); + constexpr auto nrBytesPerSample = (bitDepth + 7) / 8; + constexpr auto maxValue = (1 << bitDepth) - 1; + + const auto nrSamples = planeSize.width * planeSize.height * nrBytesPerSample; + const auto xPlusYMax = planeSize.width + planeSize.height; + + QByteArray data; + data.resize(nrSamples); + const auto *rawPtr = InValueType(data.data()); + + for (unsigned x = 0; x < planeSize.width; x++) + { + for (unsigned y = 0; y < planeSize.height; y++) + { + const auto val = (x + y) * maxValue / xPlusYMax; + const auto idx = y * planeSize.width + x; + rawPtr[idx] = InValueType(val); + } + } + return data; +} + +} // namespace + +QByteArray generateYUVVideo(Size frameSize, int nrFrames, video::yuv::PixelFormatYUV pixelFormat) +{ + if (pixelFormat.isValid()) + return {}; + if (!pixelFormat.isPlanar() || pixelFormat.isUVInterleaved()) + throw NotImplemented(); + + QByteArray data; + for (int frameNr = 0; frameNr < nrFrames; frameNr++) + { + data += generatePlane(frameSize, pixelFormat.getBitsPerSample()); + + auto chromaSize = Size(frameSize.width / pixelFormat.getSubsamplingHor(), + frameSize.height / pixelFormat.getSubsamplingVer()); + for (int planeIdx = 1; planeIdx < pixelFormat.getNrPlanes(); planeIdx++) + data += generatePlane(chromaSize, pixelFormat.getBitsPerSample()); + } + return data; +} + +} // namespace helper diff --git a/YUViewUnitTest/helper/YUVGenerator.h b/YUViewUnitTest/helper/YUVGenerator.h new file mode 100644 index 000000000..efb32cbf9 --- /dev/null +++ b/YUViewUnitTest/helper/YUVGenerator.h @@ -0,0 +1,45 @@ +/* This file is part of YUView - The YUV player with advanced analytics toolset + * + * Copyright (C) 2015 Institut für Nachrichtentechnik, RWTH Aachen University, GERMANY + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations including + * the two. + * + * You must obey the GNU General Public License in all respects for all + * of the code used other than OpenSSL. If you modify file(s) with this + * exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do + * so, delete this exception statement from your version. If you delete + * this exception statement from all source files in the program, then + * also delete it here. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include +#include