Commit 5eee0d72 authored by Khaled Mammou's avatar Khaled Mammou Committed by David Flynn
Browse files

m42640/attr: lifting scheme for lossy attribute encoding

This commit implements a lifting scheme optimized for lossy attribute
compression, consisting of predict/update operators, and an adaptive
quantization strategy.
parent 85474147
......@@ -129,7 +129,9 @@ AttributeDecoder::decodeHeader(
PCCReadFromBuffer<uint8_t>(bitstream.buffer, transType, bitstream.size);
transformType = TransformType(transType);
if (transformType == TransformType::kIntegerLift) {
if (
transformType == TransformType::kIntegerLift
|| transformType == TransformType::kLift) {
uint8_t numberOfNearestNeighborsCount = 0;
PCCReadFromBuffer<uint8_t>(
bitstream.buffer, numberOfNearestNeighborsCount, bitstream.size);
......@@ -193,6 +195,10 @@ AttributeDecoder::decodeReflectances(
case TransformType::kIntegerLift:
decodeReflectancesIntegerLift(decoder, pointCloud);
break;
case TransformType::kLift:
decodeReflectancesLift(decoder, pointCloud);
break;
}
decoder.stop();
......@@ -218,6 +224,8 @@ AttributeDecoder::decodeColors(
case TransformType::kIntegerLift:
decodeColorsIntegerLift(decoder, pointCloud);
break;
case TransformType::kLift: decodeColorsLift(decoder, pointCloud); break;
}
decoder.stop();
......@@ -575,6 +583,131 @@ AttributeDecoder::decodeColorsRaht(
delete[] weight;
}
//----------------------------------------------------------------------------
void
AttributeDecoder::decodeColorsLift(
PCCResidualsDecoder& decoder, PCCPointSet3& pointCloud)
{
const size_t pointCount = pointCloud.getPointCount();
std::vector<uint32_t> numberOfPointsPerLOD;
std::vector<uint32_t> indexesLOD;
PCCBuildLevelOfDetail(
pointCloud, levelOfDetailCount, dist2, numberOfPointsPerLOD, indexesLOD);
const size_t lodCount = numberOfPointsPerLOD.size();
std::vector<PCCPredictor> predictors;
PCCComputePredictors(
pointCloud, numberOfPointsPerLOD, indexesLOD,
numberOfNearestNeighborsInPrediction, predictors);
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
predictors[predictorIndex].computeWeights();
}
std::vector<PCCVector3D> colors;
colors.resize(pointCount);
std::vector<double> weights;
PCCComputeQuantizationWeights(predictors, weights);
// decompress
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
uint32_t values[3];
values[0] = decoder.decode0();
values[1] = decoder.decode1();
values[2] = decoder.decode1();
const size_t lodIndex = predictors[predictorIndex].levelOfDetailIndex;
const int64_t qs = quantizationStepsLuma[lodIndex];
const int64_t qs2 = quantizationStepsChroma[lodIndex];
const double quantWeight = sqrt(weights[predictorIndex]);
auto& color = colors[predictorIndex];
const int64_t delta = o3dgc::UIntToInt(values[0]);
const double reconstructedDelta = PCCInverseQuantization(delta, qs);
color[0] = reconstructedDelta / quantWeight;
for (size_t d = 1; d < 3; ++d) {
const int64_t delta = o3dgc::UIntToInt(values[d]);
const double reconstructedDelta = PCCInverseQuantization(delta, qs2);
color[d] = reconstructedDelta / quantWeight;
}
}
// reconstruct
for (size_t lodIndex = 1; lodIndex < lodCount; ++lodIndex) {
const size_t startIndex = numberOfPointsPerLOD[lodIndex - 1];
const size_t endIndex = numberOfPointsPerLOD[lodIndex];
PCCLiftUpdate(predictors, weights, startIndex, endIndex, false, colors);
PCCLiftPredict(predictors, startIndex, endIndex, false, colors);
}
for (size_t f = 0; f < pointCount; ++f) {
const auto& predictor = predictors[f];
PCCColor3B color;
for (size_t d = 0; d < 3; ++d) {
color[d] = uint8_t(PCCClip(std::round(colors[f][d]), 0.0, 255.0));
}
pointCloud.setColor(predictor.index, color);
}
}
//----------------------------------------------------------------------------
void
AttributeDecoder::decodeReflectancesLift(
PCCResidualsDecoder& decoder, PCCPointSet3& pointCloud)
{
std::vector<PCCPredictor> predictors;
std::vector<uint32_t> numberOfPointsPerLOD;
std::vector<uint32_t> indexesLOD;
PCCBuildLevelOfDetail(
pointCloud, levelOfDetailCount, dist2, numberOfPointsPerLOD, indexesLOD);
const size_t pointCount = predictors.size();
const size_t lodCount = numberOfPointsPerLOD.size();
PCCComputePredictors(
pointCloud, numberOfPointsPerLOD, indexesLOD,
numberOfNearestNeighborsInPrediction, predictors);
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
predictors[predictorIndex].computeWeights();
}
std::vector<double> reflectances;
reflectances.resize(pointCount);
std::vector<double> weights;
PCCComputeQuantizationWeights(predictors, weights);
// decompress
std::vector<WeightWithIndex> order;
order.reserve(pointCount);
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
order.push_back(WeightWithIndex(predictorIndex, weights[predictorIndex]));
}
std::sort(order.begin(), order.end());
for (const auto& w : order) {
const int64_t detail = decoder.decode0();
const size_t predictorIndex = w.index;
const size_t lodIndex = predictors[predictorIndex].levelOfDetailIndex;
const int64_t qs = quantizationStepsLuma[lodIndex];
const double quantWeight = sqrt(weights[predictorIndex]);
auto& reflectance = reflectances[predictorIndex];
const int64_t delta = o3dgc::UIntToInt(detail);
const double reconstructedDelta = PCCInverseQuantization(delta, qs);
reflectance = reconstructedDelta / quantWeight;
}
// reconstruct
for (size_t lodIndex = 1; lodIndex < lodCount; ++lodIndex) {
const size_t startIndex = numberOfPointsPerLOD[lodIndex - 1];
const size_t endIndex = numberOfPointsPerLOD[lodIndex];
PCCLiftUpdate(
predictors, weights, startIndex, endIndex, false, reflectances);
PCCLiftPredict(predictors, startIndex, endIndex, false, reflectances);
}
const double maxReflectance = std::numeric_limits<uint16_t>::max();
for (size_t f = 0; f < pointCount; ++f) {
pointCloud.setReflectance(
predictors[f].index,
uint16_t(PCCClip(std::round(reflectances[f]), 0.0, maxReflectance)));
}
}
//----------------------------------------------------------------------------
//============================================================================
} /* namespace pcc */
......@@ -63,6 +63,12 @@ public:
protected:
// todo(df): consider alternative encapsulation
void decodeReflectancesLift(
PCCResidualsDecoder& decoder, PCCPointSet3& pointCloud);
void
decodeColorsLift(PCCResidualsDecoder& decoder, PCCPointSet3& pointCloud);
void decodeReflectancesIntegerLift(
PCCResidualsDecoder& decoder, PCCPointSet3& pointCloud);
......
......@@ -271,7 +271,9 @@ AttributeEncoder::encodeHeader(
PCCWriteToBuffer<uint8_t>(
uint8_t(attributeParams.transformType), bitstream.buffer, bitstream.size);
if (attributeParams.transformType == TransformType::kIntegerLift) {
if (
attributeParams.transformType == TransformType::kIntegerLift
|| attributeParams.transformType == TransformType::kLift) {
PCCWriteToBuffer<uint8_t>(
uint8_t(attributeParams.numberOfNearestNeighborsInPrediction),
bitstream.buffer, bitstream.size);
......@@ -339,6 +341,10 @@ AttributeEncoder::encodeReflectances(
case TransformType::kIntegerLift:
encodeReflectancesIntegerLift(reflectanceParams, pointCloud, encoder);
break;
case TransformType::kLift:
encodeReflectancesLift(reflectanceParams, pointCloud, encoder);
break;
}
uint32_t compressedBitstreamSize = encoder.stop();
......@@ -369,6 +375,10 @@ AttributeEncoder::encodeColors(
case TransformType::kIntegerLift:
encodeColorsIntegerLift(colorParams, pointCloud, encoder);
break;
case TransformType::kLift:
encodeColorsLift(colorParams, pointCloud, encoder);
break;
}
uint32_t compressedBitstreamSize = encoder.stop();
......@@ -892,6 +902,167 @@ AttributeEncoder::encodeColorsTransformRaht(
delete[] weight;
}
//----------------------------------------------------------------------------
void
AttributeEncoder::encodeColorsLift(
const PCCAttributeEncodeParamaters& colorParams,
PCCPointSet3& pointCloud,
PCCResidualsEncoder& encoder)
{
const size_t pointCount = pointCloud.getPointCount();
std::vector<uint32_t> numberOfPointsPerLOD;
std::vector<uint32_t> indexesLOD;
PCCBuildLevelOfDetail(
pointCloud, colorParams.levelOfDetailCount, colorParams.dist2,
numberOfPointsPerLOD, indexesLOD);
std::vector<PCCPredictor> predictors;
PCCComputePredictors(
pointCloud, numberOfPointsPerLOD, indexesLOD,
colorParams.numberOfNearestNeighborsInPrediction, predictors);
const size_t lodCount = numberOfPointsPerLOD.size();
std::vector<PCCVector3D> colors;
colors.resize(pointCount);
for (size_t index = 0; index < pointCount; ++index) {
const auto& color = pointCloud.getColor(indexesLOD[index]);
for (size_t d = 0; d < 3; ++d) {
colors[index][d] = color[d];
}
}
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
predictors[predictorIndex].computeWeights();
}
std::vector<double> weights;
PCCComputeQuantizationWeights(predictors, weights);
for (size_t i = 0; (i + 1) < lodCount; ++i) {
const size_t lodIndex = lodCount - i - 1;
const size_t startIndex = numberOfPointsPerLOD[lodIndex - 1];
const size_t endIndex = numberOfPointsPerLOD[lodIndex];
PCCLiftPredict(predictors, startIndex, endIndex, true, colors);
PCCLiftUpdate(predictors, weights, startIndex, endIndex, true, colors);
}
// compress
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
const size_t lodIndex = predictors[predictorIndex].levelOfDetailIndex;
const int64_t qs = colorParams.quantizationStepsLuma[lodIndex];
const double quantWeight = sqrt(weights[predictorIndex]);
auto& color = colors[predictorIndex];
const int64_t delta = PCCQuantization(color[0] * quantWeight, qs);
const int64_t detail = o3dgc::IntToUInt(delta);
assert(detail < std::numeric_limits<uint32_t>::max());
const double reconstructedDelta = PCCInverseQuantization(delta, qs);
color[0] = reconstructedDelta / quantWeight;
uint32_t values[3];
values[0] = uint32_t(detail);
const size_t qs2 = colorParams.quantizationStepsChroma[lodIndex];
for (size_t d = 1; d < 3; ++d) {
const int64_t delta = PCCQuantization(color[d] * quantWeight, qs2);
const int64_t detail = o3dgc::IntToUInt(delta);
assert(detail < std::numeric_limits<uint32_t>::max());
const double reconstructedDelta = PCCInverseQuantization(delta, qs2);
color[d] = reconstructedDelta / quantWeight;
values[d] = uint32_t(detail);
}
encoder.encode0(values[0]);
encoder.encode1(values[1]);
encoder.encode1(values[2]);
}
// reconstruct
for (size_t lodIndex = 1; lodIndex < lodCount; ++lodIndex) {
const size_t startIndex = numberOfPointsPerLOD[lodIndex - 1];
const size_t endIndex = numberOfPointsPerLOD[lodIndex];
PCCLiftUpdate(predictors, weights, startIndex, endIndex, false, colors);
PCCLiftPredict(predictors, startIndex, endIndex, false, colors);
}
for (size_t f = 0; f < pointCount; ++f) {
const auto& predictor = predictors[f];
PCCColor3B color;
for (size_t d = 0; d < 3; ++d) {
color[d] = uint8_t(PCCClip(std::round(colors[f][d]), 0.0, 255.0));
}
pointCloud.setColor(predictor.index, color);
}
}
//----------------------------------------------------------------------------
void
AttributeEncoder::encodeReflectancesLift(
const PCCAttributeEncodeParamaters& reflectanceParams,
PCCPointSet3& pointCloud,
PCCResidualsEncoder& encoder)
{
std::vector<uint32_t> numberOfPointsPerLOD;
std::vector<uint32_t> indexesLOD;
PCCBuildLevelOfDetail(
pointCloud, reflectanceParams.levelOfDetailCount, reflectanceParams.dist2,
numberOfPointsPerLOD, indexesLOD);
std::vector<PCCPredictor> predictors;
PCCComputePredictors(
pointCloud, numberOfPointsPerLOD, indexesLOD,
reflectanceParams.numberOfNearestNeighborsInPrediction, predictors);
const size_t pointCount = predictors.size();
const size_t lodCount = numberOfPointsPerLOD.size();
std::vector<double> reflectances;
reflectances.resize(pointCount);
for (size_t index = 0; index < pointCount; ++index) {
reflectances[index] = pointCloud.getReflectance(indexesLOD[index]);
}
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
predictors[predictorIndex].computeWeights();
}
std::vector<double> weights;
PCCComputeQuantizationWeights(predictors, weights);
for (size_t i = 0; (i + 1) < lodCount; ++i) {
const size_t lodIndex = lodCount - i - 1;
const size_t startIndex = numberOfPointsPerLOD[lodIndex - 1];
const size_t endIndex = numberOfPointsPerLOD[lodIndex];
PCCLiftPredict(predictors, startIndex, endIndex, true, reflectances);
PCCLiftUpdate(
predictors, weights, startIndex, endIndex, true, reflectances);
}
// compress
for (size_t predictorIndex = 0; predictorIndex < pointCount;
++predictorIndex) {
const size_t lodIndex = predictors[predictorIndex].levelOfDetailIndex;
const int64_t qs = reflectanceParams.quantizationStepsLuma[lodIndex];
const double quantWeight = sqrt(weights[predictorIndex]);
auto& reflectance = reflectances[predictorIndex];
const int64_t delta = PCCQuantization(reflectance * quantWeight, qs);
const int64_t detail = o3dgc::IntToUInt(delta);
assert(detail < std::numeric_limits<uint32_t>::max());
const double reconstructedDelta = PCCInverseQuantization(delta, qs);
reflectance = reconstructedDelta / quantWeight;
encoder.encode0(detail);
}
// reconstruct
for (size_t lodIndex = 1; lodIndex < lodCount; ++lodIndex) {
const size_t startIndex = numberOfPointsPerLOD[lodIndex - 1];
const size_t endIndex = numberOfPointsPerLOD[lodIndex];
PCCLiftUpdate(
predictors, weights, startIndex, endIndex, false, reflectances);
PCCLiftPredict(predictors, startIndex, endIndex, false, reflectances);
}
const double maxReflectance = std::numeric_limits<uint16_t>::max();
for (size_t f = 0; f < pointCount; ++f) {
pointCloud.setReflectance(
predictors[f].index,
uint16_t(PCCClip(std::round(reflectances[f]), 0.0, maxReflectance)));
}
}
//============================================================================
} /* namespace pcc */
......@@ -87,6 +87,16 @@ public:
protected:
// todo(df): consider alternative encapsulation
void encodeReflectancesLift(
const PCCAttributeEncodeParamaters& reflectanceParams,
PCCPointSet3& pointCloud,
PCCResidualsEncoder& encoder);
void encodeColorsLift(
const PCCAttributeEncodeParamaters& colorParams,
PCCPointSet3& pointCloud,
PCCResidualsEncoder& encoder);
void encodeReflectancesIntegerLift(
const PCCAttributeEncodeParamaters& reflectanceParams,
PCCPointSet3& pointCloud,
......
......@@ -248,6 +248,12 @@ public:
findNeighbors(root, query2, result);
}
uint32_t
hasNeighborWithinRange(const PCCVector3D& point, const double radius2) const
{
return hasNeighborWithinRange(root, point, radius2);
}
private:
static PCCAxis3 nextAxis(const PCCAxis3 axis)
{
......@@ -427,6 +433,40 @@ private:
findNeighbors(second, query2, result);
}
uint32_t hasNeighborWithinRange(
const uint32_t current,
const PCCVector3D& point,
const double radius2) const
{
if (current == PCC_UNDEFINED_INDEX) {
return PCC_UNDEFINED_INDEX;
}
const PCCIncrementalKdTree3Node& node = nodes[current];
const double dist2 = (point - node.pos).getNorm2();
if (dist2 < radius2) {
return node.id;
}
const PCCAxis3 splitAxis = node.axis;
const int32_t coord = node.pos[splitAxis];
const int32_t dx = point[splitAxis] - coord;
uint32_t first, second;
if (dx < 0) {
first = node.left;
second = node.right;
} else {
first = node.right;
second = node.left;
}
const uint32_t index = hasNeighborWithinRange(first, point, radius2);
if (index != PCC_UNDEFINED_INDEX) {
return index;
}
if ((dx * dx) > radius2) {
return PCC_UNDEFINED_INDEX;
}
return hasNeighborWithinRange(second, point, radius2);
}
private:
std::vector<PCCIncrementalKdTree3Node> nodes;
uint32_t root;
......
......@@ -41,7 +41,11 @@
#include "PCCMath.h"
#include "ringbuf.h"
#include "nanoflann.hpp"
#include <cstdint>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace pcc {
......@@ -66,7 +70,8 @@ enum class GeometryCodecType
enum class TransformType
{
kIntegerLift = 0,
kRAHT = 1
kRAHT = 1,
kLift = 2
};
struct PCCOctree3Node {
......@@ -96,6 +101,27 @@ struct PCCOctree3Node {
uint8_t siblingOccupancy;
};
// Structure for sorting weights.
struct WeightWithIndex {
float weight;
int index;
WeightWithIndex() = default;
WeightWithIndex(const int index, const float weight)
: weight(weight), index(index)
{}
// NB: this definition ranks larger weights before smaller values.
bool operator<(const WeightWithIndex& rhs) const
{
// NB: index used to maintain stable sort
if (weight == rhs.weight)
return index < rhs.index;
return weight > rhs.weight;
}
};
struct PCCNeighborInfo {
double weight;
uint32_t index;
......@@ -181,6 +207,116 @@ PCCInverseQuantization(const int64_t value, const int64_t qs)
return qs == 0 ? value : (value * qs);
}
inline int64_t
PCCQuantization(const double value, const int64_t qs)
{
return int64_t(
value >= 0.0 ? std::floor(value / qs + 1.0 / 3.0)
: -std::floor(-value / qs + 1.0 / 3.0));
}
//---------------------------------------------------------------------------
template<typename T>
void
PCCLiftPredict(
const std::vector<PCCPredictor>& predictors,
const size_t startIndex,
const size_t endIndex,
const bool direct,
std::vector<T>& attributes)
{
const size_t predictorCount = endIndex - startIndex;
for (size_t index = 0; index < predictorCount; ++index) {
const size_t predictorIndex = predictorCount - index - 1 + startIndex;
const auto& predictor = predictors[predictorIndex];
T predicted(0.0);
for (size_t i = 0; i < predictor.neighborCount; ++i) {
const size_t neighborPredIndex = predictor.neighbors[i].predictorIndex;
const double weight = predictor.neighbors[i].weight;
assert(neighborPredIndex < startIndex);
predicted += weight * attributes[neighborPredIndex];
}
if (direct) {
attributes[predictorIndex] -= predicted;
} else {
attributes[predictorIndex] += predicted;
}
}
}
//---------------------------------------------------------------------------
template<typename T>
void
PCCLiftUpdate(
const std::vector<PCCPredictor>& predictors,
const std::vector<double>& quantizationWeights,
const size_t startIndex,
const size_t endIndex,
const bool direct,
std::vector<T>& attributes)
{
std::vector<double> updateWeights;
updateWeights.resize(startIndex, 0.0);
std::vector<T> updates;
updates.resize(startIndex);
for (size_t index = 0; index < startIndex; ++index) {
updates[index] = 0.0;
}
const size_t predictorCount = endIndex - startIndex;
for (size_t index = 0; index < predictorCount; ++index) {
const size_t predictorIndex = predictorCount - index - 1 + startIndex;
const auto& predictor = predictors[predictorIndex];
const double currentQuantWeight = quantizationWeights[predictorIndex];
for (size_t i = 0; i < predictor.neighborCount; ++i) {
const size_t neighborPredIndex = predictor.neighbors[i].predictorIndex;
const double weight = predictor.neighbors[i].weight * currentQuantWeight;
assert(neighborPredIndex < startIndex);
updateWeights[neighborPredIndex] += weight;
updates[neighborPredIndex] += weight * attributes[predictorIndex];
}
}
for (size_t predictorIndex = 0; predictorIndex < startIndex;
++predictorIndex) {
const double sumWeights = updateWeights[predictorIndex];
if (sumWeights > 0.0) {
const double alpha = 1.0 / sumWeights;
if (direct) {
attributes[predictorIndex] += alpha * updates[predictorIndex];
} else {
attributes[predictorIndex] -= alpha * updates[predictorIndex];
}
}
}
}
//---------------------------------------------------------------------------
inline void
PCCComputeQuantizationWeights(
const std::vector<PCCPredictor>& predictors,
std::vector<double>& quantizationWeights)
{
const size_t pointCount = predictors.size();
quantizationWeights.resize(pointCount);
for (size_t i = 0; i < pointCount; ++i) {
quantizationWeights[i] = 1.0;
}
for (size_t i = 0; i < pointCount; ++i) {
const size_t predictorIndex = pointCount - i - 1;
const auto& predictor = predictors[predictorIndex];