m43953/hls: add more flexible decoder process

This commit allows individual payloads to be inserted into the decoder
rather than the decoder expecting to decode each payload in a fixed
order.

When an output pointcloud is ready, the decoder will call a user
supplied callback with the conforming output pointcloud.  The user
may choose to rescale this point cloud as necessary.

tmc3/AttributeDecoder.cpp

@@ -36,6 +36,7 @@
 #include "AttributeDecoder.h"
 
 #include "ArithmeticCodec.h"
+#include "io_hls.h"
 #include "RAHT.h"
 
 namespace pcc {
@@ -55,7 +56,7 @@ struct PCCResidualsDecoder {
 
   PCCResidualsDecoder() { alphabetSize = PCCTMC3SymbolCount; }
 
-  void start(const PayloadBuffer& payload, uint32_t alphabetSize);
+  void start(const char* buf, int buf_len, uint32_t alphabetSize);
   void stop();
   bool decodePred();
   uint32_t decode0();
@@ -65,14 +66,14 @@ struct PCCResidualsDecoder {
 //----------------------------------------------------------------------------
 
 void
-PCCResidualsDecoder::start(const PayloadBuffer& buf, uint32_t alphabetSize)
+PCCResidualsDecoder::start(const char* buf, int buf_len, uint32_t alphabetSize)
 {
   this->alphabetSize = alphabetSize;
   multiSymbolModelDiff0.set_alphabet(alphabetSize + 1);
   multiSymbolModelDiff1.set_alphabet(alphabetSize + 1);
 
   arithmeticDecoder.set_buffer(
-    buf.size(), reinterpret_cast<uint8_t*>(const_cast<char*>(buf.data())));
+    buf_len, reinterpret_cast<uint8_t*>(const_cast<char*>(buf)));
 
   arithmeticDecoder.start_decoder();
 }
@@ -129,9 +130,13 @@ AttributeDecoder::decode(
   const PayloadBuffer& payload,
   PCCPointSet3& pointCloud)
 {
+  int abhSize;
+  /* AttributeBrickHeader abh = */ parseAbh(payload, &abhSize);
+
   PCCResidualsDecoder decoder;
   const uint32_t alphabetSize = 64;
-  decoder.start(payload, alphabetSize);
+  decoder.start(
+    payload.data() + abhSize, payload.size() - abhSize, alphabetSize);
 
   if (attr_desc.attr_count == 1) {
     switch (attr_aps.attr_encoding) {

tmc3/PCCTMC3Decoder.h

@@ -37,6 +37,7 @@
 #define PCCTMC3Decoder_h
 
 #include <assert.h>
+#include <functional>
 #include <queue>
 #include <string>
 #include "ringbuf.h"
@@ -60,9 +61,6 @@ namespace pcc {
 
 struct DecoderParams {
   bool roundOutputPositions;
-
-  // Filename for saving pre inverse scaled point cloud.
-  std::string preInvScalePath;
 };
 
 class PCCTMC3Decoder3 {
@@ -76,37 +74,93 @@ public:
   {
     _sps = nullptr;
     _gps = nullptr;
-    _aps.clear();
+    _spss.clear();
+    _gpss.clear();
+    _apss.clear();
   }
 
   int decompress(
     const DecoderParams& params,
-    // todo(df): refactor api to take payload buffers
-    std::istream& input,
-    PCCPointSet3& pointCloud)
+    const PayloadBuffer* buf,
+    std::function<void(const PCCPointSet3&)> onOutputCloud)
   {
-    init();
-
-    // todo(df): replace fixed payload ordering
-    PayloadBuffer buf;
-    readTlv(input, &buf);
-    SequenceParameterSet sps = parseSps(buf);
-    _sps = &sps;
-
-    buf = PayloadBuffer();
-    readTlv(input, &buf);
-    GeometryParameterSet gps = parseGps(buf);
-    _gps = &gps;
-
-    std::vector<AttributeParameterSet> apss;
-    apss.reserve(sps.attributeSets.size());
-    for (int i = 0; i < sps.attributeSets.size(); i++) {
-      buf = PayloadBuffer();
-      readTlv(input, &buf);
-      apss.emplace_back(parseAps(buf));
-      _aps.emplace_back(&apss[i]);
+    if (!buf) {
+      // flush decoder, output pending cloud if any
+      onOutputCloud(_currentPointCloud);
+      return 0;
+    }
+
+    switch (buf->type) {
+    case PayloadType::kSequenceParameterSet:
+      storeSps(parseSps(*buf));
+      return 0;
+
+    case PayloadType::kGeometryParameterSet:
+      storeGps(parseGps(*buf));
+      return 0;
+
+    case PayloadType::kAttributeParameterSet:
+      storeAps(parseAps(*buf));
+      return 0;
+
+    case PayloadType::kGeometryBrick:
+      // todo(df): call onOutputCloud when starting a new cloud
+      return decodeGeometryBrick(*buf);
+
+    case PayloadType::kAttributeBrick: decodeAttributeBrick(*buf); return 0;
     }
 
+    // todo(df): error, unhandled payload type
+    return 1;
+  }
+
+  //--------------------------------------------------------------------------
+
+  void storeSps(SequenceParameterSet&& sps)
+  {
+    // todo(df): handle replacement semantics
+    _spss.emplace(std::make_pair(sps.sps_seq_parameter_set_id, sps));
+  }
+
+  //--------------------------------------------------------------------------
+
+  void storeGps(GeometryParameterSet&& gps)
+  {
+    // todo(df): handle replacement semantics
+    _gpss.emplace(std::make_pair(gps.gps_geom_parameter_set_id, gps));
+  }
+
+  //--------------------------------------------------------------------------
+
+  void storeAps(AttributeParameterSet&& aps)
+  {
+    // todo(df): handle replacement semantics
+    _apss.emplace(std::make_pair(aps.aps_attr_parameter_set_id, aps));
+  }
+
+  //==========================================================================
+
+private:
+  //--------------------------------------------------------------------------
+  // Initialise the point cloud storage and decode a single geometry brick.
+  //
+  // NB: there is an implicit assumption that there is only a single geometry
+  //     brick per point cloud.
+  // todo(??): fixme
+
+  int decodeGeometryBrick(const PayloadBuffer& buf)
+  {
+    assert(buf.type == PayloadType::kGeometryBrick);
+    std::cout << "positions bitstream size " << buf.size() << " B\n";
+
+    // HACK: assume activation of the first SPS and GPS
+    // todo(df): parse brick header here for propper sps & gps activation
+    //  -- this is currently inconsistent between trisoup and octree
+    assert(!_spss.empty());
+    assert(!_gpss.empty());
+    _sps = &_spss.cbegin()->second;
+    _gps = &_gpss.cbegin()->second;
+
     // todo(df): replace with attribute mapping
     bool hasColour = std::any_of(
       _sps->attributeSets.begin(), _sps->attributeSets.end(),
@@ -120,75 +174,73 @@ public:
         return desc.attributeLabel == KnownAttributeLabel::kReflectance;
       });
 
-    pointCloud.addRemoveAttributes(hasColour, hasReflectance);
-
-    if (1) {
-      buf = PayloadBuffer();
-      readTlv(input, &buf);
-      assert(buf.type == PayloadType::kGeometryBrick);
+    _currentPointCloud.clear();
+    _currentPointCloud.addRemoveAttributes(hasColour, hasReflectance);
 
-      pcc::chrono::Stopwatch<pcc::chrono::utime_inc_children_clock> clock_user;
-      clock_user.start();
+    pcc::chrono::Stopwatch<pcc::chrono::utime_inc_children_clock> clock_user;
+    clock_user.start();
 
-      if (_gps->geom_codec_type == GeometryCodecType::kOctree) {
-        decodePositions(buf, pointCloud);
-      }
-      if (_gps->geom_codec_type == GeometryCodecType::kTriSoup) {
-        if (decodeTrisoup(buf, pointCloud))
-          return -1;
-      }
-      clock_user.stop();
+    if (_gps->geom_codec_type == GeometryCodecType::kOctree) {
+      decodePositions(buf, _currentPointCloud);
+    }
+    if (_gps->geom_codec_type == GeometryCodecType::kTriSoup) {
+      if (decodeTrisoup(buf, _currentPointCloud))
+        return -1;
+    }
+    clock_user.stop();
 
-      std::cout << "positions bitstream size " << buf.size() << " B\n";
+    auto total_user = std::chrono::duration_cast<std::chrono::milliseconds>(
+      clock_user.count());
+    std::cout << "positions processing time (user): "
+              << total_user.count() / 1000.0 << " s\n";
+    std::cout << std::endl;
 
-      auto total_user = std::chrono::duration_cast<std::chrono::milliseconds>(
-        clock_user.count());
-      std::cout << "positions processing time (user): "
-                << total_user.count() / 1000.0 << " s\n";
-      std::cout << std::endl;
-    }
+    return 0;
+  }
 
-    // todo(df): remove this loop with api change
-    // todo(df): attribute brick order should not depend upon sps attr order
-    for (int i = 0; i < sps.attributeSets.size(); i++) {
-      const auto& attr_sps = _sps->attributeSets[i];
-      const auto& attr_aps = *_aps[i];
-      const auto& label = attr_sps.attributeLabel;
+  //--------------------------------------------------------------------------
+  // Initialise the point cloud storage and decode a single geometry brick.
+  //
+  // NB: there is an implicit assumption that there is only a single geometry
+  //     brick per point cloud.
 
-      buf = PayloadBuffer();
-      readTlv(input, &buf);
-      assert(buf.type == PayloadType::kAttributeBrick);
+  void decodeAttributeBrick(const PayloadBuffer& buf)
+  {
+    assert(buf.type == PayloadType::kAttributeBrick);
+    // todo(df): replace assertions with error handling
+    assert(_sps);
+    assert(_gps);
 
-      AttributeDecoder attrDecoder;
-      pcc::chrono::Stopwatch<pcc::chrono::utime_inc_children_clock> clock_user;
+    // todo(df): validate that sps activation is not changed via the APS
+    AttributeBrickHeader abh = parseAbh(buf, nullptr);
+    const auto it_attr_aps = _apss.find(abh.attr_attr_parameter_set_id);
 
-      clock_user.start();
-      attrDecoder.decode(attr_sps, attr_aps, buf, pointCloud);
-      clock_user.stop();
+    assert(it_attr_aps != _apss.cend());
+    const auto& attr_aps = it_attr_aps->second;
 
-      std::cout << label << "s bitstream size " << buf.size() << " B\n";
+    assert(abh.attr_sps_attr_idx < _sps->attributeSets.size());
+    const auto& attr_sps = _sps->attributeSets[abh.attr_sps_attr_idx];
+    const auto& label = attr_sps.attributeLabel;
 
-      auto total_user = std::chrono::duration_cast<std::chrono::milliseconds>(
-        clock_user.count());
-      std::cout << label
-                << "s processing time (user): " << total_user.count() / 1000.0
-                << " s\n";
-      std::cout << std::endl;
-    }
+    AttributeDecoder attrDecoder;
+    pcc::chrono::Stopwatch<pcc::chrono::utime_inc_children_clock> clock_user;
 
-    // Dump the decoded colour using the pre inverse scaled geometry
-    if (!params.preInvScalePath.empty()) {
-      pcc::PCCPointSet3 tempPointCloud(pointCloud);
-      tempPointCloud.convertYUVToRGB();
-      tempPointCloud.write(params.preInvScalePath);
-    }
+    clock_user.start();
+    attrDecoder.decode(attr_sps, attr_aps, buf, _currentPointCloud);
+    clock_user.stop();
 
-    inverseQuantization(pointCloud, params.roundOutputPositions);
+    std::cout << label << "s bitstream size " << buf.size() << " B\n";
 
-    return 0;
+    auto total_user = std::chrono::duration_cast<std::chrono::milliseconds>(
+      clock_user.count());
+    std::cout << label
+              << "s processing time (user): " << total_user.count() / 1000.0
+              << " s\n";
+    std::cout << std::endl;
   }
 
-private:
+  //--------------------------------------------------------------------------
+
   int decodeTrisoup(const PayloadBuffer& buf, PCCPointSet3& pointCloud)
   {
     assert(buf.type == PayloadType::kGeometryBrick);
@@ -676,6 +728,9 @@ private:
     arithmeticDecoder.stop_decoder();
   }
 
+  //==========================================================================
+
+public:
   void inverseQuantization(
     PCCPointSet3& pointCloud, const bool roundOutputPositions)
   {
@@ -699,13 +754,22 @@ private:
     }
   }
 
+  //==========================================================================
+
 private:
   PCCVector3D minPositions;
 
+  // The point cloud currently being decoded
+  PCCPointSet3 _currentPointCloud;
+
+  // Received parameter sets, mapping parameter set id -> parameterset
+  std::map<int, SequenceParameterSet> _spss;
+  std::map<int, GeometryParameterSet> _gpss;
+  std::map<int, AttributeParameterSet> _apss;
+
   // The active SPS
-  SequenceParameterSet* _sps;
-  GeometryParameterSet* _gps;
-  std::vector<AttributeParameterSet*> _aps;
+  const SequenceParameterSet* _sps;
+  const GeometryParameterSet* _gps;
 };
 }  // namespace pcc
 

tmc3/PCCTMC3Encoder.h

@@ -233,8 +233,15 @@ public:
       PayloadBuffer payload(PayloadType::kAttributeBrick);
 
       pcc::chrono::Stopwatch<pcc::chrono::utime_inc_children_clock> clock_user;
-
       clock_user.start();
+
+      // todo(df): move elsewhere?
+      AttributeBrickHeader abh;
+      abh.attr_attr_parameter_set_id = attr_aps.aps_attr_parameter_set_id;
+      abh.attr_sps_attr_idx = attrIdx;
+      abh.attr_geom_brick_id = 0;
+      write(abh, &payload);
+
       AttributeEncoder attrEncoder;
       attrEncoder.encode(attr_sps, attr_aps, pointCloud, &payload);
       clock_user.stop();

tmc3/TMC3.cpp

@@ -50,6 +50,9 @@ struct Parameters {
   std::string compressedStreamPath;
   std::string reconstructedDataPath;
 
+  // Filename for saving pre inverse scaled point cloud.
+  std::string preInvScalePath;
+
   pcc::EncoderParams encoder;
   pcc::DecoderParams decoder;
 
@@ -232,7 +235,7 @@ ParseParameters(int argc, char* argv[], Parameters& params)
     "Recolored pointcloud file path (encoder only)")
 
   ("preInvScalePath",
-    params.decoder.preInvScalePath, {},
+    params.preInvScalePath, {},
     "Pre inverse scaled pointcloud file path (decoder only)")
 
   // general
@@ -541,26 +544,56 @@ Decompress(Parameters& params, Stopwatch& clock)
 
   clock.start();
 
+  PayloadBuffer buf;
   PCCTMC3Decoder3 decoder;
-  PCCPointSet3 pointCloud;
 
-  int ret = decoder.decompress(params.decoder, fin, pointCloud);
-  if (ret) {
-    cout << "Error: can't decompress point cloud!" << endl;
-    return -1;
-  }
-  std::cout << "Total bitstream size " << fin.tellg() << " B" << std::endl;
+  while (true) {
+    PayloadBuffer* buf_ptr = &buf;
+    readTlv(fin, &buf);
 
-  if (params.colorTransform == COLOR_TRANSFORM_RGB_TO_YCBCR) {
-    pointCloud.convertYUVToRGB();
-  }
+    // at end of file (or other error), flush decoder
+    if (!fin)
+      buf_ptr = nullptr;
 
-  clock.stop();
+    int ret = decoder.decompress(
+      params.decoder, buf_ptr, [&](const PCCPointSet3& decodedPointCloud) {
+        PCCPointSet3 pointCloud(decodedPointCloud);
 
-  if (!pointCloud.write(params.reconstructedDataPath, true)) {
-    cout << "Error: can't open output file!" << endl;
-    return -1;
+        if (params.colorTransform == COLOR_TRANSFORM_RGB_TO_YCBCR) {
+          pointCloud.convertYUVToRGB();
+        }
+
+        // Dump the decoded colour using the pre inverse scaled geometry
+        if (!params.preInvScalePath.empty()) {
+          pointCloud.write(params.preInvScalePath);
+        }
+
+        decoder.inverseQuantization(
+          pointCloud, params.decoder.roundOutputPositions);
+
+        clock.stop();
+
+        if (!pointCloud.write(params.reconstructedDataPath, true)) {
+          cout << "Error: can't open output file!" << endl;
+        }
+
+        clock.start();
+      });
+
+    if (ret) {
+      cout << "Error: can't decompress point cloud!" << endl;
+      return -1;
+    }
+
+    if (!buf_ptr)
+      break;
   }
 
+  fin.clear();
+  fin.seekg(0, ios_base::end);
+  std::cout << "Total bitstream size " << fin.tellg() << " B" << std::endl;
+
+  clock.stop();
+
   return 0;
 }

tmc3/hls.h

@@ -253,6 +253,7 @@ struct AttributeParameterSet {
 //============================================================================
 
 struct AttributeBrickHeader {
+  int attr_sps_attr_idx;
   int attr_attr_parameter_set_id;
   int attr_geom_brick_id;
 };

tmc3/io_hls.cpp

@@ -406,6 +406,7 @@ write(const AttributeBrickHeader& abh, PayloadBuffer* buf)
   auto bs = makeBitWriter(std::back_inserter(*buf));
 
   bs.writeUe(abh.attr_attr_parameter_set_id);
+  bs.writeUe(abh.attr_sps_attr_idx);
   //bs.writeUe(abh.attr_geom_brick_id);
 
   bs.byteAlign();
@@ -421,6 +422,7 @@ parseAbh(const PayloadBuffer& buf, int* bytesRead)
   auto bs = makeBitReader(buf.begin(), buf.end());
 
   bs.readUe(&abh.attr_attr_parameter_set_id);
+  bs.readUe(&abh.attr_sps_attr_idx);
   //bs.readUe(&abh.attr_geom_brick_id);
 
   bs.byteAlign();

tmc3/io_tlv.cpp

@@ -35,6 +35,8 @@
 
 #include "io_tlv.h"
 
+#include <cstdint>
+
 namespace pcc {
 
 //============================================================================
@@ -59,6 +61,7 @@ writeTlv(const PayloadBuffer& buf, std::ostream& os)
 std::istream&
 readTlv(std::istream& is, PayloadBuffer* buf)
 {
+  buf->resize(0);
   buf->type = PayloadType(static_cast<unsigned>(is.get()));
 
   uint32_t length = 0;
@@ -67,7 +70,9 @@ readTlv(std::istream& is, PayloadBuffer* buf)
   length = (length << 8) | static_cast<unsigned>(is.get());
   length = (length << 8) | static_cast<unsigned>(is.get());
 
-  buf->resize(0);
+  if (!is)
+    return is;
+
   buf->resize(length);
   is.read(buf->data(), length);
   return is;