hls/m51025: support multi-frame sequences

This adds:
 - a frame_idx syntax element to every geometry slice header.
 - (decoder) handling of a frame boundary data unit.

Frame boundaries are represented by changes in the value of frame_idx
or by the presence of a frame boundary data unit.

tmc3/PCCTMC3Decoder.h

@@ -80,8 +80,10 @@ public:
   //==========================================================================
 
 private:
+  void activateParameterSets(const GeometryBrickHeader& gbh);
   int decodeGeometryBrick(const PayloadBuffer& buf);
   void decodeAttributeBrick(const PayloadBuffer& buf);
+  bool frameIdxChanged(const GeometryBrickHeader& gbh) const;
 
   //==========================================================================
 
@@ -97,6 +99,9 @@ private:
   // Current identifier of payloads with the same geometry
   int _sliceId;
 
+  // The last decoded frame_idx
+  int _currentFrameIdx;
+
   // Position of the slice in the translated+scaled co-ordinate system.
   Vec3<int> _sliceOrigin;
 

tmc3/PCCTMC3Encoder.h

@@ -86,7 +86,7 @@ struct EncoderParams {
 
 class PCCTMC3Encoder3 {
 public:
-  PCCTMC3Encoder3() = default;
+  PCCTMC3Encoder3();
   PCCTMC3Encoder3(const PCCTMC3Encoder3&) = default;
   PCCTMC3Encoder3& operator=(const PCCTMC3Encoder3& rhs) = default;
   ~PCCTMC3Encoder3() = default;
@@ -138,6 +138,10 @@ private:
   // Identifies the current tile
   int _tileId;
 
+  // Current frame number.
+  // NB: only the log2_max_frame_idx LSBs are sampled for frame_idx
+  int _frameCounter;
+
   // Map quantized points to the original input points
   std::multimap<Vec3<double>, int32_t> quantizedToOrigin;
 };

tmc3/decoder.cpp

@@ -54,6 +54,7 @@ namespace pcc {
 void
 PCCTMC3Decoder3::init()
 {
+  _currentFrameIdx = -1;
   _sps = nullptr;
   _gps = nullptr;
   _spss.clear();
@@ -63,15 +64,23 @@ PCCTMC3Decoder3::init()
 
 //============================================================================
 
+static bool
+payloadStartsNewSlice(PayloadType type)
+{
+  return type == PayloadType::kGeometryBrick
+    || type == PayloadType::kFrameBoundaryMarker;
+}
+
+//============================================================================
+
 int
 PCCTMC3Decoder3::decompress(
   const PayloadBuffer* buf,
   std::function<void(const PCCPointSet3&)> onOutputCloud)
 {
-  // todo(df): call _accumCloud.clear() at start of frame
   // Starting a new geometry brick/slice/tile, transfer any
   // finished points to the output accumulator
-  if (!buf || buf->type == PayloadType::kGeometryBrick) {
+  if (!buf || payloadStartsNewSlice(buf->type)) {
     if (size_t numPoints = _currentPointCloud.getPointCount()) {
       for (size_t i = 0; i < numPoints; i++)
         for (int k = 0; k < 3; k++)
@@ -83,6 +92,7 @@ PCCTMC3Decoder3::decompress(
   if (!buf) {
     // flush decoder, output pending cloud if any
     onOutputCloud(_accumCloud);
+    _accumCloud.clear();
     return 0;
   }
 
@@ -93,8 +103,21 @@ PCCTMC3Decoder3::decompress(
 
   case PayloadType::kAttributeParameterSet: storeAps(parseAps(*buf)); return 0;
 
+  // the frame boundary marker flushes the current frame.
+  // NB: frame counter is reset to avoid outputing a runt point cloud
+  //     on the next slice.
+  case PayloadType::kFrameBoundaryMarker:
+    onOutputCloud(_accumCloud);
+    _accumCloud.clear();
+    _currentFrameIdx = -1;
+    return 0;
+
   case PayloadType::kGeometryBrick:
-    // todo(df): call onOutputCloud when starting a new cloud
+    activateParameterSets(parseGbhIds(*buf));
+    if (frameIdxChanged(parseGbh(*_sps, *_gps, *buf, nullptr))) {
+      onOutputCloud(_accumCloud);
+      _accumCloud.clear();
+    }
     return decodeGeometryBrick(*buf);
 
   case PayloadType::kAttributeBrick: decodeAttributeBrick(*buf); return 0;
@@ -145,14 +168,21 @@ PCCTMC3Decoder3::storeTileInventory(TileInventory&& inventory)
 }
 
 //==========================================================================
-// Initialise the point cloud storage and decode a single geometry slice.
 
-int
-PCCTMC3Decoder3::decodeGeometryBrick(const PayloadBuffer& buf)
+bool
+PCCTMC3Decoder3::frameIdxChanged(const GeometryBrickHeader& gbh) const
 {
-  assert(buf.type == PayloadType::kGeometryBrick);
-  std::cout << "positions bitstream size " << buf.size() << " B\n";
+  // dont treat the first frame in the sequence as a frame boundary
+  if (_currentFrameIdx < 0)
+    return false;
+  return _currentFrameIdx != gbh.frame_idx;
+}
 
+//==========================================================================
+
+void
+PCCTMC3Decoder3::activateParameterSets(const GeometryBrickHeader& gbh)
+{
   // HACK: assume activation of the first SPS and GPS
   // todo(df): parse brick header here for propper sps & gps activation
   //  -- this is currently inconsistent between trisoup and octree
@@ -160,6 +190,16 @@ PCCTMC3Decoder3::decodeGeometryBrick(const PayloadBuffer& buf)
   assert(!_gpss.empty());
   _sps = &_spss.cbegin()->second;
   _gps = &_gpss.cbegin()->second;
+}
+
+//==========================================================================
+// Initialise the point cloud storage and decode a single geometry slice.
+
+int
+PCCTMC3Decoder3::decodeGeometryBrick(const PayloadBuffer& buf)
+{
+  assert(buf.type == PayloadType::kGeometryBrick);
+  std::cout << "positions bitstream size " << buf.size() << " B\n";
 
   // todo(df): replace with attribute mapping
   bool hasColour = std::any_of(
@@ -181,9 +221,10 @@ PCCTMC3Decoder3::decodeGeometryBrick(const PayloadBuffer& buf)
   clock_user.start();
 
   int gbhSize;
-  _gbh = parseGbh(*_gps, buf, &gbhSize);
+  _gbh = parseGbh(*_sps, *_gps, buf, &gbhSize);
   _sliceId = _gbh.geom_slice_id;
   _sliceOrigin = _gbh.geomBoxOrigin;
+  _currentFrameIdx = _gbh.frame_idx;
 
   EntropyDecoder arithmeticDecoder;
   arithmeticDecoder.enableBypassStream(_sps->cabac_bypass_stream_enabled_flag);

tmc3/encoder.cpp

@@ -50,6 +50,11 @@ namespace pcc {
 
 //============================================================================
 
+PCCTMC3Encoder3::PCCTMC3Encoder3() : _frameCounter(-1)
+{}
+
+//============================================================================
+
 int
 PCCTMC3Encoder3::compress(
   const PCCPointSet3& inputPointCloud,
@@ -57,6 +62,9 @@ PCCTMC3Encoder3::compress(
   std::function<void(const PayloadBuffer&)> outputFn,
   PCCPointSet3* reconstructedCloud)
 {
+  // start of frame
+  _frameCounter++;
+
   fixupParameterSets(params);
 
   // Determine input bounding box (for SPS metadata) if not manually set
@@ -264,6 +272,9 @@ PCCTMC3Encoder3::fixupParameterSets(EncoderParams* params)
   params->sps.profileCompatibility.profile_compatibility_flags = 0;
   params->sps.level = 0;
 
+  // use one bit to indicate frame boundaries
+  params->sps.log2_max_frame_idx = 1;
+
   // the encoder writes out the slice origin in the GBH
   // NB: this may be disabled during encoding
   params->gps.geom_box_present_flag = true;
@@ -456,11 +467,12 @@ PCCTMC3Encoder3::encodeGeometryBrick(PayloadBuffer* buf)
   gbh.geom_geom_parameter_set_id = _gps->gps_geom_parameter_set_id;
   gbh.geom_slice_id = _sliceId;
   gbh.geom_tile_id = std::max(0, _tileId);
+  gbh.frame_idx = _frameCounter & ((1 << _sps->log2_max_frame_idx) - 1);
   gbh.geomBoxOrigin = _sliceOrigin;
   gbh.geom_box_log2_scale = 0;
   gbh.geom_max_node_size_log2 = nodeSizeLog2;
   gbh.geom_num_points = int(pointCloud.getPointCount());
-  write(*_gps, gbh, buf);
+  write(*_sps, *_gps, gbh, buf);
 
   // todo(df): remove estimate when arithmetic codec is replaced
   int maxAcBufLen = int(pointCloud.getPointCount()) * 3 * 4 + 1024;

tmc3/hls.h

@@ -53,6 +53,7 @@ enum class PayloadType
   kAttributeParameterSet = 3,
   kAttributeBrick = 4,
   kTileInventory = 5,
+  kFrameBoundaryMarker = 6,
 };
 
 //============================================================================
@@ -158,6 +159,9 @@ struct SequenceParameterSet {
   // NB: attributeSets.size() = num_attribute_sets
   std::vector<AttributeDescription> attributeSets;
 
+  // The number of bits to use for frame_idx
+  int log2_max_frame_idx;
+
   // Controls whether bypass bins are written to a seperate sub-stream, or
   // encoded as ep bins via CABAC.
   bool cabac_bypass_stream_enabled_flag;
@@ -224,6 +228,7 @@ struct GeometryBrickHeader {
   int geom_geom_parameter_set_id;
   int geom_tile_id;
   int geom_slice_id;
+  int frame_idx;
 
   // derived from geom_box_origin_{x,y,z} * (1 << geom_box_log2_scale)
   Vec3<int> geomBoxOrigin;

tmc3/io_hls.cpp

@@ -111,6 +111,7 @@ write(const SequenceParameterSet& sps)
     }
   }
 
+  bs.writeUn(5, sps.log2_max_frame_idx);
   bs.write(sps.cabac_bypass_stream_enabled_flag);
 
   bool sps_extension_flag = false;
@@ -171,6 +172,7 @@ parseSps(const PayloadBuffer& buf)
     }
   }
 
+  bs.readUn(5, &sps.log2_max_frame_idx);
   bs.read(&sps.cabac_bypass_stream_enabled_flag);
 
   bool sps_extension_flag = bs.read();
@@ -371,6 +373,7 @@ parseAps(const PayloadBuffer& buf)
 
 void
 write(
+  const SequenceParameterSet& sps,
   const GeometryParameterSet& gps,
   const GeometryBrickHeader& gbh,
   PayloadBuffer* buf)
@@ -381,6 +384,7 @@ write(
   bs.writeUe(gbh.geom_geom_parameter_set_id);
   bs.writeUe(gbh.geom_tile_id);
   bs.writeUe(gbh.geom_slice_id);
+  bs.writeUn(sps.log2_max_frame_idx, gbh.frame_idx);
 
   if (gps.geom_box_present_flag) {
     int geomBoxLog2Scale = gbh.geomBoxLog2Scale(gps);
@@ -404,7 +408,10 @@ write(
 
 GeometryBrickHeader
 parseGbh(
-  const GeometryParameterSet& gps, const PayloadBuffer& buf, int* bytesRead)
+  const SequenceParameterSet& sps,
+  const GeometryParameterSet& gps,
+  const PayloadBuffer& buf,
+  int* bytesRead)
 {
   GeometryBrickHeader gbh;
   assert(buf.type == PayloadType::kGeometryBrick);
@@ -413,6 +420,7 @@ parseGbh(
   bs.readUe(&gbh.geom_geom_parameter_set_id);
   bs.readUe(&gbh.geom_tile_id);
   bs.readUe(&gbh.geom_slice_id);
+  bs.readUn(sps.log2_max_frame_idx, &gbh.frame_idx);
 
   if (gps.geom_box_present_flag) {
     if (gps.geom_box_log2_scale_present_flag)
@@ -440,6 +448,25 @@ parseGbh(
   return gbh;
 }
 
+//----------------------------------------------------------------------------
+
+GeometryBrickHeader
+parseGbhIds(const PayloadBuffer& buf)
+{
+  GeometryBrickHeader gbh;
+  assert(buf.type == PayloadType::kGeometryBrick);
+  auto bs = makeBitReader(buf.begin(), buf.end());
+
+  bs.readUe(&gbh.geom_geom_parameter_set_id);
+  bs.readUe(&gbh.geom_tile_id);
+  bs.readUe(&gbh.geom_slice_id);
+
+  /* NB: this function only decodes ids at the start of the header. */
+  /* NB: do not attempt to parse any further */
+
+  return gbh;
+}
+
 //============================================================================
 
 void

tmc3/io_hls.h

@@ -55,6 +55,7 @@ TileInventory parseTileInventory(const PayloadBuffer& buf);
 //----------------------------------------------------------------------------
 
 void write(
+  const SequenceParameterSet& sps,
   const GeometryParameterSet& gps,
   const GeometryBrickHeader& gbh,
   PayloadBuffer* buf);
@@ -65,11 +66,20 @@ void write(
   PayloadBuffer* buf);
 
 GeometryBrickHeader parseGbh(
-  const GeometryParameterSet& gps, const PayloadBuffer& buf, int* bytesRead);
+  const SequenceParameterSet& sps,
+  const GeometryParameterSet& gps,
+  const PayloadBuffer& buf,
+  int* bytesRead);
 
 AttributeBrickHeader parseAbh(
   const AttributeParameterSet& aps, const PayloadBuffer& buf, int* bytesRead);
 
+/**
+ * Parse @buf, decoding only the parameter set, slice, tile.
+ * NB: the returned header is intentionally incomplete.
+ */
+GeometryBrickHeader parseGbhIds(const PayloadBuffer& buf);
+
 /**
  * Parse @buf, decoding only the parameter set and slice ids.
  * NB: the returned header is intentionally incomplete.