tools/m44813: add ply-merge to merge/split point clouds

The ply-merge tool combines point clouds from multiple ply files into a
single output with an extra per-attribute frameindex property that
identifies which input frame each point belongs to.  The tool is also
able to reverse the process and split a merged point cloud into
individual frames.

README.tools.md

+ply-merge: A tool to merge/split ply frames
+===========================================
+
+The ply-merge tool combines point clouds from multiple ply files into a
+single output with an extra per-attribute frameindex property that
+identifies which input frame each point belongs to.  The tool is also
+able to reverse the process and split a merged point cloud into
+individual frames.
+
+Merge operation
+---------------
+From a sequence of input ply files, the merge mode repeatedly takes
+`groupSize` frames, merges their contents and produces a single output
+file with an additional frameindex property.
+
+Split operation
+---------------
+From a sequence of input ply files, and for each value of the frameindex
+property, the split mode extracts all points with the same frameindex.
+
+
+Options
+-------
+
+### `--help`
+Print a list of available command line (and configuration file) options
+along with their default values and exit.
+
+
+### `--config=FILE`, `-c`
+Specifies a configuration file to be immediately loaded.
+
+
+### `--mode=MODE`
+Selects the mode of operation.
+
+  | Value | Description                                |
+  |:-----:| ------------------------------------------ |
+  | merge | Combines multiple input files into outputs |
+  | split | Splits input files into multiple outputs   |
+
+
+### `--outputBinaryPly=0|1`
+Sets the output format of PLY files (Binary=1, ASCII=0).  Reading and
+writing binary PLY files is more efficient than the ASCII variant,
+but are less suited to simple scripts and direct human inspection.
+
+If outputting non-integer point co-ordinates (eg, due to the output
+geometry scaling), the precision of the binary and ASCII versions are
+not identical.
+
+
+### `--srcPath=FILESPEC`, `--outPath=FILESPEC`
+Specify the input and output ply file names.  The tool respectively
+replaces any instance of a '%d' printf format directive with the
+current input/output frame numbers.
+
+
+### `--firstFrameNum=INT-VALUE`, `--firstOutputFrameNum=INT-VALUE`
+The initial frame number of the respective input/output sequences.
+
+
+### `--frameCount=INT-VALUE`
+The number of input frames to process
+
+
+### `--groupSize=INT-VALUE`
+(Merge mode only)
+The number of input frames to merge into each output frame.
+
+
+Example usage
+-------------
+
+### Merging
+
+```console
+$ build/tmc3/ply-merge --mode=merge \
+    --srcPath=path/to/Ford_01_q_1mm/Ford_01_vox1mm-%.04d.ply \
+    --outPath=ford-01-vox1mm-merge8f-%.04d.ply \
+    --firstFrameNum=100 \
+    --frameCount=32
+MPEG PCC ply merge/split tool from Test Model C13
+    help                : 0
+    config              : ...
+    mode                : merge
+    srcPath             : "path/to/Ford_01_q_1mm/Ford_01_vox1mm-%.04d.ply"
+    outPath             : "ford-01-vox1mm-merge8f-%.04d.ply"
+    outputBinaryPly     : 0
+    firstFrameNum       : 100
+    firstOutputFrameNum : 0
+    frameCount          : 32
+    groupSize           : 8
+ford-01-vox1mm-merge8f-0000.ply
+ford-01-vox1mm-merge8f-0001.ply
+ford-01-vox1mm-merge8f-0002.ply
+ford-01-vox1mm-merge8f-0003.ply
+```
+
+### Splitting
+
+```console
+$ build/tmc3/ply-merge --mode=split \
+    --srcPath=ford-01-vox1mm-merge8f-%.04d.ply \
+    --outPath=split-Ford_01_vox1mm-%.04d.ply \
+    --firstOutputFrameNum=100 \
+    --frameCount=4
+```
+    help                : 0
+    config              : ...
+    mode                : split
+    srcPath             : "ford-01-vox1mm-merge8f-%.04d.ply"
+    outPath             : "split-Ford_01_vox1mm-%.04d.ply"
+    outputBinaryPly     : 0
+    firstFrameNum       : 0
+    firstOutputFrameNum : 100
+    frameCount          : 32
+    groupSize           : 8
+split-Ford_01_vox1mm-0100.ply
+split-Ford_01_vox1mm-0101.ply
+split-Ford_01_vox1mm-0102.ply
+...
+split-Ford_01_vox1mm-0131.ply
+```

tmc3/CMakeLists.txt

@@ -147,4 +147,12 @@ add_executable (tmc3
 )
 add_dependencies(tmc3 genversion)
 
+add_executable (ply-merge EXCLUDE_FROM_ALL
+  "../tools/ply-merge.cpp"
+  "misc.cpp"
+  "../dependencies/program-options-lite/program_options_lite.cpp"
+  ${VERSION_FILE}
+)
+add_dependencies(ply-merge genversion)
+
 install (TARGETS tmc3 DESTINATION bin)

tmc3/PCCMisc.h

@@ -60,6 +60,12 @@ PCCSystemEndianness()
                                                  : PCC_BIG_ENDIAN;
 }
 
+//---------------------------------------------------------------------------
+// Replace any occurence of %d with formatted number.  The %d format
+// specifier may use the formatting conventions of snprintf().
+
+std::string expandNum(const std::string& src, int num);
+
 //---------------------------------------------------------------------------
 // Population count -- return the number of bits set in @x.
 //

tmc3/PCCPointSet.h

@@ -231,6 +231,7 @@ public:
   {
     withColors = false;
     withReflectances = false;
+    withFrameIndex = false;
   }
   PCCPointSet3(const PCCPointSet3&) = default;
   PCCPointSet3& operator=(const PCCPointSet3& rhs) = default;
@@ -242,8 +243,10 @@ public:
     swap(positions, other.positions);
     swap(colors, other.colors);
     swap(reflectances, other.reflectances);
+    swap(frameidx, other.frameidx);
     swap(withColors, other.withColors);
     swap(withReflectances, other.withReflectances);
+    swap(withFrameIndex, other.withFrameIndex);
   }
 
   PCCPoint3D operator[](const size_t index) const
@@ -299,6 +302,36 @@ public:
     reflectances.resize(0);
   }
 
+  uint8_t getFrameIndex(const size_t index) const
+  {
+    assert(index < frameidx.size() && withFrameIndex);
+    return frameidx[index];
+  }
+
+  uint8_t& getFrameIndex(const size_t index)
+  {
+    assert(index < frameidx.size() && withFrameIndex);
+    return frameidx[index];
+  }
+
+  void setFrameIndex(const size_t index, const uint8_t frameindex)
+  {
+    assert(index < frameidx.size() && withFrameIndex);
+    frameidx[index] = frameindex;
+  }
+
+  bool hasFrameIndex() const { return withFrameIndex; }
+  void addFrameIndex()
+  {
+    withFrameIndex = true;
+    resize(getPointCount());
+  }
+  void removeFrameIndex()
+  {
+    withFrameIndex = false;
+    frameidx.resize(0);
+  }
+
   bool hasColors() const { return withColors; }
   void addColors()
   {
@@ -334,6 +367,9 @@ public:
     if (hasReflectances()) {
       reflectances.resize(size);
     }
+    if (hasFrameIndex()) {
+      frameidx.resize(size);
+    }
   }
   void reserve(const size_t size)
   {
@@ -344,12 +380,16 @@ public:
     if (hasReflectances()) {
       reflectances.reserve(size);
     }
+    if (hasFrameIndex()) {
+      frameidx.reserve(size);
+    }
   }
   void clear()
   {
     positions.clear();
     colors.clear();
     reflectances.clear();
+    frameidx.clear();
   }
 
   void append(const PCCPointSet3& src)
@@ -443,6 +483,7 @@ public:
     if (!fout.is_open()) {
       return false;
     }
+
     const size_t pointCount = getPointCount();
     fout << "ply" << std::endl;
 
@@ -475,6 +516,9 @@ public:
     if (hasReflectances()) {
       fout << "property uint16 refc" << std::endl;
     }
+    if (hasFrameIndex()) {
+      fout << "property uint8 frameindex" << std::endl;
+    }
     fout << "element face 0" << std::endl;
     fout << "property list uint8 int32 vertex_index" << std::endl;
     fout << "end_header" << std::endl;
@@ -493,6 +537,9 @@ public:
         if (hasReflectances()) {
           fout << " " << static_cast<int>(getReflectance(i));
         }
+        if (hasFrameIndex()) {
+          fout << " " << static_cast<int>(getFrameIndex(i));
+        }
         fout << std::endl;
       }
     } else {
@@ -513,6 +560,10 @@ public:
           fout.write(
             reinterpret_cast<const char*>(&reflectance), sizeof(uint16_t));
         }
+        if (hasFrameIndex()) {
+          const uint16_t& findex = getFrameIndex(i);
+          fout.write(reinterpret_cast<const char*>(&findex), sizeof(uint16_t));
+        }
       }
     }
     fout.close();
@@ -645,6 +696,10 @@ public:
     size_t indexG = PCC_UNDEFINED_INDEX;
     size_t indexB = PCC_UNDEFINED_INDEX;
     size_t indexReflectance = PCC_UNDEFINED_INDEX;
+    size_t indexFrame = PCC_UNDEFINED_INDEX;
+    size_t indexNX = PCC_UNDEFINED_INDEX;
+    size_t indexNY = PCC_UNDEFINED_INDEX;
+    size_t indexNZ = PCC_UNDEFINED_INDEX;
     const size_t attributeCount = attributesInfo.size();
     for (size_t a = 0; a < attributeCount; ++a) {
       const auto& attributeInfo = attributesInfo[a];
@@ -672,6 +727,21 @@ public:
         (attributeInfo.name == "reflectance" || attributeInfo.name == "refc")
         && attributeInfo.byteCount <= 2) {
         indexReflectance = a;
+      } else if (
+        attributeInfo.name == "frameindex" && attributeInfo.byteCount <= 2) {
+        indexFrame = a;
+      } else if (
+        attributeInfo.name == "nx"
+        && (attributeInfo.byteCount == 8 || attributeInfo.byteCount == 4)) {
+        indexNX = a;
+      } else if (
+        attributeInfo.name == "ny"
+        && (attributeInfo.byteCount == 8 || attributeInfo.byteCount == 4)) {
+        indexNY = a;
+      } else if (
+        attributeInfo.name == "nz"
+        && (attributeInfo.byteCount == 8 || attributeInfo.byteCount == 4)) {
+        indexNZ = a;
       }
     }
     if (
@@ -683,6 +753,7 @@ public:
     withColors = indexR != PCC_UNDEFINED_INDEX && indexG != PCC_UNDEFINED_INDEX
       && indexB != PCC_UNDEFINED_INDEX;
     withReflectances = indexReflectance != PCC_UNDEFINED_INDEX;
+    withFrameIndex = indexFrame != PCC_UNDEFINED_INDEX;
     resize(pointCount);
     if (isAscii) {
       size_t pointCounter = 0;
@@ -709,6 +780,9 @@ public:
           reflectances[pointCounter] =
             uint16_t(atoi(tokens[indexReflectance].c_str()));
         }
+        if (hasFrameIndex()) {
+          frameidx[pointCounter] = uint8_t(atoi(tokens[indexFrame].c_str()));
+        }
         ++pointCounter;
       }
     } else {
@@ -766,6 +840,15 @@ public:
               ifs.read(
                 reinterpret_cast<char*>(&reflectance), sizeof(uint16_t));
             }
+          } else if (a == indexFrame && attributeInfo.byteCount <= 2) {
+            if (attributeInfo.byteCount == 1) {
+              auto& findex = frameidx[pointCounter];
+              ifs.read(reinterpret_cast<char*>(&findex), sizeof(uint8_t));
+            } else {
+              uint16_t findex;
+              ifs.read(reinterpret_cast<char*>(&findex), sizeof(uint16_t));
+              frameidx[pointCounter] = uint8_t(findex);
+            }
           } else {
             char buffer[128];
             ifs.read(buffer, attributeInfo.byteCount);
@@ -815,8 +898,10 @@ private:
   std::vector<PCCPoint3D> positions;
   std::vector<PCCColor3B> colors;
   std::vector<uint16_t> reflectances;
+  std::vector<uint8_t> frameidx;
   bool withColors;
   bool withReflectances;
+  bool withFrameIndex;
 };
 
 //===========================================================================

tmc3/misc.cpp

+/* The copyright in this software is being made available under the BSD
+ * Licence, included below.  This software may be subject to other third
+ * party and contributor rights, including patent rights, and no such
+ * rights are granted under this licence.
+ *
+ * Copyright (c) 2018-2019, ISO/IEC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the ISO/IEC nor the names of its contributors
+ *   may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "PCCMisc.h"
+
+#include <cstdint>
+#include <cstdio>
+#include <string>
+
+namespace pcc {
+
+//============================================================================
+
+std::string
+expandNum(const std::string& src, int num)
+{
+  int idx = 0;
+  auto nextChar = [&]() { return idx + 1 < src.size() ? src[idx + 1] : '\0'; };
+
+  std::string out;
+  out.reserve(src.size());
+
+  // Search for each occurrence of a %d format pattern,
+  //  - validate it, and pass off to snprintf for formatting
+  for (; nextChar() != '\0'; idx++) {
+    // Find start of pattern:
+    int prev = idx;
+    int start = idx = src.find('%', prev);
+
+    // copy intermediate bytes
+    if (start == std::string::npos) {
+      out.append(src, prev, std::string::npos);
+      break;
+    }
+    out.append(src, prev, start - prev);
+
+    char c;
+    while ((c = nextChar()) && (c == '#' || c == '0' || c == ' '))
+      idx++; /* flags */
+
+    if ((c = nextChar()) && (c >= '1' && c <= '9')) {
+      idx++; /* width[0] */
+
+      while ((c = nextChar()) && (c >= '0' && c <= '9'))
+        idx++; /* width[>0] */
+    }
+
+    if ((c = nextChar()) && (c == '.')) {
+      idx++; /* precision[0] */
+
+      if ((c = nextChar()) && (c == '-'))
+        idx++; /* precision[1] */
+
+      while ((c = nextChar()) && (c >= '0' && c <= '9'))
+        idx++; /* precision[>=1] */
+    }
+
+    c = nextChar();
+    // Permit a %% escape to handle a literal %d
+    if (c == '%' && idx == start) {
+      out.push_back('%');
+      idx++;
+    } else if (c == 'd' && idx - start < 30) {
+      char fmt[32];
+      char buf[32];
+      int fmtlen = src.copy(fmt, idx - start + 2, start);
+      fmt[fmtlen] = '\0';
+      int len = snprintf(buf, 32, fmt, num);
+      if (len < 32) {
+        out.append(buf);
+      } else {
+        out.append(fmt);
+      }
+      idx++;
+    } else {
+      out.append(src, start, 2);
+      idx++;
+    }
+  }
+
+  return out;
+}
+
+//============================================================================
+
+}  // namespace pcc

tools/ply-merge.cpp

+/* The copyright in this software is being made available under the BSD
+ * Licence, included below.  This software may be subject to other third
+ * party and contributor rights, including patent rights, and no such
+ * rights are granted under this licence.
+ *
+ * Copyright (c) 2019, ISO/IEC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the ISO/IEC nor the names of its contributors
+ *   may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <climits>
+#include <exception>
+
+#include "PCCMisc.h"
+#include "PCCPointSet.h"
+#include "program_options_lite.h"
+#include "version.h"
+
+using namespace std;
+using namespace pcc;
+
+//============================================================================
+
+struct Options;
+
+bool parseParameters(int argc, char* argv[], Options& opts);
+void runMerge(const Options& opts);
+void runSplit(const Options& opts);
+
+//============================================================================
+
+struct Options {
+  enum class Mode
+  {
+    Merge,
+    Split
+  } mode;
+
+  // output mode for ply writing (binary or ascii)
+  bool outputBinaryPly;
+
+  // path (using %d to indicate frame number) of input files
+  std::string srcPath;
+
+  // path (using %d to indicate frame number) of output files
+  std::string outPath;
+
+  // number of src frames to process;
+  int frameCount;
+
+  // first frame number in input sequence
+  int firstFrameNum;
+
+  // first frame number in output sequence
+  int firstOutputFrameNum;
+
+  // number of src frames per merged output frame
+  int groupSize;
+};
+
+//============================================================================
+
+static std::istream&
+operator>>(std::istream& in, Options::Mode& val)
+{
+  std::string word;
+  in >> word;
+  if (word == "merge")
+    val = Options::Mode::Merge;
+  else if (word == "split")
+    val = Options::Mode::Split;
+  else
+    throw std::exception();
+  return in;
+}
+
+//----------------------------------------------------------------------------
+
+static std::ostream&
+operator<<(std::ostream& out, const Options::Mode& val)
+{
+  switch (val) {
+  case Options::Mode::Merge: out << "merge"; break;
+  case Options::Mode::Split: out << "split"; break;
+  }
+  return out;
+}
+
+//============================================================================
+
+int
+main(int argc, char* argv[])
+{
+  cout << "MPEG PCC ply merge/split tool from Test Model C13" << endl;
+
+  Options opts;
+  if (!parseParameters(argc, argv, opts)) {
+    return 1;
+  }
+
+  try {
+    switch (opts.mode) {
+    case Options::Mode::Merge: runMerge(opts); break;
+    case Options::Mode::Split: runSplit(opts); break;
+    }
+  }
+  catch (const exception& e) {
+    cerr << "Error:" << e.what() << endl;
+  }
+
+  return 0;
+}
+
+//---------------------------------------------------------------------------
+// :: Command line / config parsing
+
+bool
+parseParameters(int argc, char* argv[], Options& params)
+{
+  namespace po = df::program_options_lite;
+  bool print_help = false;
+
+  /* clang-format off */
+  // The definition of the program/config options, along with default values.
+  //
+  // NB: when updating the following tables:
+  //      (a) please keep to 80-columns for easier reading at a glance,
+  //      (b) do not vertically align values -- it breaks quickly
+  //
+  po::Options opts;
+  opts.addOptions()
+  ("help", print_help, false, "this help text")
+  ("config,c", po::parseConfigFile, "configuration file name")
+
+  ("mode", params.mode, Options::Mode::Merge,
+    "The combine/split mode:\n"
+    "  split: extract n ply files from one\n"
+    "  merge: combine n ply files into one")
+
+  ("srcPath",
+    params.srcPath, {},
+    "Input pointcloud file path")