使用 C++ 构建图

C++ 图构建器是一款强大的工具,适用于:

  • 构建复杂图
  • 将图表参数化(例如,在 InferenceCalculator 上设置委托、 启用/停用图表的某些部分)
  • 删除重复图表(例如,不是 pbtxt 中的 CPU 和 GPU 专用图表) 您只需使用一个代码来构建所需图表, )
  • 支持可选的图表输入/输出
  • 为每个平台自定义图表

基本用法

我们来看看如何将 C++ 图构建器用于简单图:

# Graph inputs.
input_stream: "input_tensors"
input_side_packet: "model"

# Graph outputs.
output_stream: "output_tensors"

node {
  calculator: "InferenceCalculator"
  input_stream: "TENSORS:input_tensors"
  input_side_packet: "MODEL:model"
  output_stream: "TENSORS:output_tensors"
  options: {
    [drishti.InferenceCalculatorOptions.ext] {
      # Requesting GPU delegate.
      delegate { gpu {} }
    }
  }
}

用于构建上述 CalculatorGraphConfig 的函数可能如下所示:

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Graph inputs.
  Stream<std::vector<Tensor>> input_tensors =
      graph.In(0).SetName("input_tensors").Cast<std::vector<Tensor>>();
  SidePacket<TfLiteModelPtr> model =
      graph.SideIn(0).SetName("model").Cast<TfLiteModelPtr>();

  auto& inference_node = graph.AddNode("InferenceCalculator");
  auto& inference_opts =
      inference_node.GetOptions<InferenceCalculatorOptions>();
  // Requesting GPU delegate.
  inference_opts.mutable_delegate()->mutable_gpu();
  input_tensors.ConnectTo(inference_node.In("TENSORS"));
  model.ConnectTo(inference_node.SideIn("MODEL"));
  Stream<std::vector<Tensor>> output_tensors =
      inference_node.Out("TENSORS").Cast<std::vector<Tensor>>();

  // Graph outputs.
  output_tensors.SetName("output_tensors").ConnectTo(graph.Out(0));

  // Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
  return graph.GetConfig();
}

简短摘要:

  • 使用 Graph::In/SideIn 获取图表输入作为 Stream/SidePacket
  • 使用 Node::Out/SideOutStream/SidePacket 形式获取节点输出
  • 使用 Stream/SidePacket::ConnectTo 将数据流和侧边数据包连接到 节点输入 (Node::In/SideIn) 和图表输出 (Graph::Out/SideOut)
    • 有一个“快捷方式”运算符 >>,您可以用来代替 ConnectTo 函数(例如 x >> node.In("IN"))。
  • Stream/SidePacket::Cast 用于投放 AnyType 的串流或侧包 (例如 Stream<AnyType> in = graph.In(0);)更改为特定类型
    • 使用实际类型而不是 AnyType 有助于您找到更好的 释放图表构建器功能并改进图表 可读性。

高级用法

实用函数

让我们将推理构造代码提取到一个专用的实用函数中, 关于可读性和代码重用的帮助:

// Updates graph to run inference.
Stream<std::vector<Tensor>> RunInference(
    Stream<std::vector<Tensor>> tensors, SidePacket<TfLiteModelPtr> model,
    const InferenceCalculatorOptions::Delegate& delegate, Graph& graph) {
  auto& inference_node = graph.AddNode("InferenceCalculator");
  auto& inference_opts =
      inference_node.GetOptions<InferenceCalculatorOptions>();
  *inference_opts.mutable_delegate() = delegate;
  tensors.ConnectTo(inference_node.In("TENSORS"));
  model.ConnectTo(inference_node.SideIn("MODEL"));
  return inference_node.Out("TENSORS").Cast<std::vector<Tensor>>();
}

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Graph inputs.
  Stream<std::vector<Tensor>> input_tensors =
      graph.In(0).SetName("input_tensors").Cast<std::vector<Tensor>>();
  SidePacket<TfLiteModelPtr> model =
      graph.SideIn(0).SetName("model").Cast<TfLiteModelPtr>();

  InferenceCalculatorOptions::Delegate delegate;
  delegate.mutable_gpu();
  Stream<std::vector<Tensor>> output_tensors =
      RunInference(input_tensors, model, delegate, graph);

  // Graph outputs.
  output_tensors.SetName("output_tensors").ConnectTo(graph.Out(0));

  return graph.GetConfig();
}

因此,RunInference 提供了一个清晰的接口,说明 输入/输出及其类型。

可轻松重复使用,例如如果您想运行额外的代码 模型推断:

  // Run first inference.
  Stream<std::vector<Tensor>> output_tensors =
      RunInference(input_tensors, model, delegate, graph);
  // Run second inference on the output of the first one.
  Stream<std::vector<Tensor>> extra_output_tensors =
      RunInference(output_tensors, extra_model, delegate, graph);

无需使用重复的名称和标签(InferenceCalculatorTENSORSMODEL),或者在此处和任何位置引入专用常量,即 详细信息已本地化为 RunInference 函数。

实用程序类

当然,这不仅仅与函数有关,在某些情况下, 介绍实用程序类,这些实用程序类可以帮助您制作图构造代码 更易于读取且更不易出错。

MediaPipe 提供 PassThroughCalculator 计算器,可轻松通过 通过其输入:

input_stream: "float_value"
input_stream: "int_value"
input_stream: "bool_value"

output_stream: "passed_float_value"
output_stream: "passed_int_value"
output_stream: "passed_bool_value"

node {
  calculator: "PassThroughCalculator"
  input_stream: "float_value"
  input_stream: "int_value"
  input_stream: "bool_value"
  # The order must be the same as for inputs (or you can use explicit indexes)
  output_stream: "passed_float_value"
  output_stream: "passed_int_value"
  output_stream: "passed_bool_value"
}

我们来看一下用于创建上述图表的简单 C++ 构建代码:

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Graph inputs.
  Stream<float> float_value = graph.In(0).SetName("float_value").Cast<float>();
  Stream<int> int_value = graph.In(1).SetName("int_value").Cast<int>();
  Stream<bool> bool_value = graph.In(2).SetName("bool_value").Cast<bool>();

  auto& pass_node = graph.AddNode("PassThroughCalculator");
  float_value.ConnectTo(pass_node.In("")[0]);
  int_value.ConnectTo(pass_node.In("")[1]);
  bool_value.ConnectTo(pass_node.In("")[2]);
  Stream<float> passed_float_value = pass_node.Out("")[0].Cast<float>();
  Stream<int> passed_int_value = pass_node.Out("")[1].Cast<int>();
  Stream<bool> passed_bool_value = pass_node.Out("")[2].Cast<bool>();

  // Graph outputs.
  passed_float_value.SetName("passed_float_value").ConnectTo(graph.Out(0));
  passed_int_value.SetName("passed_int_value").ConnectTo(graph.Out(1));
  passed_bool_value.SetName("passed_bool_value").ConnectTo(graph.Out(2));

  // Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
  return graph.GetConfig();
}

虽然 pbtxt 表示法可能容易出错(当我们有许多输入要传递时 C++ 代码更糟糕了:重复的空标记和 Cast 调用。让我们 看看我们可以通过引入 PassThroughNodeBuilder 来做得更好:

class PassThroughNodeBuilder {
 public:
  explicit PassThroughNodeBuilder(Graph& graph)
      : node_(graph.AddNode("PassThroughCalculator")) {}

  template <typename T>
  Stream<T> PassThrough(Stream<T> stream) {
    stream.ConnectTo(node_.In(index_));
    return node_.Out(index_++).Cast<T>();
  }

 private:
  int index_ = 0;
  GenericNode& node_;
};

现在,图表构建代码可能如下所示:

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Graph inputs.
  Stream<float> float_value = graph.In(0).SetName("float_value").Cast<float>();
  Stream<int> int_value = graph.In(1).SetName("int_value").Cast<int>();
  Stream<bool> bool_value = graph.In(2).SetName("bool_value").Cast<bool>();

  PassThroughNodeBuilder pass_node_builder(graph);
  Stream<float> passed_float_value = pass_node_builder.PassThrough(float_value);
  Stream<int> passed_int_value = pass_node_builder.PassThrough(int_value);
  Stream<bool> passed_bool_value = pass_node_builder.PassThrough(bool_value);

  // Graph outputs.
  passed_float_value.SetName("passed_float_value").ConnectTo(graph.Out(0));
  passed_int_value.SetName("passed_int_value").ConnectTo(graph.Out(1));
  passed_bool_value.SetName("passed_bool_value").ConnectTo(graph.Out(2));

  // Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
  return graph.GetConfig();
}

现在,卡券中的顺序或索引不能不正确 从 PassThrough 中猜测 Cast 的类型,以节省一些内容 输入。

正确做法和错误做法

如果可能,在一开始就定义图输入

在下面的代码中:

  • 您可能很难猜测图表中有多少输入。
  • 总体上容易出错,以后难以维护(例如, 索引是否正确?名称?如果某些输入被移除或设为可选,该怎么办? 等)。
  • RunSomething的重复使用会受到限制,因为其他图表可能具有不同的 输入

错误做法 - 错误代码示例。

Stream<D> RunSomething(Stream<A> a, Stream<B> b, Graph& graph) {
  Stream<C> c = graph.In(2).SetName("c").Cast<C>();  // Bad.
  // ...
}

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  Stream<A> a = graph.In(0).SetName("a").Cast<A>();
  // 10/100/N lines of code.
  Stream<B> b = graph.In(1).SetName("b").Cast<B>()  // Bad.
  Stream<D> d = RunSomething(a, b, graph);
  // ...

  return graph.GetConfig();
}

请改为在图表构建器的开头定义图表输入:

正确做法 - 优质代码示例。

Stream<D> RunSomething(Stream<A> a, Stream<B> b, Stream<C> c, Graph& graph) {
  // ...
}

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Inputs.
  Stream<A> a = graph.In(0).SetName("a").Cast<A>();
  Stream<B> b = graph.In(1).SetName("b").Cast<B>();
  Stream<C> c = graph.In(2).SetName("c").Cast<C>();

  // 10/100/N lines of code.
  Stream<D> d = RunSomething(a, b, c, graph);
  // ...

  return graph.GetConfig();
}

如果您有非同一种输入流或侧边数据包,请使用 std::optional 并放在最前面:

正确做法 - 优质代码示例。

std::optional<Stream<A>> a;
if (needs_a) {
  a = graph.In(0).SetName(a).Cast<A>();
}

在最后定义图输出

在下面的代码中:

  • 您可能很难猜测图表中有多少输出。
  • 总体上容易出错,以后难以维护(例如, 索引是否正确?名称?如果某些输出接口被移除或设为可选,该怎么办? 等)。
  • 由于其他图可能会有不同的输出,因此 RunSomething 的重复使用会受到限制

错误做法 - 错误代码示例。

void RunSomething(Stream<Input> input, Graph& graph) {
  // ...
  node.Out("OUTPUT_F")
      .SetName("output_f").ConnectTo(graph.Out(2));  // Bad.
}

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // 10/100/N lines of code.
  node.Out("OUTPUT_D")
      .SetName("output_d").ConnectTo(graph.Out(0));  // Bad.
  // 10/100/N lines of code.
  node.Out("OUTPUT_E")
      .SetName("output_e").ConnectTo(graph.Out(1));  // Bad.
  // 10/100/N lines of code.
  RunSomething(input, graph);
  // ...

  return graph.GetConfig();
}

请改为在图表构建器的末尾定义图表输出:

正确做法 - 优质代码示例。

Stream<F> RunSomething(Stream<Input> input, Graph& graph) {
  // ...
  return node.Out("OUTPUT_F").Cast<F>();
}

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // 10/100/N lines of code.
  Stream<D> d = node.Out("OUTPUT_D").Cast<D>();
  // 10/100/N lines of code.
  Stream<E> e = node.Out("OUTPUT_E").Cast<E>();
  // 10/100/N lines of code.
  Stream<F> f = RunSomething(input, graph);
  // ...

  // Outputs.
  d.SetName("output_d").ConnectTo(graph.Out(0));
  e.SetName("output_e").ConnectTo(graph.Out(1));
  f.SetName("output_f").ConnectTo(graph.Out(2));

  return graph.GetConfig();
}

使节点彼此分离

在 MediaPipe 中,数据包流和侧边数据包与处理数据一样有意义 节点。所有节点输入要求和输出产品都 而且消耗的数据包和数据包数量 生成的内容。

错误做法 - 错误代码示例。

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Inputs.
  Stream<A> a = graph.In(0).Cast<A>();

  auto& node1 = graph.AddNode("Calculator1");
  a.ConnectTo(node1.In("INPUT"));

  auto& node2 = graph.AddNode("Calculator2");
  node1.Out("OUTPUT").ConnectTo(node2.In("INPUT"));  // Bad.

  auto& node3 = graph.AddNode("Calculator3");
  node1.Out("OUTPUT").ConnectTo(node3.In("INPUT_B"));  // Bad.
  node2.Out("OUTPUT").ConnectTo(node3.In("INPUT_C"));  // Bad.

  auto& node4 = graph.AddNode("Calculator4");
  node1.Out("OUTPUT").ConnectTo(node4.In("INPUT_B"));  // Bad.
  node2.Out("OUTPUT").ConnectTo(node4.In("INPUT_C"));  // Bad.
  node3.Out("OUTPUT").ConnectTo(node4.In("INPUT_D"));  // Bad.

  // Outputs.
  node1.Out("OUTPUT").SetName("b").ConnectTo(graph.Out(0));  // Bad.
  node2.Out("OUTPUT").SetName("c").ConnectTo(graph.Out(1));  // Bad.
  node3.Out("OUTPUT").SetName("d").ConnectTo(graph.Out(2));  // Bad.
  node4.Out("OUTPUT").SetName("e").ConnectTo(graph.Out(3));  // Bad.

  return graph.GetConfig();
}

在上述代码中:

  • 节点彼此耦合,例如node4 知道其输入的位置 (node1node2node3),这会使重构变得非常复杂, 维护和代码重复使用
    • 这种使用模式是从 proto 表示法降级而来,其中节点 默认是分离的
  • node#.Out("OUTPUT") 调用会重复,并且可读性会受到影响 可以使用更简洁的名称,并提供实际类型。

因此,要解决上述问题,您可以编写以下图表构造代码:

正确做法 - 优质代码示例。

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Inputs.
  Stream<A> a = graph.In(0).Cast<A>();

  // `node1` usage is limited to 3 lines below.
  auto& node1 = graph.AddNode("Calculator1");
  a.ConnectTo(node1.In("INPUT"));
  Stream<B> b = node1.Out("OUTPUT").Cast<B>();

  // `node2` usage is limited to 3 lines below.
  auto& node2 = graph.AddNode("Calculator2");
  b.ConnectTo(node2.In("INPUT"));
  Stream<C> c = node2.Out("OUTPUT").Cast<C>();

  // `node3` usage is limited to 4 lines below.
  auto& node3 = graph.AddNode("Calculator3");
  b.ConnectTo(node3.In("INPUT_B"));
  c.ConnectTo(node3.In("INPUT_C"));
  Stream<D> d = node3.Out("OUTPUT").Cast<D>();

  // `node4` usage is limited to 5 lines below.
  auto& node4 = graph.AddNode("Calculator4");
  b.ConnectTo(node4.In("INPUT_B"));
  c.ConnectTo(node4.In("INPUT_C"));
  d.ConnectTo(node4.In("INPUT_D"));
  Stream<E> e = node4.Out("OUTPUT").Cast<E>();

  // Outputs.
  b.SetName("b").ConnectTo(graph.Out(0));
  c.SetName("c").ConnectTo(graph.Out(1));
  d.SetName("d").ConnectTo(graph.Out(2));
  e.SetName("e").ConnectTo(graph.Out(3));

  return graph.GetConfig();
}

现在,您可以根据需要轻松移除 node1 并将 b 设为图表输入, 需要对 node2node3node4 进行更新(与 proto 表示法中的相同) 因为它们彼此分离。

总体而言,上述代码与 proto 图进行了更密切的复制:

input_stream: "a"

node {
  calculator: "Calculator1"
  input_stream: "INPUT:a"
  output_stream: "OUTPUT:b"
}

node {
  calculator: "Calculator2"
  input_stream: "INPUT:b"
  output_stream: "OUTPUT:C"
}

node {
  calculator: "Calculator3"
  input_stream: "INPUT_B:b"
  input_stream: "INPUT_C:c"
  output_stream: "OUTPUT:d"
}

node {
  calculator: "Calculator4"
  input_stream: "INPUT_B:b"
  input_stream: "INPUT_C:c"
  input_stream: "INPUT_D:d"
  output_stream: "OUTPUT:e"
}

output_stream: "b"
output_stream: "c"
output_stream: "d"
output_stream: "e"

此外,现在您还可以提取实用函数,以便在其他图表中进一步重复使用:

正确做法 - 优质代码示例。

Stream<B> RunCalculator1(Stream<A> a, Graph& graph) {
  auto& node = graph.AddNode("Calculator1");
  a.ConnectTo(node.In("INPUT"));
  return node.Out("OUTPUT").Cast<B>();
}

Stream<C> RunCalculator2(Stream<B> b, Graph& graph) {
  auto& node = graph.AddNode("Calculator2");
  b.ConnectTo(node.In("INPUT"));
  return node.Out("OUTPUT").Cast<C>();
}

Stream<D> RunCalculator3(Stream<B> b, Stream<C> c, Graph& graph) {
  auto& node = graph.AddNode("Calculator3");
  b.ConnectTo(node.In("INPUT_B"));
  c.ConnectTo(node.In("INPUT_C"));
  return node.Out("OUTPUT").Cast<D>();
}

Stream<E> RunCalculator4(Stream<B> b, Stream<C> c, Stream<D> d, Graph& graph) {
  auto& node = graph.AddNode("Calculator4");
  b.ConnectTo(node.In("INPUT_B"));
  c.ConnectTo(node.In("INPUT_C"));
  d.ConnectTo(node.In("INPUT_D"));
  return node.Out("OUTPUT").Cast<E>();
}

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Inputs.
  Stream<A> a = graph.In(0).Cast<A>();

  Stream<B> b = RunCalculator1(a, graph);
  Stream<C> c = RunCalculator2(b, graph);
  Stream<D> d = RunCalculator3(b, c, graph);
  Stream<E> e = RunCalculator4(b, c, d, graph);

  // Outputs.
  b.SetName("b").ConnectTo(graph.Out(0));
  c.SetName("c").ConnectTo(graph.Out(1));
  d.SetName("d").ConnectTo(graph.Out(2));
  e.SetName("e").ConnectTo(graph.Out(3));

  return graph.GetConfig();
}

拆分节点以提高可读性

错误做法 - 错误代码示例。

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Inputs.
  Stream<A> a = graph.In(0).Cast<A>();
  auto& node1 = graph.AddNode("Calculator1");
  a.ConnectTo(node1.In("INPUT"));
  Stream<B> b = node1.Out("OUTPUT").Cast<B>();
  auto& node2 = graph.AddNode("Calculator2");
  b.ConnectTo(node2.In("INPUT"));
  Stream<C> c = node2.Out("OUTPUT").Cast<C>();
  auto& node3 = graph.AddNode("Calculator3");
  b.ConnectTo(node3.In("INPUT_B"));
  c.ConnectTo(node3.In("INPUT_C"));
  Stream<D> d = node3.Out("OUTPUT").Cast<D>();
  auto& node4 = graph.AddNode("Calculator4");
  b.ConnectTo(node4.In("INPUT_B"));
  c.ConnectTo(node4.In("INPUT_C"));
  d.ConnectTo(node4.In("INPUT_D"));
  Stream<E> e = node4.Out("OUTPUT").Cast<E>();
  // Outputs.
  b.SetName("b").ConnectTo(graph.Out(0));
  c.SetName("c").ConnectTo(graph.Out(1));
  d.SetName("d").ConnectTo(graph.Out(2));
  e.SetName("e").ConnectTo(graph.Out(3));

  return graph.GetConfig();
}

在上面的代码中,您可能很难理解每个节点从哪里开始, 结束。若要改进此功能并帮助代码读者,您可以先将空白 行:

正确做法 - 优质代码示例。

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Inputs.
  Stream<A> a = graph.In(0).Cast<A>();

  auto& node1 = graph.AddNode("Calculator1");
  a.ConnectTo(node1.In("INPUT"));
  Stream<B> b = node1.Out("OUTPUT").Cast<B>();

  auto& node2 = graph.AddNode("Calculator2");
  b.ConnectTo(node2.In("INPUT"));
  Stream<C> c = node2.Out("OUTPUT").Cast<C>();

  auto& node3 = graph.AddNode("Calculator3");
  b.ConnectTo(node3.In("INPUT_B"));
  c.ConnectTo(node3.In("INPUT_C"));
  Stream<D> d = node3.Out("OUTPUT").Cast<D>();

  auto& node4 = graph.AddNode("Calculator4");
  b.ConnectTo(node4.In("INPUT_B"));
  c.ConnectTo(node4.In("INPUT_C"));
  d.ConnectTo(node4.In("INPUT_D"));
  Stream<E> e = node4.Out("OUTPUT").Cast<E>();

  // Outputs.
  b.SetName("b").ConnectTo(graph.Out(0));
  c.SetName("c").ConnectTo(graph.Out(1));
  d.SetName("d").ConnectTo(graph.Out(2));
  e.SetName("e").ConnectTo(graph.Out(3));

  return graph.GetConfig();
}

此外,上面的表示法与 CalculatorGraphConfig proto 匹配 更好的表示方式。

如果将节点提取到实用函数中,则它们的范围限定在函数内 而且它们的起始和结束位置也很清晰,所以完全可以 具有:

正确做法 - 优质代码示例。

CalculatorGraphConfig BuildGraph() {
  Graph graph;

  // Inputs.
  Stream<A> a = graph.In(0).Cast<A>();

  Stream<B> b = RunCalculator1(a, graph);
  Stream<C> c = RunCalculator2(b, graph);
  Stream<D> d = RunCalculator3(b, c, graph);
  Stream<E> e = RunCalculator4(b, c, d, graph);

  // Outputs.
  b.SetName("b").ConnectTo(graph.Out(0));
  c.SetName("c").ConnectTo(graph.Out(1));
  d.SetName("d").ConnectTo(graph.Out(2));
  e.SetName("e").ConnectTo(graph.Out(3));

  return graph.GetConfig();
}