C++ 圖形建構工具是功能強大的工具,可用於:
- 建立複雜的圖形
- 將圖表參數化 (例如在
InferenceCalculator
上設定委派項目、啟用/停用圖表的特定部分) - 簡化圖表 (例如不必使用 pbtxt 中的 CPU 和 GPU 專屬圖表,您可以使用單一程式碼建構所需圖表並盡可能分享)
- 支援選用圖表輸入/輸出內容
- 依平台自訂圖表
基本用法
來看看如何將 C++ 圖形建構工具用於簡易圖表:
# Graph inputs.
input_stream: "input_tensors"
input_side_packet: "model"
# Graph outputs.
output_stream: "output_tensors"
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
input_side_packet: "MODEL:model"
output_stream: "TENSORS:output_tensors"
options: {
[drishti.InferenceCalculatorOptions.ext] {
# Requesting GPU delegate.
delegate { gpu {} }
}
}
}
用於建構上述 CalculatorGraphConfig
的函式可能如下所示:
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<std::vector<Tensor>> input_tensors =
graph.In(0).SetName("input_tensors").Cast<std::vector<Tensor>>();
SidePacket<TfLiteModelPtr> model =
graph.SideIn(0).SetName("model").Cast<TfLiteModelPtr>();
auto& inference_node = graph.AddNode("InferenceCalculator");
auto& inference_opts =
inference_node.GetOptions<InferenceCalculatorOptions>();
// Requesting GPU delegate.
inference_opts.mutable_delegate()->mutable_gpu();
input_tensors.ConnectTo(inference_node.In("TENSORS"));
model.ConnectTo(inference_node.SideIn("MODEL"));
Stream<std::vector<Tensor>> output_tensors =
inference_node.Out("TENSORS").Cast<std::vector<Tensor>>();
// Graph outputs.
output_tensors.SetName("output_tensors").ConnectTo(graph.Out(0));
// Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
return graph.GetConfig();
}
簡短摘要:
- 使用
Graph::In/SideIn
將圖表輸入內容做為Stream/SidePacket
- 使用
Node::Out/SideOut
取得節點輸出內容Stream/SidePacket
- 使用
Stream/SidePacket::ConnectTo
將串流和側邊封包連結至節點輸入 (Node::In/SideIn
) 和圖表輸出 (Graph::Out/SideOut
)- 有一種「捷徑」運算子
>>
,可用於取代ConnectTo
函式 (例如x >> node.In("IN")
)。
- 有一種「捷徑」運算子
Stream/SidePacket::Cast
是用來將AnyType
(例如Stream<AnyType> in = graph.In(0);
) 的串流或側封包轉換為特定類型的類型- 使用實際類型而非
AnyType
能進一步提升圖表建構工具的功能,並提高圖表可讀性。
- 使用實際類型而非
進階用法
Utility 函式
請將推論建構程式碼擷取至專屬的公用程式函式中,以協助重複使用程式碼:
// Updates graph to run inference.
Stream<std::vector<Tensor>> RunInference(
Stream<std::vector<Tensor>> tensors, SidePacket<TfLiteModelPtr> model,
const InferenceCalculatorOptions::Delegate& delegate, Graph& graph) {
auto& inference_node = graph.AddNode("InferenceCalculator");
auto& inference_opts =
inference_node.GetOptions<InferenceCalculatorOptions>();
*inference_opts.mutable_delegate() = delegate;
tensors.ConnectTo(inference_node.In("TENSORS"));
model.ConnectTo(inference_node.SideIn("MODEL"));
return inference_node.Out("TENSORS").Cast<std::vector<Tensor>>();
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<std::vector<Tensor>> input_tensors =
graph.In(0).SetName("input_tensors").Cast<std::vector<Tensor>>();
SidePacket<TfLiteModelPtr> model =
graph.SideIn(0).SetName("model").Cast<TfLiteModelPtr>();
InferenceCalculatorOptions::Delegate delegate;
delegate.mutable_gpu();
Stream<std::vector<Tensor>> output_tensors =
RunInference(input_tensors, model, delegate, graph);
// Graph outputs.
output_tensors.SetName("output_tensors").ConnectTo(graph.Out(0));
return graph.GetConfig();
}
因此,RunInference
會提供清楚的介面,說明輸入內容/輸出內容及其類型。
這些資料可以輕鬆重複使用。舉例來說,如果您想執行額外的模型推論,只要執行幾行程式碼就夠了:
// Run first inference.
Stream<std::vector<Tensor>> output_tensors =
RunInference(input_tensors, model, delegate, graph);
// Run second inference on the output of the first one.
Stream<std::vector<Tensor>> extra_output_tensors =
RunInference(output_tensors, extra_model, delegate, graph);
此外,您不需要重複的名稱和標記 (InferenceCalculator
、TENSORS
、MODEL
),或在這裡導入專屬常數,這些詳細資料都已本地化為 RunInference
函式。
:inference_calculator
公用程式類別
當然,這不僅是函式相關,在某些情況下,引進公用程式類別也有助於提升圖表建構程式碼的可讀性,也較不容易出錯。
MediaPipe 提供 PassThroughCalculator
計算機,可直接傳遞輸入內容:
input_stream: "float_value"
input_stream: "int_value"
input_stream: "bool_value"
output_stream: "passed_float_value"
output_stream: "passed_int_value"
output_stream: "passed_bool_value"
node {
calculator: "PassThroughCalculator"
input_stream: "float_value"
input_stream: "int_value"
input_stream: "bool_value"
# The order must be the same as for inputs (or you can use explicit indexes)
output_stream: "passed_float_value"
output_stream: "passed_int_value"
output_stream: "passed_bool_value"
}
我們來看看建立上述圖表的簡單 C++ 建構程式碼:
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<float> float_value = graph.In(0).SetName("float_value").Cast<float>();
Stream<int> int_value = graph.In(1).SetName("int_value").Cast<int>();
Stream<bool> bool_value = graph.In(2).SetName("bool_value").Cast<bool>();
auto& pass_node = graph.AddNode("PassThroughCalculator");
float_value.ConnectTo(pass_node.In("")[0]);
int_value.ConnectTo(pass_node.In("")[1]);
bool_value.ConnectTo(pass_node.In("")[2]);
Stream<float> passed_float_value = pass_node.Out("")[0].Cast<float>();
Stream<int> passed_int_value = pass_node.Out("")[1].Cast<int>();
Stream<bool> passed_bool_value = pass_node.Out("")[2].Cast<bool>();
// Graph outputs.
passed_float_value.SetName("passed_float_value").ConnectTo(graph.Out(0));
passed_int_value.SetName("passed_int_value").ConnectTo(graph.Out(1));
passed_bool_value.SetName("passed_bool_value").ConnectTo(graph.Out(2));
// Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
return graph.GetConfig();
}
雖然 pbtxt
表示法可能容易發生錯誤 (當我們有許多輸入內容可以傳遞時),但 C++ 程式碼看起來更糟:重複的空白標記和 Cast
呼叫。我們來看看如何導入 PassThroughNodeBuilder
,才能進一步提升成效:
class PassThroughNodeBuilder {
public:
explicit PassThroughNodeBuilder(Graph& graph)
: node_(graph.AddNode("PassThroughCalculator")) {}
template <typename T>
Stream<T> PassThrough(Stream<T> stream) {
stream.ConnectTo(node_.In(index_));
return node_.Out(index_++).Cast<T>();
}
private:
int index_ = 0;
GenericNode& node_;
};
現在,圖形建構程式碼看起來會像這樣:
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Graph inputs.
Stream<float> float_value = graph.In(0).SetName("float_value").Cast<float>();
Stream<int> int_value = graph.In(1).SetName("int_value").Cast<int>();
Stream<bool> bool_value = graph.In(2).SetName("bool_value").Cast<bool>();
PassThroughNodeBuilder pass_node_builder(graph);
Stream<float> passed_float_value = pass_node_builder.PassThrough(float_value);
Stream<int> passed_int_value = pass_node_builder.PassThrough(int_value);
Stream<bool> passed_bool_value = pass_node_builder.PassThrough(bool_value);
// Graph outputs.
passed_float_value.SetName("passed_float_value").ConnectTo(graph.Out(0));
passed_int_value.SetName("passed_int_value").ConnectTo(graph.Out(1));
passed_bool_value.SetName("passed_bool_value").ConnectTo(graph.Out(2));
// Get `CalculatorGraphConfig` to pass it into `CalculatorGraph`
return graph.GetConfig();
}
現在,您無法從建構程式碼猜測 Cast
的類型,從而儲存錯誤順序或索引,而且會猜測 PassThrough
輸入的類型來節省輸入時間。
建議做法
盡可能在一開始就定義圖表輸入內容
在下列程式碼中:
- 一般很難猜測圖表輸入多少資料,
- 整體而言,問題是否容易出錯,日後維護起來也很困難 (例如這是正確的索引、名稱?如果部分輸入遭到移除或設為選用,該怎麼辦?)。
RunSomething
重複使用受到限制,因為其他圖表的輸入內容可能不同
錯誤做法:程式碼有誤的示例。
Stream<D> RunSomething(Stream<A> a, Stream<B> b, Graph& graph) {
Stream<C> c = graph.In(2).SetName("c").Cast<C>(); // Bad.
// ...
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
Stream<A> a = graph.In(0).SetName("a").Cast<A>();
// 10/100/N lines of code.
Stream<B> b = graph.In(1).SetName("b").Cast<B>() // Bad.
Stream<D> d = RunSomething(a, b, graph);
// ...
return graph.GetConfig();
}
請改為在圖表建構工具的開頭定義圖形輸入內容:
正確:良好程式碼的範例。
Stream<D> RunSomething(Stream<A> a, Stream<B> b, Stream<C> c, Graph& graph) {
// ...
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).SetName("a").Cast<A>();
Stream<B> b = graph.In(1).SetName("b").Cast<B>();
Stream<C> c = graph.In(2).SetName("c").Cast<C>();
// 10/100/N lines of code.
Stream<D> d = RunSomething(a, b, c, graph);
// ...
return graph.GetConfig();
}
如果您有非一律定義的輸入串流或側邊封包,並將該封包放在最開頭,請使用 std::optional
:
正確:良好程式碼的範例。
std::optional<Stream<A>> a;
if (needs_a) {
a = graph.In(0).SetName(a).Cast<A>();
}
定義結尾的圖形輸出內容
在下列程式碼中:
- 因此很難猜測圖表中會顯示多少輸出內容。
- 整體而言容易出錯,日後維護起來可能很困難 (例如這是正確的索引、名稱嗎?如果部分 Outpus 遭到移除或設為選用等等)。
RunSomething
可重複使用的次數有限,因為其他圖表的輸出內容不同
錯誤做法:程式碼有誤的示例。
void RunSomething(Stream<Input> input, Graph& graph) {
// ...
node.Out("OUTPUT_F")
.SetName("output_f").ConnectTo(graph.Out(2)); // Bad.
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// 10/100/N lines of code.
node.Out("OUTPUT_D")
.SetName("output_d").ConnectTo(graph.Out(0)); // Bad.
// 10/100/N lines of code.
node.Out("OUTPUT_E")
.SetName("output_e").ConnectTo(graph.Out(1)); // Bad.
// 10/100/N lines of code.
RunSomething(input, graph);
// ...
return graph.GetConfig();
}
請改為在圖形建構工具的結尾定義圖形輸出:
正確:良好程式碼的範例。
Stream<F> RunSomething(Stream<Input> input, Graph& graph) {
// ...
return node.Out("OUTPUT_F").Cast<F>();
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// 10/100/N lines of code.
Stream<D> d = node.Out("OUTPUT_D").Cast<D>();
// 10/100/N lines of code.
Stream<E> e = node.Out("OUTPUT_E").Cast<E>();
// 10/100/N lines of code.
Stream<F> f = RunSomething(input, graph);
// ...
// Outputs.
d.SetName("output_d").ConnectTo(graph.Out(0));
e.SetName("output_e").ConnectTo(graph.Out(1));
f.SetName("output_f").ConnectTo(graph.Out(2));
return graph.GetConfig();
}
讓節點彼此分離
在 MediaPipe 中,封包串流和側封包與處理節點一樣有意義。而且任何節點輸入要求和輸出產品,均能以使用及產生的串流和側邊封包明確表示。
錯誤做法:程式碼有誤的示例。
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
auto& node2 = graph.AddNode("Calculator2");
node1.Out("OUTPUT").ConnectTo(node2.In("INPUT")); // Bad.
auto& node3 = graph.AddNode("Calculator3");
node1.Out("OUTPUT").ConnectTo(node3.In("INPUT_B")); // Bad.
node2.Out("OUTPUT").ConnectTo(node3.In("INPUT_C")); // Bad.
auto& node4 = graph.AddNode("Calculator4");
node1.Out("OUTPUT").ConnectTo(node4.In("INPUT_B")); // Bad.
node2.Out("OUTPUT").ConnectTo(node4.In("INPUT_C")); // Bad.
node3.Out("OUTPUT").ConnectTo(node4.In("INPUT_D")); // Bad.
// Outputs.
node1.Out("OUTPUT").SetName("b").ConnectTo(graph.Out(0)); // Bad.
node2.Out("OUTPUT").SetName("c").ConnectTo(graph.Out(1)); // Bad.
node3.Out("OUTPUT").SetName("d").ConnectTo(graph.Out(2)); // Bad.
node4.Out("OUTPUT").SetName("e").ConnectTo(graph.Out(3)); // Bad.
return graph.GetConfig();
}
在上述程式碼中:
- 節點會相互連結,例如
node4
知道其輸入內容來自何處 (node1
、node2
、node3
),而且能簡化重構、維護及程式碼重複使用作業。- 這類使用模式從 proto 表示法降級,其中節點預設為分離。
- 重複的
node#.Out("OUTPUT")
呼叫會重複且容易閱讀,因為您可以改用更簡潔的名稱並提供實際類型。
因此,如要修正上述問題,請編寫下列圖形建構程式碼:
正確:良好程式碼的範例。
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
// `node1` usage is limited to 3 lines below.
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
Stream<B> b = node1.Out("OUTPUT").Cast<B>();
// `node2` usage is limited to 3 lines below.
auto& node2 = graph.AddNode("Calculator2");
b.ConnectTo(node2.In("INPUT"));
Stream<C> c = node2.Out("OUTPUT").Cast<C>();
// `node3` usage is limited to 4 lines below.
auto& node3 = graph.AddNode("Calculator3");
b.ConnectTo(node3.In("INPUT_B"));
c.ConnectTo(node3.In("INPUT_C"));
Stream<D> d = node3.Out("OUTPUT").Cast<D>();
// `node4` usage is limited to 5 lines below.
auto& node4 = graph.AddNode("Calculator4");
b.ConnectTo(node4.In("INPUT_B"));
c.ConnectTo(node4.In("INPUT_C"));
d.ConnectTo(node4.In("INPUT_D"));
Stream<E> e = node4.Out("OUTPUT").Cast<E>();
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
現在,您可以視需要輕鬆移除 node1
,讓 b
成為圖形輸入,且不需要更新 node2
、node3
、node4
(與 proto 表示法相同),因為兩者會彼此分離。
整體來說,上述程式碼會更密切地複製 proto 圖:
input_stream: "a"
node {
calculator: "Calculator1"
input_stream: "INPUT:a"
output_stream: "OUTPUT:b"
}
node {
calculator: "Calculator2"
input_stream: "INPUT:b"
output_stream: "OUTPUT:C"
}
node {
calculator: "Calculator3"
input_stream: "INPUT_B:b"
input_stream: "INPUT_C:c"
output_stream: "OUTPUT:d"
}
node {
calculator: "Calculator4"
input_stream: "INPUT_B:b"
input_stream: "INPUT_C:c"
input_stream: "INPUT_D:d"
output_stream: "OUTPUT:e"
}
output_stream: "b"
output_stream: "c"
output_stream: "d"
output_stream: "e"
除此之外,您現在可以擷取公用程式函式,以便在其他圖表中重複使用:
正確:良好程式碼的範例。
Stream<B> RunCalculator1(Stream<A> a, Graph& graph) {
auto& node = graph.AddNode("Calculator1");
a.ConnectTo(node.In("INPUT"));
return node.Out("OUTPUT").Cast<B>();
}
Stream<C> RunCalculator2(Stream<B> b, Graph& graph) {
auto& node = graph.AddNode("Calculator2");
b.ConnectTo(node.In("INPUT"));
return node.Out("OUTPUT").Cast<C>();
}
Stream<D> RunCalculator3(Stream<B> b, Stream<C> c, Graph& graph) {
auto& node = graph.AddNode("Calculator3");
b.ConnectTo(node.In("INPUT_B"));
c.ConnectTo(node.In("INPUT_C"));
return node.Out("OUTPUT").Cast<D>();
}
Stream<E> RunCalculator4(Stream<B> b, Stream<C> c, Stream<D> d, Graph& graph) {
auto& node = graph.AddNode("Calculator4");
b.ConnectTo(node.In("INPUT_B"));
c.ConnectTo(node.In("INPUT_C"));
d.ConnectTo(node.In("INPUT_D"));
return node.Out("OUTPUT").Cast<E>();
}
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
Stream<B> b = RunCalculator1(a, graph);
Stream<C> c = RunCalculator2(b, graph);
Stream<D> d = RunCalculator3(b, c, graph);
Stream<E> e = RunCalculator4(b, c, d, graph);
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
分離節點可提高可讀性
錯誤做法:程式碼有誤的示例。
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
Stream<B> b = node1.Out("OUTPUT").Cast<B>();
auto& node2 = graph.AddNode("Calculator2");
b.ConnectTo(node2.In("INPUT"));
Stream<C> c = node2.Out("OUTPUT").Cast<C>();
auto& node3 = graph.AddNode("Calculator3");
b.ConnectTo(node3.In("INPUT_B"));
c.ConnectTo(node3.In("INPUT_C"));
Stream<D> d = node3.Out("OUTPUT").Cast<D>();
auto& node4 = graph.AddNode("Calculator4");
b.ConnectTo(node4.In("INPUT_B"));
c.ConnectTo(node4.In("INPUT_C"));
d.ConnectTo(node4.In("INPUT_D"));
Stream<E> e = node4.Out("OUTPUT").Cast<E>();
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
在上述程式碼中,您很難掌握每個節點開始和結束的位置。想要改善此功能並協助程式碼讀取器,只要在每個節點前後加上空白行即可:
正確:良好程式碼的範例。
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
auto& node1 = graph.AddNode("Calculator1");
a.ConnectTo(node1.In("INPUT"));
Stream<B> b = node1.Out("OUTPUT").Cast<B>();
auto& node2 = graph.AddNode("Calculator2");
b.ConnectTo(node2.In("INPUT"));
Stream<C> c = node2.Out("OUTPUT").Cast<C>();
auto& node3 = graph.AddNode("Calculator3");
b.ConnectTo(node3.In("INPUT_B"));
c.ConnectTo(node3.In("INPUT_C"));
Stream<D> d = node3.Out("OUTPUT").Cast<D>();
auto& node4 = graph.AddNode("Calculator4");
b.ConnectTo(node4.In("INPUT_B"));
c.ConnectTo(node4.In("INPUT_C"));
d.ConnectTo(node4.In("INPUT_D"));
Stream<E> e = node4.Out("OUTPUT").Cast<E>();
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}
此外,上述表示法更符合 CalculatorGraphConfig
proto 表示法。
如果您將節點擷取至公用函式中,這些節點的範圍已經限定在函式中,並能從位置開始和結束位置也很明確,因此可以完全掌握:
正確:良好程式碼的範例。
CalculatorGraphConfig BuildGraph() {
Graph graph;
// Inputs.
Stream<A> a = graph.In(0).Cast<A>();
Stream<B> b = RunCalculator1(a, graph);
Stream<C> c = RunCalculator2(b, graph);
Stream<D> d = RunCalculator3(b, c, graph);
Stream<E> e = RunCalculator4(b, c, d, graph);
// Outputs.
b.SetName("b").ConnectTo(graph.Out(0));
c.SetName("c").ConnectTo(graph.Out(1));
d.SetName("d").ConnectTo(graph.Out(2));
e.SetName("e").ConnectTo(graph.Out(3));
return graph.GetConfig();
}