Enable cross-service propagation of otel trace metadata

nonfree/controller/CentralDB.hpp

@@ -112,7 +112,20 @@ class CentralDB : public DB {
 	std::string _myAddressStr;
 	std::string _connString;
 
-	BlockingQueue<std::pair<nlohmann::json, bool> > _commitQueue;
+	struct _queueItem {
+		_queueItem() : jsonData(), notifyListeners(false), traceContext()
+		{
+		}
+
+		~_queueItem()
+		{
+		}
+
+		nlohmann::json jsonData;
+		bool notifyListeners;
+		std::map<std::string, std::string> traceContext;
+	};
+	BlockingQueue<_queueItem> _commitQueue;
 
 	std::thread _heartbeatThread;
 	std::shared_ptr<NotificationListener> _membersDbWatcher;

nonfree/controller/OtelCarrier.hpp

@@ -0,0 +1,47 @@
+#ifndef OTEL_CARRIER_HPP
+#define OTEL_CARRIER_HPP
+
+#include "opentelemetry/context/propagation/text_map_propagator.h"
+#include "opentelemetry/trace/propagation/http_trace_context.h"
+
+namespace nostd = opentelemetry::nostd;
+
+namespace ZeroTier {
+
+template <typename T> class OtelCarrier : public opentelemetry::context::propagation::TextMapCarrier {
+  public:
+	OtelCarrier<T>(T& headers) : headers_(headers)
+	{
+	}
+
+	OtelCarrier() = default;
+
+	virtual nostd::string_view Get(nostd::string_view key) const noexcept override
+	{
+		std::string key_to_compare = key.data();
+
+		if (key == opentelemetry::trace::propagation::kTraceParent) {
+			key_to_compare = "traceparent";
+		}
+		else if (key == opentelemetry::trace::propagation::kTraceState) {
+			key_to_compare = "tracestate";
+		}
+		auto it = headers_.find(key_to_compare);
+		if (it != headers_.end()) {
+			return it->second;
+		}
+
+		return "";
+	}
+
+	virtual void Set(nostd::string_view key, nostd::string_view value) noexcept override
+	{
+		headers_.insert(std::pair<std::string, std::string>(std::string(key), std::string(value)));
+	}
+
+	T& headers_;
+};
+
+}	// namespace ZeroTier
+
+#endif	 // OTEL_CARRIER_HPP

nonfree/controller/PubSubListener.cpp

@@ -4,11 +4,16 @@
 #include "ControllerConfig.hpp"
 #include "CtlUtil.hpp"
 #include "DB.hpp"
+#include "OtelCarrier.hpp"
 #include "member.pb.h"
 #include "network.pb.h"
+#include "opentelemetry/context/propagation/global_propagator.h"
+#include "opentelemetry/trace/propagation/http_trace_context.h"
 #include "opentelemetry/trace/provider.h"
+#include "opentelemetry/trace/tracer.h"
 #include "rustybits.h"
 
+#include <google/cloud/opentelemetry_options.h>
 #include <google/cloud/pubsub/admin/subscription_admin_client.h>
 #include <google/cloud/pubsub/admin/subscription_admin_connection.h>
 #include <google/cloud/pubsub/admin/topic_admin_client.h>
@@ -50,7 +55,9 @@ PubSubListener::PubSubListener(std::string controller_id, std::string project, s
 		create_gcp_pubsub_subscription_if_needed(_project, _subscription_id, _topic, _controller_id);
 	}
 
-	_subscriber = std::make_shared<pubsub::Subscriber>(pubsub::MakeSubscriberConnection(*_subscription));
+	_subscriber = std::make_shared<pubsub::Subscriber>(
+		pubsub::MakeSubscriberConnection(*_subscription),
+		google::cloud::Options {}.set<google::cloud::OpenTelemetryTracingOption>(true));
 
 	_run = true;
 	_subscriberThread = std::thread(&PubSubListener::subscribe, this);
@@ -77,20 +84,38 @@ void PubSubListener::subscribe()
 			auto session = _subscriber->Subscribe([this](pubsub::Message const& m, pubsub::AckHandler h) {
 				auto provider = opentelemetry::trace::Provider::GetTracerProvider();
 				auto tracer = provider->GetTracer("PubSubListener");
-				auto span = tracer->StartSpan("PubSubListener::onMessage");
-				auto scope = tracer->WithActiveSpan(span);
-				span->SetAttribute("message_id", m.message_id());
-				span->SetAttribute("ordering_key", m.ordering_key());
 
-				fprintf(stderr, "Received message %s\n", m.message_id().c_str());
+				auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator();
-				if (onNotification(m.data())) {
+				auto attrs = m.attributes();
-					std::move(h).ack();
+				std::map<std::string, std::string> attrs_map;
-					span->SetStatus(opentelemetry::trace::StatusCode::kOk);
+				for (auto const& kv : m.attributes()) {
-					return true;
+					fprintf(stderr, "Message attribute: %s=%s\n", kv.first.c_str(), kv.second.c_str());
+					attrs_map.emplace(kv.first, kv.second);
 				}
-				else {
+
-					span->SetStatus(opentelemetry::trace::StatusCode::kError, "onNotification failed");
+				OtelCarrier<std::map<std::string, std::string> > carrier(attrs_map);
-					return false;
+
+				auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent();
+				auto new_context = propagator->Extract(carrier, current_ctx);
+				auto remote_span = opentelemetry::trace::GetSpan(new_context);
+				auto remote_scope = tracer->WithActiveSpan(remote_span);
+
+				{
+					auto span = tracer->StartSpan("PubSubListener::onMessage");
+					auto scope = tracer->WithActiveSpan(span);
+					span->SetAttribute("message_id", m.message_id());
+					span->SetAttribute("ordering_key", m.ordering_key());
+
+					fprintf(stderr, "Received message %s\n", m.message_id().c_str());
+					if (onNotification(m.data())) {
+						std::move(h).ack();
+						span->SetStatus(opentelemetry::trace::StatusCode::kOk);
+						return true;
+					}
+					else {
+						span->SetStatus(opentelemetry::trace::StatusCode::kError, "onNotification failed");
+						return false;
+					}
 				}
 			});
 

nonfree/controller/PubSubWriter.cpp

@@ -2,9 +2,11 @@
 
 #include "../../osdep/OSUtils.hpp"
 #include "CtlUtil.hpp"
+#include "OtelCarrier.hpp"
 #include "member.pb.h"
 #include "member_status.pb.h"
 #include "network.pb.h"
+#include "opentelemetry/context/propagation/global_propagator.h"
 
 #include <chrono>
 #include <google/cloud/options.h>
@@ -42,7 +44,8 @@ PubSubWriter::PubSubWriter(std::string project, std::string topic, std::string c
 		::google::cloud::Options {}
 			.set<pubsub::RetryPolicyOption>(pubsub::LimitedTimeRetryPolicy(std::chrono::seconds(5)).clone())
 			.set<pubsub::BackoffPolicyOption>(
-				pubsub::ExponentialBackoffPolicy(std::chrono::milliseconds(100), std::chrono::seconds(2), 1.3).clone());
+				pubsub::ExponentialBackoffPolicy(std::chrono::milliseconds(100), std::chrono::seconds(2), 1.3).clone())
+			.set<pubsub::MessageOrderingOption>(true);
 	auto publisher = pubsub::MakePublisherConnection(pubsub::Topic(project, topic), std::move(options));
 	_publisher = std::make_shared<pubsub::Publisher>(std::move(publisher));
 }
@@ -56,10 +59,26 @@ bool PubSubWriter::publishMessage(
 	const std::string& frontend,
 	const std::string& orderingKey)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("PubSubWriter");
+	auto span = tracer->StartSpan("PubSubWriter::publishMessage");
+	auto scope = tracer->WithActiveSpan(span);
+
 	fprintf(stderr, "Publishing message to %s\n", _topic.c_str());
 	std::vector<std::pair<std::string, std::string> > attributes;
 	attributes.emplace_back("controller_id", _controller_id);
 
+	std::map<std::string, std::string> attrs_map;
+	OtelCarrier<std::map<std::string, std::string> > carrier(attrs_map);
+	auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator();
+	auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent();
+	propagator->Inject(carrier, current_ctx);
+
+	for (const auto& kv : attrs_map) {
+		fprintf(stderr, "Attributes injected: %s=%s\n", kv.first.c_str(), kv.second.c_str());
+		attributes.emplace_back(kv.first, kv.second);
+	}
+
 	if (! frontend.empty()) {
 		attributes.emplace_back("frontend", frontend);
 	}

service/OneService.cpp

@@ -49,6 +49,10 @@
 #endif
 
 #ifdef ZT_OPENTELEMETRY_ENABLED
+#include "opentelemetry/baggage/propagation/baggage_propagator.h"
+#include "opentelemetry/context/propagation/composite_propagator.h"
+#include "opentelemetry/context/propagation/global_propagator.h"
+#include "opentelemetry/context/propagation/text_map_propagator.h"
 #include "opentelemetry/exporters/memory/in_memory_data.h"
 #include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h"
 #include "opentelemetry/exporters/otlp/otlp_grpc_log_record_exporter.h"
@@ -68,6 +72,7 @@
 #include "opentelemetry/sdk/trace/tracer.h"
 #include "opentelemetry/sdk/trace/tracer_context.h"
 #include "opentelemetry/sdk/trace/tracer_provider.h"
+#include "opentelemetry/trace/propagation/http_trace_context.h"
 #include "opentelemetry/trace/provider.h"
 
 namespace sdktrace = opentelemetry::v1::sdk::trace;
@@ -1169,6 +1174,19 @@ class OneServiceImpl : public OneService {
 			_traceProvider = opentelemetry::nostd::shared_ptr<sdktrace::TracerProvider>(
 				new sdktrace::TracerProvider(std::move(tracer_context)));
 			sdktrace::Provider::SetTracerProvider(_traceProvider);
+
+			std::vector<std::unique_ptr<opentelemetry::context::propagation::TextMapPropagator> > propagators;
+			propagators.push_back(
+				std::unique_ptr<opentelemetry::context::propagation::TextMapPropagator>(
+					new opentelemetry::trace::propagation::HttpTraceContext()));
+			propagators.push_back(
+				std::unique_ptr<opentelemetry::context::propagation::TextMapPropagator>(
+					new opentelemetry::baggage::propagation::BaggagePropagator()));
+
+			auto p = opentelemetry::nostd::shared_ptr<opentelemetry::context::propagation::TextMapPropagator>(
+				new opentelemetry::context::propagation::CompositePropagator(std::move(propagators)));
+
+			opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator(p);
 		}
 	}