Skip to content

Add in-VM OTLP telemetry export sink #308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
archandatta wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Add in-VM OTLP telemetry export sink #308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5be369e
Add in-VM OTLP telemetry export sink
archandatta Jul 7, 2026
bfda873
Address review: BTEL_OTLP_* env prefix, non-fatal sink init
archandatta Jul 9, 2026
e132c09
Make OTLP batch-queue drops observable (review E1)
archandatta Jul 10, 2026
ff3f848
Update config test for OTLP path/service-name defaults
archandatta Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions server/cmd/api/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"github.com/ghodss/yaml"
"github.com/go-chi/chi/v5"
chiMiddleware "github.com/go-chi/chi/v5/middleware"
"github.com/go-logr/logr"
"go.opentelemetry.io/otel"
"golang.org/x/sync/errgroup"

serverpkg "github.com/kernel/kernel-images/server"
Expand Down Expand Up @@ -125,6 +127,36 @@ func main() {
}
}

// Optional OTLP export sink. Independent of S2; both can run together.
var otlpWriter *events.OTLPStorageWriter
if config.OTLPEndpoint != "" {
headers := map[string]string{}
if config.InstanceJWT != "" {
headers["Authorization"] = "Bearer " + config.InstanceJWT
}
slogger.Info("OTLP export enabled", "endpoint", config.OTLPEndpoint, "path", config.OTLPPath)
// The OTel log SDK reports batch-queue drops through its global logger at
// logr V(1), which slog renders just below Info, so a default Info handler
// would swallow it. Wire it to a handler that admits that level so records
// dropped under sustained backpressure are observable, not silently lost.
otelDiag := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo - 1})
otel.SetLogger(logr.FromSlogHandler(otelDiag))
otlpWriter = events.NewOTLPStorageWriter(eventStream, events.OTLPConfig{
Endpoint: config.OTLPEndpoint,
URLPath: config.OTLPPath,
Insecure: config.OTLPInsecure,
Headers: headers,
ServiceName: config.OTLPServiceName,
InstanceName: config.InstanceName,
Metro: config.MetroName,
}, slogger)
if err := otlpWriter.Start(ctx); err != nil {

@Sayan- Sayan- Jul 8, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this makes sense wrt parity to s2. I'd guess ~some events will race with any intermediate relay having the information required to make routing decisions. may need to reshape depending on what we want to guarantee

// Best-effort sink: a failed exporter must not take down the browser.
slogger.Error("OTLP export disabled: storage writer failed to start", "err", err)
otlpWriter = nil
}
}

apiService, err := api.New(
recorder.NewFFmpegManager(),
recorder.NewFFmpegRecorderFactory(config.PathToFFmpeg, defaultParams, stz),
Expand Down Expand Up @@ -306,6 +338,14 @@ func main() {
slogger.Error("s2 storage writer stop failed", "err", err)
}
}

if otlpWriter != nil {
stopCtx, stopCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer stopCancel()
if err := otlpWriter.Stop(stopCtx); err != nil {
slogger.Error("otlp storage writer stop failed", "err", err)
}
}
}

func mustFFmpeg() {
Expand Down
24 changes: 24 additions & 0 deletions server/cmd/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,19 @@ type Config struct {
S2Basin string `envconfig:"S2_BASIN" default:""`
S2AccessToken string `envconfig:"S2_ACCESS_TOKEN" default:""`
S2Stream string `envconfig:"S2_STREAM" default:""`

// Browser-telemetry OTLP export. OTLPEndpoint (host[:port]) must be set to
// enable the OTLP sink. The BTEL_ prefix avoids collision with the standard
// OTEL_ vars that configure the API server's own telemetry.
OTLPEndpoint string `envconfig:"BTEL_OTLP_ENDPOINT" default:""`
OTLPPath string `envconfig:"BTEL_OTLP_PATH" default:"/v1/logs"`
OTLPInsecure bool `envconfig:"BTEL_OTLP_INSECURE" default:"false"`
OTLPServiceName string `envconfig:"BTEL_OTLP_SERVICE_NAME" default:"kernel-browser"`
// Platform-injected identity, reused to stamp the OTLP Resource. These are
// the same envs the VM already receives.
InstanceJWT string `envconfig:"KERNEL_INSTANCE_JWT" default:""`
InstanceName string `envconfig:"INST_NAME" default:""`
MetroName string `envconfig:"METRO_NAME" default:""`
}

// LogValue implements slog.LogValuer, redacting secret fields.
Expand All @@ -55,6 +68,10 @@ func (c *Config) LogValue() slog.Value {
if c.S2AccessToken != "" {
s2AccessToken = "[redacted]"
}
otlpJWT := ""
if c.InstanceJWT != "" {
otlpJWT = "[redacted]"
}
return slog.GroupValue(
slog.Int("port", c.Port),
slog.Int("frame_rate", c.FrameRate),
Expand All @@ -73,6 +90,13 @@ func (c *Config) LogValue() slog.Value {
slog.String("s2_basin", c.S2Basin),
slog.String("s2_access_token", s2AccessToken),
slog.String("s2_stream", c.S2Stream),
slog.String("otlp_endpoint", c.OTLPEndpoint),
slog.String("otlp_path", c.OTLPPath),
slog.Bool("otlp_insecure", c.OTLPInsecure),
slog.String("otlp_instance_jwt", otlpJWT),
slog.String("otlp_service_name", c.OTLPServiceName),
slog.String("otlp_instance_name", c.InstanceName),
slog.String("otlp_metro", c.MetroName),
)
}

Expand Down
6 changes: 6 additions & 0 deletions server/cmd/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ func TestLoad(t *testing.T) {
ChromeDriverProxyPort: 9224,
ChromeDriverUpstreamAddr: "127.0.0.1:9225",
DevToolsProxyAddr: "127.0.0.1:9222",
OTLPPath: "/v1/logs",
OTLPServiceName: "kernel-browser",
},
},
{
Expand Down Expand Up @@ -63,6 +65,8 @@ func TestLoad(t *testing.T) {
ChromeDriverProxyPort: 5432,
ChromeDriverUpstreamAddr: "127.0.0.1:9999",
DevToolsProxyAddr: "127.0.0.1:9876",
OTLPPath: "/v1/logs",
OTLPServiceName: "kernel-browser",
},
},
{
Expand All @@ -85,6 +89,8 @@ func TestLoad(t *testing.T) {
ChromeDriverProxyPort: 9224,
ChromeDriverUpstreamAddr: "127.0.0.1:9225",
DevToolsProxyAddr: "10.0.0.1:1234",
OTLPPath: "/v1/logs",
OTLPServiceName: "kernel-browser",
},
},
{
Expand Down
42 changes: 23 additions & 19 deletions server/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/ghodss/yaml v1.0.0
github.com/glebarez/sqlite v1.11.0
github.com/go-chi/chi/v5 v5.2.1
github.com/go-logr/logr v1.4.3
github.com/google/uuid v1.6.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/kernel/hypeman-go v0.20.0
Expand All @@ -27,9 +28,16 @@ require (
github.com/samber/lo v1.52.0
github.com/stretchr/testify v1.11.1
github.com/testcontainers/testcontainers-go v0.40.0
golang.org/x/sync v0.18.0
golang.org/x/sys v0.39.0
golang.org/x/term v0.37.0
go.opentelemetry.io/otel v1.44.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.20.0
go.opentelemetry.io/otel/log v0.20.0
go.opentelemetry.io/otel/sdk v1.44.0
go.opentelemetry.io/otel/sdk/log v0.20.0
go.opentelemetry.io/proto/otlp v1.10.0
golang.org/x/sync v0.20.0
golang.org/x/sys v0.45.0
golang.org/x/term v0.43.0
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
)

Expand All @@ -39,6 +47,7 @@ require (
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
Expand All @@ -53,12 +62,11 @@ require (
github.com/ebitengine/purego v0.8.4 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/glebarez/go-sqlite v1.21.2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand Down Expand Up @@ -100,21 +108,17 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/sdk v1.39.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect
golang.org/x/crypto v0.44.0 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/tools v0.38.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.1 // indirect
google.golang.org/protobuf v1.36.10 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
golang.org/x/crypto v0.51.0 // indirect
golang.org/x/mod v0.35.0 // indirect
golang.org/x/net v0.55.0 // indirect
golang.org/x/text v0.37.0 // indirect
golang.org/x/tools v0.44.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
google.golang.org/grpc v1.81.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gorm.io/gorm v1.25.7 // indirect
modernc.org/libc v1.22.5 // indirect
Expand Down
Loading
Loading