Go HTTP server dengan OpenTelemetry tracing

Service Go 50ms latency naik jadi 800ms — di mana bottleneck? DB? Cache? Downstream API? OpenTelemetry tracing kasih jawaban visual dengan timeline waterfall. Snippet ini Go HTTP server lengkap dengan auto-instrument net/http + manual span untuk operasi custom.

Kode

package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
	"go.opentelemetry.io/otel/propagation"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
	"go.opentelemetry.io/otel/trace"
)

const serviceName = "api-tokopedia"

// initTracer setup TracerProvider global
func initTracer(ctx context.Context) (*sdktrace.TracerProvider, error) {
	endpoint := os.Getenv("OTEL_ENDPOINT")
	if endpoint == "" {
		endpoint = "tempo.observability.svc:4318"
	}

	exporter, err := otlptracehttp.New(ctx,
		otlptracehttp.WithEndpoint(endpoint),
		otlptracehttp.WithInsecure(),
	)
	if err != nil {
		return nil, fmt.Errorf("init exporter: %w", err)
	}

	res, err := resource.New(ctx,
		resource.WithAttributes(
			semconv.ServiceName(serviceName),
			semconv.ServiceVersion(os.Getenv("APP_VERSION")),
			semconv.DeploymentEnvironment(os.Getenv("ENV")),
		),
	)
	if err != nil {
		return nil, err
	}

	tp := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(exporter,
			sdktrace.WithMaxQueueSize(2048),
			sdktrace.WithMaxExportBatchSize(512),
			sdktrace.WithBatchTimeout(5*time.Second),
		),
		sdktrace.WithResource(res),
		// Head sampling: 10% di production, 100% di staging
		sdktrace.WithSampler(sdktrace.ParentBased(
			sdktrace.TraceIDRatioBased(0.10),
		)),
	)

	otel.SetTracerProvider(tp)
	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
		propagation.TraceContext{},
		propagation.Baggage{},
	))

	return tp, nil
}

// Server handlers
type Server struct {
	db     *sql.DB
	tracer trace.Tracer
	client *http.Client
}

func NewServer(db *sql.DB) *Server {
	return &Server{
		db:     db,
		tracer: otel.Tracer(serviceName),
		// HTTP client otomatis propagate trace context ke downstream
		client: &http.Client{
			Transport: otelhttp.NewTransport(http.DefaultTransport),
			Timeout:   5 * time.Second,
		},
	}
}

// /api/produk/:id — handler dengan manual span untuk DB call + external API
func (s *Server) handleGetProduk(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	produkID := r.PathValue("id")

	// Manual span untuk DB call
	produk, err := s.fetchProdukFromDB(ctx, produkID)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Manual span untuk external API (rekomendasi service)
	rekomendasi, err := s.fetchRekomendasi(ctx, produkID)
	if err != nil {
		// Non-fatal — degrade gracefully
		rekomendasi = []string{}
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]any{
		"produk":      produk,
		"rekomendasi": rekomendasi,
	})
}

func (s *Server) fetchProdukFromDB(ctx context.Context, id string) (map[string]any, error) {
	ctx, span := s.tracer.Start(ctx, "db.fetch_produk",
		trace.WithAttributes(
			attribute.String("db.system", "postgresql"),
			attribute.String("produk.id", id),
		),
	)
	defer span.End()

	var nama string
	var harga int
	err := s.db.QueryRowContext(ctx,
		"SELECT nama, harga FROM produk WHERE id = $1",
		id,
	).Scan(&nama, &harga)
	if err != nil {
		span.RecordError(err)
		span.SetStatus(codes.Error, err.Error())
		return nil, err
	}

	span.SetAttributes(
		attribute.String("produk.nama", nama),
		attribute.Int("produk.harga", harga),
	)
	span.SetStatus(codes.Ok, "")

	return map[string]any{"nama": nama, "harga": harga}, nil
}

func (s *Server) fetchRekomendasi(ctx context.Context, produkID string) ([]string, error) {
	// HTTP client sudah pakai otelhttp — span auto-create
	req, err := http.NewRequestWithContext(ctx, "GET",
		"http://ml-service:8080/rekomendasi/"+produkID, nil)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("ml-service: status %d", resp.StatusCode)
	}

	var out []string
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		return nil, err
	}
	return out, nil
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
	defer stop()

	tp, err := initTracer(ctx)
	if err != nil {
		fmt.Println("init tracer:", err)
		os.Exit(1)
	}
	defer func() {
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		tp.Shutdown(shutdownCtx)
	}()

	db, err := sql.Open("pgx", os.Getenv("DATABASE_URL"))
	if err != nil {
		fmt.Println("db:", err)
		os.Exit(1)
	}
	defer db.Close()

	server := NewServer(db)
	mux := http.NewServeMux()
	mux.HandleFunc("GET /api/produk/{id}", server.handleGetProduk)
	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("ok"))
	})

	// Wrap mux dengan otelhttp — semua route auto-trace
	handler := otelhttp.NewHandler(mux, "http.server",
		otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string {
			return r.Method + " " + r.Pattern
		}),
	)

	srv := &http.Server{
		Addr:         ":8080",
		Handler:      handler,
		ReadTimeout:  15 * time.Second,
		WriteTimeout: 30 * time.Second,
	}

	go func() {
		fmt.Println("listening :8080")
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			fmt.Println("server:", err)
		}
	}()

	<-ctx.Done()
	fmt.Println("shutting down")

	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	srv.Shutdown(shutdownCtx)
}

Pemakaian

# Run dengan endpoint Tempo lokal
OTEL_ENDPOINT=localhost:4318 \
APP_VERSION=v1.2.3 \
ENV=staging \
go run main.go

# Trigger request
curl http://localhost:8080/api/produk/123

# Trace muncul di Tempo / Jaeger UI:
# - http.server GET /api/produk/{id}  (parent, 150ms)
#   - db.fetch_produk                 (child, 30ms)
#   - HTTP GET ml-service/rekomendasi (child, 110ms, auto)

# Query trace di Grafana — span dengan error
{ resource.service.name="api-tokopedia" }
| status = error
| duration > 1s

Kapan dipakai

• Microservice yang call multiple downstream.
• Slow endpoint investigation — drill down per phase.
• Performance regression detection antar version.
• SLO compliance check — p95 latency, error rate.

Catatan

• otelhttp.NewTransport wajib pada HTTP client supaya trace propagate ke downstream service. Tanpa ini, trace putus di network boundary.
• otelhttp.NewHandler wrap mux untuk auto-create parent span per request.
• Span name pattern: pakai route template (GET /api/produk/{id}) bukan URL literal — cardinality friendly di backend tracing.
• RecordError + SetStatus untuk error — tanpa SetStatus, span tetap “ok” di UI walau error.
• TraceIDRatioBased sampling — 10% production cukup. Penuh 100% di staging untuk debug.
• Shutdown wait — saat SIGTERM, span yang belum di-export bisa lost. Selalu call tp.Shutdown di defer.

Trace storage mahal. Tempo / Jaeger dengan retention 7 hari sudah okay untuk debugging hari ini. Untuk compliance long-term, sample 1% saja dan archive ke S3.