Part 13 of 15 • 🏆 Expert (Mastery) • ⏱️ 240 minutes

Live Observability & Telemetry for watsonx Orchestrate: Monitoring Agent Health, Latency, and Token Spend

Tracking token usage, latency histograms, error rates, and tool execution health in a real-time web dashboard.

---

Step 1: Write Tool Logic (telemetry_collector.py)

import time
from collections import defaultdict
from datetime import datetime

class AgentTelemetry:
    def __init__(self):
        self.metrics = defaultdict(lambda: {"invocations": 0, "total_ms": 0, "errors": 0, "tokens": 0})

    def record(self, agent_name: str, duration_ms: float, tokens: int, error: bool = False):
        m = self.metrics[agent_name]
        m["invocations"] += 1
        m["total_ms"] += duration_ms
        m["tokens"] += tokens
        if error:
            m["errors"] += 1

    def summary(self):
        print(f"📊 LIVE TELEMETRY SNAPSHOT [{datetime.utcnow().strftime('%H:%M:%S')}]")
        print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
        for agent, m in self.metrics.items():
            avg_lat = m["total_ms"] / m["invocations"] if m["invocations"] > 0 else 0
            err_rate = (m["errors"] / m["invocations"] * 100) if m["invocations"] > 0 else 0
            print(f"Agent: {agent:25} | Invocations: {m['invocations']:5} | Avg Latency: {avg_lat:6.1f}ms | Tokens: {m['tokens']:7} | Errors: {err_rate:4.1f}%")

collector = AgentTelemetry()
collector.record("hello_world_greeter", 320, 185)
collector.record("finance_approval_agent", 840, 420)
collector.record("async_job_manager", 1450, 610)
collector.summary()

Step 2: Dependencies (requirements.txt)

ibm-watsonx-orchestrate>=2.9.0

Step 3: Agent Manifest (agent.yaml)

spec_version: v1
kind: native
name: telemetry_enabled_agent
title: "Monitored Agent"
model: ibm/granite-3-8b-instruct

Step 4: Live Execution

python3 telemetry_collector.py

📊 LIVE TELEMETRY SNAPSHOT [20:25:00]
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Agent: hello_world_greeter       | Invocations:     1 | Avg Latency:  320.0ms | Tokens:     185 | Errors:  0.0%
Agent: finance_approval_agent    | Invocations:     1 | Avg Latency:  840.0ms | Tokens:     420 | Errors:  0.0%
Agent: async_job_manager         | Invocations:     1 | Avg Latency: 1450.0ms | Tokens:     610 | Errors:  0.0%

---

💡 Pro Tips & Architecture Insights

---