feat: Add OpenTelemetry instrumentation and configuration for enhanced telemetry tracking

12 files changed, 191 insertions, 12 deletions

diff --git a/src/Enmarcha.Backend/Configuration/AppConfiguration.cs b/src/Enmarcha.Backend/Configuration/AppConfiguration.cs
index ca99e3c..f52c89e 100644
--- a/src/Enmarcha.Backend/Configuration/AppConfiguration.cs
+++ b/src/Enmarcha.Backend/Configuration/AppConfiguration.cs
@@ -5,4 +5,11 @@ public class AppConfiguration
     public required string OpenTripPlannerBaseUrl { get; set; }
     public required string GeoapifyApiKey { get; set; }
     public string NominatimBaseUrl { get; set; } = "https://nominatim.openstreetmap.org";
+    public OpenTelemetryConfiguration? OpenTelemetry { get; set; }
+}
+
+public class OpenTelemetryConfiguration
+{
+    public string? Endpoint { get; set; }
+    public string? Headers { get; set; }
 }
diff --git a/src/Enmarcha.Backend/Controllers/ArrivalsController.cs b/src/Enmarcha.Backend/Controllers/ArrivalsController.cs
index a23c69c..eb147fc 100644
--- a/src/Enmarcha.Backend/Controllers/ArrivalsController.cs
+++ b/src/Enmarcha.Backend/Controllers/ArrivalsController.cs
@@ -46,6 +46,10 @@ public partial class ArrivalsController : ControllerBase
         [FromQuery] bool reduced
     )
     {
+        using var activity = Telemetry.Source.StartActivity("GetArrivals");
+        activity?.SetTag("stop.id", id);
+        activity?.SetTag("reduced", reduced);
+
         var tz = TimeZoneInfo.FindSystemTimeZoneById("Europe/Madrid");
         var nowLocal = TimeZoneInfo.ConvertTime(DateTime.UtcNow, tz);
         var todayLocal = nowLocal.Date;
@@ -65,12 +69,14 @@ public partial class ArrivalsController : ControllerBase
 
         if (responseBody is not { IsSuccess: true } || responseBody.Data?.Stop == null)
         {
+            activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, "Error fetching stop data from OTP");
             LogErrorFetchingStopData(response.StatusCode, await response.Content.ReadAsStringAsync());
             return StatusCode(500, "Error fetching stop data");
         }
 
         var stop = responseBody.Data.Stop;
         _logger.LogInformation("Fetched {Count} arrivals for stop {StopName} ({StopId})", stop.Arrivals.Count, stop.Name, id);
+        activity?.SetTag("arrivals.count", stop.Arrivals.Count);
 
         List<Arrival> arrivals = [];
         foreach (var item in stop.Arrivals)
diff --git a/src/Enmarcha.Backend/Controllers/TileController.cs b/src/Enmarcha.Backend/Controllers/TileController.cs
index 4065ecd..3fdedfb 100644
--- a/src/Enmarcha.Backend/Controllers/TileController.cs
+++ b/src/Enmarcha.Backend/Controllers/TileController.cs
@@ -43,12 +43,19 @@ public class TileController : ControllerBase
     [HttpGet("stops/{z:int}/{x:int}/{y:int}")]
     public async Task<IActionResult> Stops(int z, int x, int y)
     {
+        using var activity = Telemetry.Source.StartActivity("GenerateStopsTile");
+        activity?.SetTag("tile.z", z);
+        activity?.SetTag("tile.x", x);
+        activity?.SetTag("tile.y", y);
+
         if (z is < 9 or > 20)
         {
             return BadRequest("Zoom level out of range (9-20)");
         }
 
         var cacheHit = _cache.TryGetValue($"stops-tile-{z}-{x}-{y}", out byte[]? cachedTile);
+        activity?.SetTag("cache.hit", cacheHit);
+
         if (cacheHit && cachedTile != null)
         {
             Response.Headers.Append("X-Cache-Hit", "true");
@@ -78,6 +85,7 @@ public class TileController : ControllerBase
 
         if (responseBody is not { IsSuccess: true })
         {
+            activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, "Error fetching stop data from OTP");
             _logger.LogError(
                 "Error fetching stop data, received {StatusCode} {ResponseBody}",
                 response.StatusCode,
diff --git a/src/Enmarcha.Backend/Controllers/TransitController.cs b/src/Enmarcha.Backend/Controllers/TransitController.cs
index 00e5fb7..4853e66 100644
--- a/src/Enmarcha.Backend/Controllers/TransitController.cs
+++ b/src/Enmarcha.Backend/Controllers/TransitController.cs
@@ -38,14 +38,19 @@ public class TransitController : ControllerBase
     [HttpGet("routes")]
     public async Task<ActionResult<List<RouteDto>>> GetRoutes([FromQuery] string[] feeds)
     {
+        using var activity = Telemetry.Source.StartActivity("GetRoutes");
         if (feeds.Length == 0)
         {
             feeds = ["tussa", "vitrasa", "tranvias", "feve"];
         }
+        activity?.SetTag("feeds", string.Join(",", feeds));
 
         var serviceDate = DateTime.Now.ToString("yyyy-MM-dd");
         var cacheKey = $"routes_{string.Join("_", feeds)}_{serviceDate}";
-        if (_cache.TryGetValue(cacheKey, out List<RouteDto>? cachedRoutes))
+        var cacheHit = _cache.TryGetValue(cacheKey, out List<RouteDto>? cachedRoutes);
+        activity?.SetTag("cache.hit", cacheHit);
+
+        if (cacheHit && cachedRoutes != null)
         {
             return Ok(cachedRoutes);
         }
@@ -71,6 +76,7 @@ public class TransitController : ControllerBase
         }
         catch (Exception e)
         {
+            activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, e.Message);
             _logger.LogError(e, "Error fetching routes");
             return StatusCode(500, "An error occurred while fetching routes.");
         }
@@ -79,10 +85,16 @@ public class TransitController : ControllerBase
     [HttpGet("routes/{id}")]
     public async Task<ActionResult<RouteDetailsDto>> GetRouteDetails(string id)
     {
+        using var activity = Telemetry.Source.StartActivity("GetRouteDetails");
+        activity?.SetTag("route.id", id);
+
         var serviceDate = DateTime.Now.ToString("yyyy-MM-dd");
         var cacheKey = $"route_details_{id}_{serviceDate}";
 
-        if (_cache.TryGetValue(cacheKey, out RouteDetailsDto? cachedDetails))
+        var cacheHit = _cache.TryGetValue(cacheKey, out RouteDetailsDto? cachedDetails);
+        activity?.SetTag("cache.hit", cacheHit);
+
+        if (cacheHit && cachedDetails != null)
         {
             return Ok(cachedDetails);
         }
@@ -104,7 +116,7 @@ public class TransitController : ControllerBase
         }
         catch (Exception e)
         {
-            _logger.LogError(e, "Error fetching route details for {Id}", id);
+            activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, e.Message); _logger.LogError(e, "Error fetching route details for {Id}", id);
             return StatusCode(500, "An error occurred while fetching route details.");
         }
     }
diff --git a/src/Enmarcha.Backend/Enmarcha.Backend.csproj b/src/Enmarcha.Backend/Enmarcha.Backend.csproj
index 941286b..1591e7c 100644
--- a/src/Enmarcha.Backend/Enmarcha.Backend.csproj
+++ b/src/Enmarcha.Backend/Enmarcha.Backend.csproj
@@ -21,6 +21,11 @@
 
     <PackageReference Include="CsvHelper" />
     <PackageReference Include="FuzzySharp" />
+
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/Enmarcha.Backend/Program.cs b/src/Enmarcha.Backend/Program.cs
index a13abec..8599a5c 100644
--- a/src/Enmarcha.Backend/Program.cs
+++ b/src/Enmarcha.Backend/Program.cs
@@ -1,14 +1,113 @@
 using System.Text.Json.Serialization;
+using Enmarcha.Backend;
 using Enmarcha.Backend.Configuration;
 using Enmarcha.Backend.Services;
 using Enmarcha.Backend.Services.Geocoding;
 using Enmarcha.Backend.Services.Processors;
 using Enmarcha.Backend.Services.Providers;
+using Microsoft.AspNetCore.WebUtilities;
+using OpenTelemetry.Logs;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 
 var builder = WebApplication.CreateBuilder(args);
 
 builder.Services.Configure<AppConfiguration>(builder.Configuration.GetSection("App"));
 
+var appConfig = builder.Configuration.GetSection("App").Get<AppConfiguration>();
+var otelConfig = appConfig?.OpenTelemetry;
+
+builder.Logging.AddOpenTelemetry(options =>
+{
+    options.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Enmarcha.Backend"));
+    options.IncludeFormattedMessage = true;
+    options.IncludeScopes = true;
+
+    if (otelConfig?.Endpoint != null)
+    {
+        options.AddOtlpExporter(exporterOptions =>
+        {
+            exporterOptions.Endpoint = new Uri(otelConfig.Endpoint);
+            exporterOptions.Headers = otelConfig.Headers;
+        });
+    }
+
+#if DEBUG
+    options.AddOtlpExporter(exporterOptions =>
+    {
+        exporterOptions.Endpoint = new Uri("http://localhost:17011");
+    });
+#endif
+});
+
+builder.Services.AddOpenTelemetry()
+    .WithTracing(tracing =>
+    {
+        tracing
+            .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Enmarcha.Backend"))
+            .AddSource(Telemetry.Source.Name)
+            .AddAspNetCoreInstrumentation()
+            .AddHttpClientInstrumentation(options =>
+            {
+                options.EnrichWithHttpRequestMessage = (activity, req) =>
+                {
+                    var host = req.RequestUri?.Host;
+                    if (host == null) return;
+
+                    // Set default peer service to host
+                    activity.SetTag("peer.service", host);
+                    activity.SetTag("server.address", host);
+
+                    if (host == "api.geoapify.com")
+                    {
+                        activity.SetTag("peer.service", "Geoapify");
+                        var query = QueryHelpers.ParseQuery(req.RequestUri!.Query);
+                        if (query.ContainsKey("apiKey"))
+                        {
+                            var uriBuilder = new UriBuilder(req.RequestUri);
+                            var newQuery = query.ToDictionary(x => x.Key, x => x.Value.ToString());
+                            newQuery["apiKey"] = "REDACTED";
+                            uriBuilder.Query = string.Join("&", newQuery.Select(x => $"{x.Key}={x.Value}"));
+                            activity.SetTag("http.url", uriBuilder.ToString());
+                        }
+                    }
+                    else if (host.Contains("tussa.org"))
+                    {
+                        activity.SetTag("peer.service", "TUSSA");
+                    }
+                    else if (host.Contains("itranvias.com"))
+                    {
+                        activity.SetTag("peer.service", "Tranvías Coruña");
+                    }
+                    else if (host.Contains("vigo.org"))
+                    {
+                        activity.SetTag("peer.service", "Vitrasa");
+                    }
+                    else if (appConfig?.OpenTripPlannerBaseUrl != null && req.RequestUri!.ToString().StartsWith(appConfig.OpenTripPlannerBaseUrl))
+                    {
+                        activity.SetTag("peer.service", "OpenTripPlanner");
+                    }
+                };
+            })
+            .SetSampler(new TraceIdRatioBasedSampler(0.75));
+
+        if (otelConfig?.Endpoint != null)
+        {
+            tracing.AddOtlpExporter(exporterOptions =>
+            {
+                exporterOptions.Endpoint = new Uri(otelConfig.Endpoint);
+                exporterOptions.Headers = otelConfig.Headers;
+            });
+        }
+
+#if DEBUG
+        tracing.AddOtlpExporter(exporterOptions =>
+        {
+            exporterOptions.Endpoint = new Uri("http://localhost:17011");
+        });
+#endif
+    });
+
 builder.Services
     .AddControllers()
     .AddJsonOptions(options =>
@@ -40,9 +139,21 @@ builder.Services.AddScoped<ArrivalsPipeline>();
 // builder.Services.AddKeyedScoped<IGeocodingService, NominatimGeocodingService>("Nominatim");
 builder.Services.AddHttpClient<IGeocodingService, GeoapifyGeocodingService>();
 builder.Services.AddHttpClient<OtpService>();
+builder.Services.AddHttpClient<Enmarcha.Sources.TranviasCoruna.CorunaRealtimeEstimatesProvider>();
+builder.Services.AddHttpClient<Enmarcha.Sources.Tussa.SantiagoRealtimeEstimatesProvider>();
+builder.Services.AddHttpClient<Costasdev.VigoTransitApi.VigoTransitApiClient>();
 
 var app = builder.Build();
 
+app.Use(async (context, next) =>
+{
+    if (context.Request.Headers.TryGetValue("X-Session-Id", out var sessionId))
+    {
+        System.Diagnostics.Activity.Current?.SetTag("session.id", sessionId.ToString());
+    }
+    await next();
+});
+
 app.MapControllers();
 
 app.Run();
diff --git a/src/Enmarcha.Backend/Services/ArrivalsPipeline.cs b/src/Enmarcha.Backend/Services/ArrivalsPipeline.cs
index 57a46e1..4f49afe 100644
--- a/src/Enmarcha.Backend/Services/ArrivalsPipeline.cs
+++ b/src/Enmarcha.Backend/Services/ArrivalsPipeline.cs
@@ -53,8 +53,10 @@ public class ArrivalsPipeline
     /// </summary>
     public async Task ExecuteAsync(ArrivalsContext context)
     {
+        using var activity = Telemetry.Source.StartActivity("ArrivalsPipeline");
         foreach (var processor in _processors)
         {
+            using var processorActivity = Telemetry.Source.StartActivity($"Processor:{processor.GetType().Name}");
             await processor.ProcessAsync(context);
         }
     }
diff --git a/src/Enmarcha.Backend/Services/Geocoding/GeoapifyGeocodingService.cs b/src/Enmarcha.Backend/Services/Geocoding/GeoapifyGeocodingService.cs
index d6cf5f6..86386e8 100644
--- a/src/Enmarcha.Backend/Services/Geocoding/GeoapifyGeocodingService.cs
+++ b/src/Enmarcha.Backend/Services/Geocoding/GeoapifyGeocodingService.cs
@@ -32,13 +32,19 @@ public class GeoapifyGeocodingService : IGeocodingService
 
     public async Task<List<PlannerSearchResult>> GetAutocompleteAsync(string query)
     {
+        using var activity = Telemetry.Source.StartActivity("GeoapifyAutocomplete");
+        activity?.SetTag("query", query);
+
         if (string.IsNullOrWhiteSpace(query))
         {
             return [];
         }
 
         var cacheKey = $"nominatim_autocomplete_{query.ToLowerInvariant()}";
-        if (_cache.TryGetValue(cacheKey, out List<PlannerSearchResult>? cachedResults) && cachedResults != null)
+        var cacheHit = _cache.TryGetValue(cacheKey, out List<PlannerSearchResult>? cachedResults);
+        activity?.SetTag("cache.hit", cacheHit);
+
+        if (cacheHit && cachedResults != null)
         {
             return cachedResults;
         }
@@ -55,11 +61,13 @@ public class GeoapifyGeocodingService : IGeocodingService
                 .Select(MapToPlannerSearchResult)
                 .ToList() ?? [];
 
+            activity?.SetTag("results.count", results.Count);
             _cache.Set(cacheKey, results, TimeSpan.FromMinutes(60));
             return results;
         }
         catch (Exception ex)
         {
+            activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, ex.Message);
             _logger.LogError(ex, "Error fetching Geoapify autocomplete results from {Url}", url);
             return new List<PlannerSearchResult>();
         }
@@ -67,8 +75,15 @@ public class GeoapifyGeocodingService : IGeocodingService
 
     public async Task<PlannerSearchResult?> GetReverseGeocodeAsync(double lat, double lon)
     {
+        using var activity = Telemetry.Source.StartActivity("GeoapifyReverseGeocode");
+        activity?.SetTag("lat", lat);
+        activity?.SetTag("lon", lon);
+
         var cacheKey = $"nominatim_reverse_{lat:F5}_{lon:F5}";
-        if (_cache.TryGetValue(cacheKey, out PlannerSearchResult? cachedResult) && cachedResult != null)
+        var cacheHit = _cache.TryGetValue(cacheKey, out PlannerSearchResult? cachedResult);
+        activity?.SetTag("cache.hit", cacheHit);
+
+        if (cacheHit && cachedResult != null)
         {
             return cachedResult;
         }
@@ -88,6 +103,7 @@ public class GeoapifyGeocodingService : IGeocodingService
         }
         catch (Exception ex)
         {
+            activity?.SetStatus(System.Diagnostics.ActivityStatusCode.Error, ex.Message);
             _logger.LogError(ex, "Error fetching Geoapify reverse geocode results from {Url}", url);
             return null;
         }
diff --git a/src/Enmarcha.Backend/Services/Processors/CorunaRealTimeProcessor.cs b/src/Enmarcha.Backend/Services/Processors/CorunaRealTimeProcessor.cs
index 6a9d9dc..b933bf4 100644
--- a/src/Enmarcha.Backend/Services/Processors/CorunaRealTimeProcessor.cs
+++ b/src/Enmarcha.Backend/Services/Processors/CorunaRealTimeProcessor.cs
@@ -13,12 +13,12 @@ public class CorunaRealTimeProcessor : AbstractRealTimeProcessor
     private readonly ShapeTraversalService _shapeService;
 
     public CorunaRealTimeProcessor(
-        HttpClient http,
+        CorunaRealtimeEstimatesProvider realtime,
         FeedService feedService,
         ILogger<CorunaRealTimeProcessor> logger,
         ShapeTraversalService shapeService)
     {
-        _realtime = new CorunaRealtimeEstimatesProvider(http);
+        _realtime = realtime;
         _feedService = feedService;
         _logger = logger;
         _shapeService = shapeService;
@@ -41,6 +41,7 @@ public class CorunaRealTimeProcessor : AbstractRealTimeProcessor
             }
 
             var realtime = await _realtime.GetEstimatesForStop(numericStopId);
+            System.Diagnostics.Activity.Current?.SetTag("realtime.count", realtime.Count);
 
             var usedTripIds = new HashSet<string>();
 
diff --git a/src/Enmarcha.Backend/Services/Processors/SantiagoRealTimeProcessor.cs b/src/Enmarcha.Backend/Services/Processors/SantiagoRealTimeProcessor.cs
index 929e439..a4f7d5b 100644
--- a/src/Enmarcha.Backend/Services/Processors/SantiagoRealTimeProcessor.cs
+++ b/src/Enmarcha.Backend/Services/Processors/SantiagoRealTimeProcessor.cs
@@ -12,11 +12,11 @@ public class SantiagoRealTimeProcessor : AbstractRealTimeProcessor
     private readonly ILogger<SantiagoRealTimeProcessor> _logger;
 
     public SantiagoRealTimeProcessor(
-        HttpClient http,
+        SantiagoRealtimeEstimatesProvider realtime,
         FeedService feedService,
         ILogger<SantiagoRealTimeProcessor> logger)
     {
-        _realtime = new SantiagoRealtimeEstimatesProvider(http);
+        _realtime = realtime;
         _feedService = feedService;
         _logger = logger;
     }
@@ -31,6 +31,7 @@ public class SantiagoRealTimeProcessor : AbstractRealTimeProcessor
         try
         {
             var realtime = await _realtime.GetEstimatesForStop(numericStopId);
+            System.Diagnostics.Activity.Current?.SetTag("realtime.count", realtime.Count);
 
             var usedTripIds = new HashSet<string>();
 
@@ -46,7 +47,7 @@ public class SantiagoRealTimeProcessor : AbstractRealTimeProcessor
                         RouteMatch = true
                     })
                     .Where(x => x.RouteMatch) // Strict route matching
-                    .Where(x => x.TimeDiff is >= -5 and <= 25) // Allow 2m early (RealTime < Schedule) or 25m late (RealTime > Schedule)
+                    .Where(x => x.TimeDiff is >= -5 and <= 35) // Allow 2m early (RealTime < Schedule) or 25m late (RealTime > Schedule)
                     .OrderBy(x => Math.Abs(x.TimeDiff)) // Best time fit
                     .FirstOrDefault();
 
diff --git a/src/Enmarcha.Backend/Services/Processors/VitrasaRealTimeProcessor.cs b/src/Enmarcha.Backend/Services/Processors/VitrasaRealTimeProcessor.cs
index 5d44995..0ce54c2 100644
--- a/src/Enmarcha.Backend/Services/Processors/VitrasaRealTimeProcessor.cs
+++ b/src/Enmarcha.Backend/Services/Processors/VitrasaRealTimeProcessor.cs
@@ -16,13 +16,13 @@ public class VitrasaRealTimeProcessor : AbstractRealTimeProcessor
     private readonly AppConfiguration _configuration;
 
     public VitrasaRealTimeProcessor(
-        HttpClient http,
+        VigoTransitApiClient api,
         FeedService feedService,
         ILogger<VitrasaRealTimeProcessor> logger,
         ShapeTraversalService shapeService,
         IOptions<AppConfiguration> options)
     {
-        _api = new VigoTransitApiClient(http);
+        _api = api;
         _feedService = feedService;
         _logger = logger;
         _shapeService = shapeService;
@@ -52,6 +52,8 @@ public class VitrasaRealTimeProcessor : AbstractRealTimeProcessor
                 .Where(e => !string.IsNullOrWhiteSpace(e.Route) && !e.Route.Trim().EndsWith('*'))
                 .ToList();
 
+            System.Diagnostics.Activity.Current?.SetTag("realtime.count", estimates.Count);
+
             var usedTripIds = new HashSet<string>();
             var newArrivals = new List<Arrival>();
 
diff --git a/src/Enmarcha.Backend/Telemetry.cs b/src/Enmarcha.Backend/Telemetry.cs
new file mode 100644
index 0000000..35c5348
--- /dev/null
+++ b/src/Enmarcha.Backend/Telemetry.cs
@@ -0,0 +1,8 @@
+using System.Diagnostics;
+
+namespace Enmarcha.Backend;
+
+public static class Telemetry
+{
+    public static readonly ActivitySource Source = new("Enmarcha.Backend");
+}