ECMJobRunner/ECMJobRunner.WebCron/Program.cs

using ECMJobRunner.Application;
using ECMJobRunner.Infrastructure;
using ECMJobRunner.WebCron;
using ECMJobRunner.WebCron.ProfileWorker;
using Hangfire;
using Hangfire.SqlServer;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Serilog;
using Serilog.Ui.Core.Extensions;
using Serilog.Ui.SqliteDataProvider.Extensions;
using Serilog.Ui.Web.Extensions;
using System.Text;

// Enable Serilog self-diagnostics
Serilog.Debugging.SelfLog.Enable(msg => Console.WriteLine($"[SERILOG] {msg}"));

// Build temporary configuration to read log directory
var tempConfig = new ConfigurationBuilder()
    .SetBasePath(Directory.GetCurrentDirectory())
    .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
    .Build();

// Get log directory from configuration
var logDirectory = tempConfig.GetValue<string>("Application:LogDirectory") 
    ?? throw new InvalidOperationException("Application:LogDirectory not found in configuration.");
var sqliteDbPath = Path.Combine(logDirectory, "logs.db");

Console.WriteLine($"[INFO] SQLite Log Database Path: {sqliteDbPath}");

// Configure Serilog with SQLite sink
Log.Logger = new LoggerConfiguration()
    .MinimumLevel.Information()
    .MinimumLevel.Override("Microsoft.AspNetCore", Serilog.Events.LogEventLevel.Warning)
    .MinimumLevel.Override("Microsoft.EntityFrameworkCore", Serilog.Events.LogEventLevel.Warning)
    .MinimumLevel.Override("Hangfire", Serilog.Events.LogEventLevel.Information)
    .WriteTo.Console(outputTemplate: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}")
    .WriteTo.SQLite(sqliteDbPath, storeTimestampInUtc: true)
    .Enrich.FromLogContext()
    .CreateLogger();

try
{
    Log.Information("Starting ECMJobRunner.WebCron application");

    var builder = WebApplication.CreateBuilder(args);

    // Use Serilog for logging
    builder.Host.UseSerilog();

    // Configure Windows Service hosting if enabled
    if (builder.Configuration.GetValue<bool>("HostingOptions:UseWindowsService"))
    {
        builder.Host.UseWindowsService();
    }

// Add services to the container.
builder.Services.AddControllers();
builder.Services.AddProfileWorker(builder.Configuration);

// Register services
var cnnStr = builder.Configuration.GetConnectionString("SDD-VMP04-SQL17")
    ?? throw new InvalidOperationException("Connection string 'SDD-VMP04-SQL17' not found.");
builder.Services.AddJobRunnerInfrastructure(cnnStr);

var recClientApiUrl = builder.Configuration.GetValue<string>("ReC:ApiUrl")
    ?? throw new InvalidOperationException("ReC:ApiUrl not found.");
builder.Services.AddJobRunnerServices(recClientApiUrl);

// Get Hangfire storage configuration
var useInMemory = builder.Configuration.GetValue<bool>("Hangfire:InMemory");

// Add Hangfire services with configurable storage
builder.Services.AddHangfire(configuration =>
{
    configuration
        .SetDataCompatibilityLevel(CompatibilityLevel.Version_180)
        .UseSimpleAssemblyNameTypeSerializer()
        .UseRecommendedSerializerSettings();

    // Configure storage based on appsettings
    if (useInMemory)
    {
        configuration.UseInMemoryStorage();
    }
    else // Use SQL Server storage
    {
        configuration.UseSqlServerStorage(cnnStr, new SqlServerStorageOptions
        {
            CommandBatchMaxTimeout = TimeSpan.FromMinutes(5),
            SlidingInvisibilityTimeout = TimeSpan.FromMinutes(5),
            QueuePollInterval = TimeSpan.Zero,
            UseRecommendedIsolationLevel = true,
            DisableGlobalLocks = true,
            SqlClientFactory = SqlClientFactory.Instance
        });
    }
});

// Add Hangfire server
builder.Services.AddHangfireServer();

builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();

// Add health checks
builder.Services.AddHealthChecks()
    .AddCheck<ProfileWorker>("profile-worker", tags: new[] { "ready", "worker" });

// Add Serilog.UI with SQLite provider - use same path from configuration
var serilogUiLogDirectory = builder.Configuration.GetValue<string>("Application:LogDirectory")
    ?? throw new InvalidOperationException("Application:LogDirectory not found in configuration.");
var serilogUiDbPath = Path.Combine(serilogUiLogDirectory, "logs.db");

builder.Services.AddSerilogUi(logUIOpt =>
{
    logUIOpt.UseSqliteServer(dbOpt =>
    {
        dbOpt.WithConnectionString($"Data Source={serilogUiDbPath}");
        dbOpt.WithTable("Logs");
    });
});

var app = builder.Build();

// Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment())
{
    app.UseSwagger();
    app.UseSwaggerUI();
}

app.UseHttpsRedirection();

// Enable static files for wwwroot
app.UseStaticFiles();

app.UseAuthorization();

// Add Hangfire Dashboard with no authentication (for development)
app.UseHangfireDashboard("/hangfire", new DashboardOptions
{
    Authorization = new[] { new AllowAllDashboardAuthorizationFilter() }
});

// Add Health Check UI route (accessible outside Hangfire dashboard)
app.MapGet("/health-ui", async (HealthCheckService healthCheckService, HttpContext context) =>
{
    var report = await healthCheckService.CheckHealthAsync(context.RequestAborted);
    
    var html = GenerateHealthCheckHtml(report);
    
    context.Response.ContentType = "text/html";
    await context.Response.WriteAsync(html);
    
    return Results.Empty;
});

// Add Serilog.UI Dashboard
app.UseSerilogUi();

app.MapControllers();

// Map health check endpoints
app.MapHealthChecks("/health", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
{
    Predicate = _ => true,
    ResponseWriter = async (context, report) =>
    {
        context.Response.ContentType = "application/json";
        var result = System.Text.Json.JsonSerializer.Serialize(new
        {
            status = report.Status.ToString(),
            timestamp = DateTime.UtcNow,
            checks = report.Entries.Select(e => new
            {
                name = e.Key,
                status = e.Value.Status.ToString(),
                description = e.Value.Description,
                duration = e.Value.Duration.TotalMilliseconds,
                exception = e.Value.Exception?.Message,
                data = e.Value.Data
            })
        });
        await context.Response.WriteAsync(result);
    }
});

app.MapHealthChecks("/health/ready", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
{
    Predicate = check => check.Tags.Contains("ready")
});

// Redirect root path to Health UI
app.MapGet("/", () => Results.Redirect("/health-ui"));

app.Run();
}
catch (Exception ex)
{
    Log.Fatal(ex, "Application terminated unexpectedly");
}
finally
{
    Log.CloseAndFlush();
}

// Helper method to generate health check HTML
static string GenerateHealthCheckHtml(HealthReport report)
{
    var sb = new StringBuilder();
    
    sb.AppendLine(@"
<!DOCTYPE html>
<html>
<head>
    <meta charset=""UTF-8"">
    <meta name=""viewport"" content=""width=device-width, initial-scale=1.0"">
    <title>Health Check - ECMJobRunner</title>
    <link rel=""stylesheet"" href=""https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css"" />
    <link rel=""stylesheet"" href=""/css/health-ui.css"" />
    <script src=""/js/health-ui.js""></script>
</head>
<body>");

    // Navigation
    sb.Append(@"
    <div class=""container-fluid"">
        <div class=""nav-links"">");
    
    // Append emojis separately (not in verbatim string)
    sb.AppendLine();
    sb.Append("            <a href=\"/\">🏠 Home</a>");
    sb.AppendLine();
    sb.Append("            <a href=\"/hangfire\">🔧 Hangfire Dashboard</a>");
    sb.AppendLine();
    sb.Append("            <a href=\"/serilog-ui\">📋 Logs (Serilog.UI)</a>");
    sb.AppendLine();
    sb.Append("            <a href=\"/health\">📊 Health API</a>");
    sb.AppendLine();
    sb.AppendLine("        </div>");

    // Overall Status Header
    var statusClass = report.Status switch
    {
        HealthStatus.Healthy => "status-healthy",
        HealthStatus.Degraded => "status-degraded",
        HealthStatus.Unhealthy => "status-unhealthy",
        _ => ""
    };

    var badgeClass = report.Status switch
    {
        HealthStatus.Healthy => "badge-healthy",
        HealthStatus.Degraded => "badge-degraded",
        HealthStatus.Unhealthy => "badge-unhealthy",
        _ => "badge-secondary"
    };

    var icon = report.Status switch
    {
        HealthStatus.Healthy => "✅",
        HealthStatus.Degraded => "⚠️",
        HealthStatus.Unhealthy => "❌",
        _ => "❓"
    };

    sb.Append($@"
        <div class=""header-card"">
            <div class=""d-flex justify-content-between align-items-center"">
                <div>
                    <h1 class=""mb-2"">");
    sb.Append(icon);
    sb.Append($@" Health Check Status</h1>
                    <p class=""text-muted mb-0"">Last checked: {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} UTC</p>
                </div>
                <div>
                    <h2><span class=""badge {badgeClass} fs-3"">{report.Status}</span></h2>
                </div>
            </div>
        </div>

        <div class=""row"">
            <div class=""col-md-4"">
                <div class=""card"">
                    <div class=""card-header bg-primary text-white"">");
    sb.AppendLine();
    sb.Append("                        📊 Summary");
    sb.AppendLine();
    sb.Append($@"                    </div>
                    <div class=""card-body"">
                        <div class=""metric"">
                            <span class=""metric-label"">Overall Status:</span>
                            <strong class=""{statusClass}"">{report.Status}</strong>
                        </div>
                        <div class=""metric"">
                            <span class=""metric-label"">Total Duration:</span>
                            <span class=""metric-value"">{report.TotalDuration.TotalMilliseconds:F0} ms</span>
                        </div>
                        <div class=""metric"">
                            <span class=""metric-label"">Checks:</span>
                            <span class=""metric-value"">{report.Entries.Count}</span>
                        </div>
                    </div>
                </div>
            </div>

            <div class=""col-md-8"">");

    // Individual Checks
    foreach (var entry in report.Entries)
    {
        var checkStatusClass = entry.Value.Status switch
        {
            HealthStatus.Healthy => "status-healthy",
            HealthStatus.Degraded => "status-degraded",
            HealthStatus.Unhealthy => "status-unhealthy",
            _ => ""
        };

        var checkBadgeClass = entry.Value.Status switch
        {
            HealthStatus.Healthy => "badge-healthy",
            HealthStatus.Degraded => "badge-degraded",
            HealthStatus.Unhealthy => "badge-unhealthy",
            _ => "badge-secondary"
        };

        var checkIcon = entry.Value.Status switch
        {
            HealthStatus.Healthy => "✅",
            HealthStatus.Degraded => "⚠️",
            HealthStatus.Unhealthy => "❌",
            _ => "❓"
        };

        sb.Append($@"
                <div class=""card"">
                    <div class=""card-header bg-light"">
                        <div class=""d-flex justify-content-between align-items-center"">
                            <span>");
        sb.Append(checkIcon);
        sb.Append($@" <strong>{entry.Key}</strong></span>
                            <span class=""badge {checkBadgeClass}"">{entry.Value.Status}</span>
                        </div>
                    </div>
                    <div class=""card-body"">
                        <div class=""metric"">
                            <span class=""metric-label"">Description:</span>
                            <span class=""metric-value"">{entry.Value.Description ?? "N/A"}</span>
                        </div>
                        <div class=""metric"">
                            <span class=""metric-label"">Duration:</span>
                            <span class=""metric-value"">{entry.Value.Duration.TotalMilliseconds:F0} ms</span>
                        </div>");

        if (entry.Value.Exception != null)
        {
            sb.AppendLine($@"
                        <div class=""metric"">
                            <span class=""metric-label"">Exception:</span>
                            <span class=""metric-value text-danger"" style=""word-break: break-all;"">{System.Net.WebUtility.HtmlEncode(entry.Value.Exception.Message)}</span>
                        </div>");
        }

        if (entry.Value.Data.Any())
        {
            sb.AppendLine(@"
                        <div class=""mt-3"">
                            <strong class=""metric-label"">Details:</strong>
                            <table class=""table table-sm table-borderless mt-2"">");

            foreach (var data in entry.Value.Data)
            {
                var value = data.Value?.ToString() ?? "N/A";
                
                // Format DateTime values
                if (data.Value is DateTime dt)
                {
                    value = dt.ToString("yyyy-MM-dd HH:mm:ss");
                }
                else if (data.Value is TimeSpan ts)
                {
                    value = $"{ts.TotalSeconds:F1}s";
                }

                sb.AppendLine($@"
                                <tr>
                                    <td class=""text-muted"" style=""width: 40%;"">{data.Key}</td>
                                    <td><strong>{value}</strong></td>
                                </tr>");
            }

            sb.AppendLine(@"
                            </table>
                        </div>");
        }

        sb.AppendLine(@"
                    </div>
                </div>");
    }

    sb.AppendLine(@"
            </div>
        </div>
    </div>

    <div class=""refresh-info"">");
    sb.AppendLine();
    sb.Append("        <span class=\"spinner\"></span> Auto-refresh in <strong><span id=\"countdown\">10</span>s</strong>");
    sb.AppendLine();
    sb.AppendLine(@"    </div>

</body>
</html>");

    return sb.ToString();
}