From 1e925242bc5d49133a14b8a27d2470a1913ba0f2 Mon Sep 17 00:00:00 2001 From: Jonathan Jenne Date: Tue, 20 Dec 2022 15:29:29 +0100 Subject: [PATCH] 20-12-2022 --- .../ECM.JobRunner.Common.xsd | 1 + .../JobRunnerReference/Reference.vb | 16 ++ .../Models/Jobs/StatusItem.vb | 57 ++++-- ECM.JobRunner.Web/Data/DashboardResponse.cs | 26 ++- ECM.JobRunner.Web/Data/DashboardService.cs | 29 +-- ECM.JobRunner.Web/ECM.JobRunner.Web.csproj | 4 - ECM.JobRunner.Web/Pages/History.razor | 172 +++++++++++++----- ECM.JobRunner.Web/Pages/Status.razor | 82 ++------- ECM.JobRunner.Web/Shared/MainLayout.razor | 3 +- ECM.JobRunner.Windows/Scheduler/JobStatus.vb | 50 +++-- .../Scheduler/Jobs/BaseJob.vb | 8 +- .../Scheduler/Jobs/FileImportJob.vb | 22 ++- 12 files changed, 291 insertions(+), 179 deletions(-) diff --git a/ECM.JobRunner.Common/Connected Services/JobRunnerReference/ECM.JobRunner.Common.xsd b/ECM.JobRunner.Common/Connected Services/JobRunnerReference/ECM.JobRunner.Common.xsd index 2325057..15da8f8 100644 --- a/ECM.JobRunner.Common/Connected Services/JobRunnerReference/ECM.JobRunner.Common.xsd +++ b/ECM.JobRunner.Common/Connected Services/JobRunnerReference/ECM.JobRunner.Common.xsd @@ -25,6 +25,7 @@ + diff --git a/ECM.JobRunner.Common/Connected Services/JobRunnerReference/Reference.vb b/ECM.JobRunner.Common/Connected Services/JobRunnerReference/Reference.vb index b8d5dce..4adcc95 100644 --- a/ECM.JobRunner.Common/Connected Services/JobRunnerReference/Reference.vb +++ b/ECM.JobRunner.Common/Connected Services/JobRunnerReference/Reference.vb @@ -322,6 +322,9 @@ Namespace JobRunnerReference _ Private UpdateTimeField As Date + _ + Private WaitingField As Boolean + _ Public Property ExtensionData() As System.Runtime.Serialization.ExtensionDataObject Implements System.Runtime.Serialization.IExtensibleDataObject.ExtensionData Get @@ -527,6 +530,19 @@ Namespace JobRunnerReference End Set End Property + _ + Public Property Waiting() As Boolean + Get + Return Me.WaitingField + End Get + Set + If (Me.WaitingField.Equals(value) <> true) Then + Me.WaitingField = value + Me.RaisePropertyChanged("Waiting") + End If + End Set + End Property + Public Event PropertyChanged As System.ComponentModel.PropertyChangedEventHandler Implements System.ComponentModel.INotifyPropertyChanged.PropertyChanged Protected Sub RaisePropertyChanged(ByVal propertyName As String) diff --git a/ECM.JobRunner.Common/Models/Jobs/StatusItem.vb b/ECM.JobRunner.Common/Models/Jobs/StatusItem.vb index 287197a..8bcdb74 100644 --- a/ECM.JobRunner.Common/Models/Jobs/StatusItem.vb +++ b/ECM.JobRunner.Common/Models/Jobs/StatusItem.vb @@ -9,35 +9,66 @@ Public Class StatusItem Public Const STEP_WARNING = "WARNING" Public Const STEP_ERROR = "ERROR" - ' Unique Job Run Id, GUID + ''' + ''' Is the job currently executing + ''' + Public Executing As Boolean = False + ''' + ''' Did the job complete without an error + ''' + Public Successful As Boolean = False + ''' + ''' Did the job do some work or is/was just waiting for some input + ''' + Public Waiting As Boolean = False + + ''' + ''' Unique Job Run Id, GUID + ''' Public Id As String - ' Job Id, corresponds to Job Schedule in DB + ''' + ''' Job Id, corresponds to Job Schedule in DB + ''' Public JobId As String - ' Job Name, corresponds to Job Schedule Key from Quartz + ''' + ''' Job Name, corresponds to Job Schedule Key from Quartz + ''' Public Name As String = "Unnamed" Public Steps As List(Of HistoryStep) ' Runtime Variables - ' Progress Counter + ''' + ''' Progress Counter + ''' Public ProgressCurrent As Integer = 0 - ' Total Progress + ''' + ''' Total Progress + ''' Public ProgressTotal As Integer = 0 - ' Flag to determin if the job is currently executing/working - Public Executing As Boolean = False - ' Creation time of job, set by Constructor + + ''' + ''' Creation time of job, set by Constructor + ''' Public CreationTime As Date = Date.Now - ' Start time of execution, set by JobStatus.Start + ''' + ''' Start time of execution, set by JobStatus.Start + ''' Public StartTime As Date - ' End time of execution, set by JobStatus.Complete + ''' + ''' End time of execution, set by JobStatus.Complete + ''' Public CompleteTime As Date - ' Time of last Progress Update + ''' + ''' Time of last Progress Update + ''' Public UpdateTime As Date - ' Total execution time, calculated by JobStatus.Complete + ''' + ''' Total execution time, calculated by JobStatus.Complete + ''' Public ExecutionTime As TimeSpan ' Completion/Failure Messages - Public Successful As Boolean = False Public SuccessMessage As String = "" Public FailureMessage As String = "" diff --git a/ECM.JobRunner.Web/Data/DashboardResponse.cs b/ECM.JobRunner.Web/Data/DashboardResponse.cs index 2c4fad7..4783616 100644 --- a/ECM.JobRunner.Web/Data/DashboardResponse.cs +++ b/ECM.JobRunner.Web/Data/DashboardResponse.cs @@ -5,14 +5,32 @@ namespace ECM.JobRunner.Web.Data public class DashboardResponse { public DateTime heartbeat = DateTime.MinValue; - public List jobHistory = new(); public List jobStatus = new(); - public List GetHistoryForLastMinutes(int pMinutes) + public class JobHistory { - return jobHistory. - Where(h => (DateTime.Now - h.CreatedAt) < new TimeSpan(0, pMinutes, 0)). + public List items; + public int total; + public int success; + public int failed; + public int waiting; + } + + public JobHistory GetHistoryForLastMinutes(int pMinutes) + { + var items = jobStatus. + Where(s => (DateTime.Now - s.CompleteTime) < new TimeSpan(0, pMinutes, 0)). + Where(s => s.Executing == false). ToList(); + + return new JobHistory() + { + items = items, + total = items.Count, + success = items.Where(i => i.Successful && i.Waiting == false).Count(), + failed = items.Where(i => i.Successful == false).Count(), + waiting = items.Where(i => i.Successful && i.Waiting == true).Count() + }; } } } diff --git a/ECM.JobRunner.Web/Data/DashboardService.cs b/ECM.JobRunner.Web/Data/DashboardService.cs index 12e45be..c0e40a3 100644 --- a/ECM.JobRunner.Web/Data/DashboardService.cs +++ b/ECM.JobRunner.Web/Data/DashboardService.cs @@ -17,7 +17,9 @@ namespace ECM.JobRunner.Web.Data { logger = Logging.LogConfig.GetLogger(); channel = Wcf.Channel; + pollingTimer.Interval = 1000; pollingTimer.Elapsed += PollingTimer_Elapsed; + pollingTimer.Enabled = true; } protected virtual void OnDataUpdated(DashboardResponse e) @@ -32,39 +34,14 @@ namespace ECM.JobRunner.Web.Data public async Task GetData() { - List jobHistory = await GetHistoryItems(); List jobStatus = await GetStatusItems(); return new DashboardResponse() { - jobHistory = jobHistory.OrderByDescending(e => e.CreatedAt).ToList(), - jobStatus = jobStatus.OrderByDescending(e => e.StartTime).ToList() + jobStatus = jobStatus.OrderByDescending(e => e.CompleteTime).ToList() }; } - private async Task> GetHistoryItems() - { - try - { - var oResponse = await channel.GetJobStatusAsync(); - - if (oResponse.OK) - { - return oResponse.HistoryItems.ToList(); - } - else - { - return new(); - } - } - catch (Exception e) - { - logger.Error(e); - return new(); - } - - } - private async Task> GetStatusItems() { try diff --git a/ECM.JobRunner.Web/ECM.JobRunner.Web.csproj b/ECM.JobRunner.Web/ECM.JobRunner.Web.csproj index 700a527..6908ee2 100644 --- a/ECM.JobRunner.Web/ECM.JobRunner.Web.csproj +++ b/ECM.JobRunner.Web/ECM.JobRunner.Web.csproj @@ -39,8 +39,4 @@ - - - - diff --git a/ECM.JobRunner.Web/Pages/History.razor b/ECM.JobRunner.Web/Pages/History.razor index 633290e..1c1ac37 100644 --- a/ECM.JobRunner.Web/Pages/History.razor +++ b/ECM.JobRunner.Web/Pages/History.razor @@ -7,58 +7,114 @@

Job History

-@if (historyEntries == null) -{ -
    -
  • Loading Job History..
    • -
-} -else if (historyEntries.Count == 0) -{ -
    -
  • No Job History yet.
    • -
-} -else -{ - @foreach (var entry in historyEntries) - { -
    -
  • -
    -
    - - - - - @entry.JobName -
    +
    +
    +
    +
    +
    +
    + Filter + +
    + + +
    +
    -
    • - @foreach (var step in entry.Steps) + + +
    + @if (filteredEntries == null) { -
  • - @step.Created.ToString("HH:mm") @step.Message -
    • +
      +
    • Loading Job History..
      • +
    } -
  • - @entry.CreatedAt.ToString("HH:mm") @entry.Message -
    • - @if (entry.Successful == false) + else if (filteredEntries.Count == 0) { -
  • Job Failed
    • +
      +
    • No Job History yet.
      • +
    } else { -
  • Job Succeeded
    • + @foreach (var entry in filteredEntries) + { +
      +
    • +
      +
      + @entry.Name +
      +
      +
      + + + + @entry.CompleteTime.ToString("HH:mm:ss") +
      +
      • + @foreach (var step in entry.Steps) + { +
    • + @step.Created.ToString("HH:mm:ss") @step.Message +
      • + } + @if (entry.Successful == false) + { +
    • + @entry.FailureMessage + + + + + @entry.ExecutionTime.ToString("mm':'ss") + +
      • + } + + @if (entry.Successful == true && entry.Waiting == false) + { +
    • + @entry.SuccessMessage + + + + + @entry.ExecutionTime.ToString("mm':'ss") + +
      • + } + + @if (entry.Successful == true && entry.Waiting == true) + { +
    • + @entry.SuccessMessage + + + + + @entry.ExecutionTime.ToString("mm':'ss") + +
      • + } +
    + } } -
- } -} + + + + + @code { private DateTime today; - private List? historyEntries; + private List? statusEntries; + private List? filteredEntries; + + private bool showSuccessful = true; + private bool showFailed = true; + private bool showWaiting = true; protected async override void OnInitialized() { @@ -68,7 +124,16 @@ else Api.DataUpdated += Api_DataUpdated; } - protected void Api_DataUpdated(object sender, DashboardResponse e) + private void CheckboxChanged(ChangeEventArgs e) + { + showWaiting = (bool)e.Value; + + UpdateEntries(statusEntries); + + InvokeAsync(StateHasChanged); + } + + protected void Api_DataUpdated(object? sender, DashboardResponse e) { UpdateData(e); } @@ -76,8 +141,27 @@ else protected void UpdateData(DashboardResponse response) { today = response.heartbeat; - historyEntries = response.jobHistory; - + statusEntries = response.jobStatus; + UpdateEntries(response.jobStatus); InvokeAsync(StateHasChanged); } + + protected void UpdateEntries(List? entries) + { + if (entries == null) + { + return; + } + + var filtered = entries. + Where(s => s.Executing == false); + + if (!showWaiting) + { + filtered = filtered. + Where(e => e.Waiting == false); + } + + filteredEntries = filtered.ToList(); + } } \ No newline at end of file diff --git a/ECM.JobRunner.Web/Pages/Status.razor b/ECM.JobRunner.Web/Pages/Status.razor index 859c926..2677146 100644 --- a/ECM.JobRunner.Web/Pages/Status.razor +++ b/ECM.JobRunner.Web/Pages/Status.razor @@ -6,8 +6,7 @@ Status - -
+
@if (last5MinutesItems != null) { @@ -19,7 +18,12 @@ Last 5 Minutes -
@last5MinutesItems.Count jobs executed.
+
+ @last5MinutesItems.total jobs executed, + @last5MinutesItems.success Successful, + @last5MinutesItems.failed Failed, + @last5MinutesItems.waiting Waiting +
@@ -34,7 +38,7 @@ Last Hour -
@lastHourItems.Count jobs executed.
+
@lastHourItems.total jobs executed, @lastHourItems.success Successful, @lastHourItems.failed Failed, @lastHourItems.waiting Waiting
@@ -43,14 +47,14 @@ @if (last12HoursItems != null) {
-
+
Last 12 Hours
-
@last12HoursItems.Count jobs executed.
+
@last12HoursItems.total jobs executed, @last12HoursItems.success Successful, @last12HoursItems.failed Failed, @last12HoursItems.waiting Waiting
@@ -58,12 +62,9 @@
-

Job Status

- -
+
@if (executingEntries == null) { -

Executing

  • Loading Job Status..
    • @@ -71,7 +72,6 @@ } else if (executingEntries.Count == 0) { -

    Executing (0)

    • @@ -83,8 +83,6 @@ } else { -

      Executing (@executingEntries.Count)

      -
        @foreach (var entry in executingEntries) { @@ -109,64 +107,14 @@
-@if (completedEntries == null) -{ -

Completed

- -
    -
  • Loading Job Status..
    • -
-} -else if (completedEntries.Count == 0) -{ -

Completed (0)

- -
    -
  • - - - - No completed jobs yet. -
    • -
-} -else -{ -

Completed (@completedEntries.Count)

- -
    - @foreach (var entry in completedEntries) - { -
  • -
    -
    - - - - - @entry.Name -
    -
      -
    • Execution Time: @((int)entry.ExecutionTime.TotalSeconds)s
      • -
    • Started: @entry.StartTime.ToLongTimeString()
      • -
    • Completed: @entry.CompleteTime.ToLongTimeString()
      • -
    -
    - Completed @entry.CompleteTime.ToLongTimeString() -
    • - } -
-} - - @code { private DateTime today; private List? executingEntries; private List? completedEntries; - private List? last5MinutesItems; - private List? lastHourItems; - private List? last12HoursItems; + private DashboardResponse.JobHistory? last5MinutesItems; + private DashboardResponse.JobHistory? lastHourItems; + private DashboardResponse.JobHistory? last12HoursItems; protected async override void OnInitialized() { @@ -201,4 +149,6 @@ else InvokeAsync(StateHasChanged); } + + } diff --git a/ECM.JobRunner.Web/Shared/MainLayout.razor b/ECM.JobRunner.Web/Shared/MainLayout.razor index 8b54c62..8012dcf 100644 --- a/ECM.JobRunner.Web/Shared/MainLayout.razor +++ b/ECM.JobRunner.Web/Shared/MainLayout.razor @@ -10,7 +10,8 @@
-
+
+ Page Title About
diff --git a/ECM.JobRunner.Windows/Scheduler/JobStatus.vb b/ECM.JobRunner.Windows/Scheduler/JobStatus.vb index de0bae4..c72e7bf 100644 --- a/ECM.JobRunner.Windows/Scheduler/JobStatus.vb +++ b/ECM.JobRunner.Windows/Scheduler/JobStatus.vb @@ -47,16 +47,26 @@ Public Class JobStatus Logger.Info("Completing Job [{0}] with Error", oStatus.Id) If oStatus IsNot Nothing Then - oStatus.ProgressCurrent = oStatus.ProgressTotal - oStatus.ExecutionTime = pJob.JobRunTime - oStatus.Executing = False - oStatus.CompleteTime = Date.Now oStatus.FailureMessage = pMessage oStatus.Successful = False - oStatus.Steps = pSteps + oStatus.Waiting = False End If - Return oStatus + Return DoComplete(pJob, pSteps) + End Function + + Public Function CompleteWithWaiting(pJob As Quartz.IJobExecutionContext, pSteps As List(Of StatusItem.HistoryStep), pMessage As String) As StatusItem + Dim oStatus = GetJobStatus(pJob) + + Logger.Info("Completing Job [{0}] with Waiting", oStatus.Id) + + If oStatus IsNot Nothing Then + oStatus.SuccessMessage = pMessage + oStatus.Successful = True + oStatus.Waiting = True + End If + + Return DoComplete(pJob, pSteps) End Function Public Function CompleteWithSuccess(pJob As Quartz.IJobExecutionContext, pSteps As List(Of StatusItem.HistoryStep), pMessage As String) As StatusItem @@ -65,23 +75,35 @@ Public Class JobStatus Logger.Info("Completing Job [{0}] with Success", oStatus.Id) If oStatus IsNot Nothing Then - oStatus.ProgressCurrent = oStatus.ProgressTotal - oStatus.ExecutionTime = pJob.JobRunTime - oStatus.Executing = False - oStatus.CompleteTime = Date.Now oStatus.SuccessMessage = pMessage - oStatus.Successful = False - oStatus.Steps = pSteps + oStatus.Successful = True + oStatus.Waiting = False End If + Return DoComplete(pJob, pSteps) + End Function + + Private Function DoComplete(pJob As Quartz.IJobExecutionContext, pSteps As List(Of StatusItem.HistoryStep)) + Dim oStatus = GetJobStatus(pJob) + + oStatus.ProgressCurrent = oStatus.ProgressTotal + oStatus.ExecutionTime = pJob.JobRunTime + oStatus.Executing = False + oStatus.CompleteTime = Date.Now + oStatus.Steps = pSteps + Return oStatus End Function Private Function GetJobStatus(pJob As Quartz.IJobExecutionContext) As StatusItem - Dim oJobId = GetJobId(pJob) + Dim oJobId As String = GetJobId(pJob) + Logger.Debug("Getting status for job id [{0}]", oJobId) + Dim oExists = Entries.Where(Function(e) e.JobId = oJobId).Any() + Logger.Debug("Job exists: [{0}]", oExists) If Not oExists Then + Logger.Debug("Creating status for job id [{0}]", oJobId) Entries.Add(New StatusItem With { .JobId = oJobId, .Id = Guid.NewGuid.ToString(), @@ -89,7 +111,7 @@ Public Class JobStatus }) End If - Return Entries.Where(Function(e) e.Id = oJobId).SingleOrDefault() + Return Entries.Where(Function(e) e.JobId = oJobId).Single() End Function Private Function GetJobId(pJob As Quartz.IJobExecutionContext) As String diff --git a/ECM.JobRunner.Windows/Scheduler/Jobs/BaseJob.vb b/ECM.JobRunner.Windows/Scheduler/Jobs/BaseJob.vb index c0ca8cf..2fd8361 100644 --- a/ECM.JobRunner.Windows/Scheduler/Jobs/BaseJob.vb +++ b/ECM.JobRunner.Windows/Scheduler/Jobs/BaseJob.vb @@ -29,11 +29,12 @@ Namespace Scheduler.Jobs Database = oJobData.Item(Constants.Scheduler.JOB_CONFIG_DATABASE) State = oJobData.Item(Constants.Scheduler.JOB_CONFIG_STATE) Windream = oJobData.Item(Constants.Scheduler.JOB_CONFIG_WINDREAM) - Logger = LogConfig.GetLogger() ExecutionId = Guid.NewGuid.ToString() Id = Integer.Parse(oArgs.Item("Id")) Name = oArgs.Item("Name") + Logger = LogConfig.GetLogger(Name) + State.JobStatus.Start(ctx) Logger.Info("Job [{0}] is starting!", Id) @@ -83,6 +84,11 @@ Namespace Scheduler.Jobs Return Task.FromResult(True) End Function + Public Function CompleteJobWithWaiting(pMessage As String) As Task(Of Boolean) + ctx.Result = State.JobStatus.CompleteWithWaiting(ctx, JobSteps, pMessage) + Return Task.FromResult(True) + End Function + Public Function CompleteJobWithError(pException As Exception) As Task(Of Boolean) ctx.Result = State.JobStatus.CompleteWithError(ctx, JobSteps, pException) diff --git a/ECM.JobRunner.Windows/Scheduler/Jobs/FileImportJob.vb b/ECM.JobRunner.Windows/Scheduler/Jobs/FileImportJob.vb index 4fd9a97..9a15acd 100644 --- a/ECM.JobRunner.Windows/Scheduler/Jobs/FileImportJob.vb +++ b/ECM.JobRunner.Windows/Scheduler/Jobs/FileImportJob.vb @@ -29,7 +29,7 @@ Namespace Scheduler.Jobs If IO.Directory.Exists(oProfile.SourceFolder) = False Then LogError("Source directory [{0}] does not exist!", oProfile.SourceFolder) - Return Task.FromResult(False) + Return CompleteJobWithError(New IO.DirectoryNotFoundException($"Source directory [{oProfile.SourceFolder}] does not exist!")) End If Dim oRecursive As Boolean = oProfile.IncludeSubfolders @@ -40,24 +40,24 @@ Namespace Scheduler.Jobs If oFileNames.Count = 0 Then Logger.Info("No Files for Profile [{0}]", Name) - Return CompleteJob("No files for profile") + Return CompleteJobWithWaiting("No files for profile") End If - LogInfo("{0} files found in source directory {1}", oFileNames.Count, oProfile.SourceFolder) + LogInfo("{0} files found in source directory {1}", oFileNames.Count.ToString, oProfile.SourceFolder) ' - [ ] Process Rules, build list of files and indexes ' - [x] Process Regex to filter out files ' - [x] Check time to filter out files ' - [ ] (Check if files can be accessed) ' - [ ] Check if backup is needed and backup files - ' - [ ] Import into windream + ' - [x] Import into windream ' - [ ] Create original subfolder structure ' - [ ] Create DateTime Subfolders ' - [x] Check if file exists and version ' - [x] Do import ' - [ ] Check for filesize 0 - ' - [ ] Write indexes (using data from getimportfile) - ' - [ ] Check if orig file should be deleted + ' - [x] Write indexes (using data from getimportfile) + ' - [x] Check if orig file should be deleted ' - [ ] (delete subdirectories in source path) Dim oFiles = oFileNames. @@ -71,12 +71,22 @@ Namespace Scheduler.Jobs LogDebug("{0} Files filtered out for being too new.", oDateFilteredCount) oFilteredFiles = oDateFilteredFiles + If oFilteredFiles.Count = 0 Then + Logger.Info("No Files for Profile [{0}]", Name) + Return CompleteJobWithWaiting("No files for profile") + End If + ' Process Regex to filter out files Dim oRegexFilteredFiles = oFilteredFiles.Where(Function(f) Not FileMatchesRegex(f, oRegexList)) Dim oRegexFilteredCount = oFilteredFiles.Except(oRegexFilteredFiles).Count() LogDebug("{0} Files filtered out for matching exclusion Regex.", oRegexFilteredCount) oFilteredFiles = oDateFilteredFiles + If oFilteredFiles.Count = 0 Then + Logger.Info("No Files for Profile [{0}]", Name) + Return CompleteJobWithWaiting("No files for profile") + End If + '------------------------------------------------------------------------------------------------- ' After this point, files are treated as processed and are being deleted before finishing the job '-------------------------------------------------------------------------------------------------