06-12-2022

This commit is contained in:
Jonathan Jenne
2022-12-06 14:08:20 +01:00
parent c867e4e3a6
commit 248be23804
46 changed files with 1513 additions and 347 deletions

View File

@@ -3,8 +3,9 @@
Public Const SERVICE_DISPLAY_NAME As String = "Digital Data ECM JobRunner Service"
Public Class Scheduler
Public Const JOB_CONFIG_LOGCONFIG = "LogConfig"
Public Const JOB_CONFIG_ARGUMENTS = "Arguments"
Public Const JOB_CONFIG_DATABASE = "Database"
Public Const JOB_CONFIG_LOGCONFIG = "__LogConfig"
Public Const JOB_CONFIG_ARGUMENTS = "__Arguments"
Public Const JOB_CONFIG_DATABASE = "__Database"
Public Const JOB_CONFIG_STATE = "__State"
End Class
End Class

View File

@@ -129,6 +129,7 @@
<Compile Include="Scheduler\JobListener.vb" />
<Compile Include="Scheduler\JobResult.vb" />
<Compile Include="Scheduler\JobScheduler.vb" />
<Compile Include="Scheduler\JobStatus.vb" />
<Compile Include="Scheduler\Jobs\DebugJob.vb" />
<Compile Include="Scheduler\Jobs\BaseJob.vb" />
<Compile Include="Scheduler\Jobs\FileImportJob.vb" />
@@ -155,7 +156,8 @@
<Compile Include="WCF\JobRunner.vb" />
<Compile Include="WCF\Methods\Base.vb" />
<Compile Include="WCF\Methods\GetJobConfig.vb" />
<Compile Include="WCF\Methods\GetJobHistory.vb" />
<Compile Include="WCF\Methods\UpdateJob.vb" />
<Compile Include="WCF\Methods\GetJobStatus.vb" />
<Compile Include="WCF\ServiceHost.vb" />
</ItemGroup>
<ItemGroup>

View File

@@ -1,9 +1,8 @@
Imports System.Collections.Specialized
Imports System.Threading
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Logging
Imports ECM.JobRunner.Common
Imports Quartz
Imports Quartz.Logging.OperationName
Namespace Scheduler
Public Class JobScheduler
@@ -44,44 +43,7 @@ Namespace Scheduler
Await Scheduler.Start()
' load job Config and setup job schedules
Logger.Info("Loading [{0}] Job Definitions..", State.JobDefinitions.Count)
For Each oJob In State.JobDefinitions
Logger.Debug("Loading Job Definition [{0}]", oJob.Name)
Select Case oJob.TypeId
Case JOB_TYPE_IMPORT
Await ScheduleJob(Of Jobs.FileImportJob)(New JobConfig With {
.Name = oJob.Name,
.Enabled = True,
.Arguments = New Dictionary(Of String, String) From {
{"Name", oJob.Name}
},
.CronSchedule = oJob.CronSchedule
})
Case JOB_TYPE_INDEX
Await ScheduleJob(Of Jobs.FileIndexJob)(New JobConfig With {
.Name = oJob.Name,
.Enabled = True,
.Arguments = New Dictionary(Of String, String) From {
{"Name", oJob.Name}
},
.CronSchedule = oJob.CronSchedule
})
End Select
Next
' setup debug job
Await ScheduleJob(Of Jobs.DebugJob)(New JobConfig With {
.Name = "Debug Job",
.Enabled = True,
.Arguments = New Dictionary(Of String, String) From {{"Arg1", "My awesome argument"}},
.CronSchedule = "0 * * * * ?"
})
ScheduleJobs()
Return True
Catch ex As Exception
@@ -91,57 +53,141 @@ Namespace Scheduler
End Try
End Function
Public Sub Reload()
' first reload the data from db
State.Reload()
' now reschedule jobs
ScheduleJobs()
End Sub
Public Async Function Shutdown() As Task
Await Scheduler.Shutdown()
End Function
Public Async Function ScheduleJob(Of T As IJob)(pJobConfig As JobConfig) As Task
Dim oJobName As String = pJobConfig.Name
Dim oTriggerName As String = $"{oJobName}-TRIGGER"
Public Async Function GetRunningJobs() As Task(Of IReadOnlyCollection(Of IJobExecutionContext))
Return Await Scheduler.GetCurrentlyExecutingJobs()
End Function
pJobConfig.Name = oJobName
Private Async Sub ScheduleJobs()
Dim oJobData As New JobDataMap From {
{Constants.Scheduler.JOB_CONFIG_LOGCONFIG, LogConfig},
{Constants.Scheduler.JOB_CONFIG_ARGUMENTS, pJobConfig.Arguments},
{Constants.Scheduler.JOB_CONFIG_DATABASE, Database}
Logger.Info("Loading [{0}] Job Definitions..", State.JobDefinitions.Count)
For Each oJob In State.JobDefinitions
Logger.Debug("Loading Job Definition [{0}]", oJob.Name)
Logger.Debug("Job Type is [{0}]", oJob.Type.Name)
Select Case oJob.TypeId
Case JOB_TYPE_IMPORT
Dim oJobConfig = BuildJobConfig(Of Jobs.FileImportJob)(oJob)
Await ScheduleJob(Of Jobs.FileImportJob)(oJobConfig)
Case JOB_TYPE_INDEX
Dim oJobConfig = BuildJobConfig(Of Jobs.FileIndexJob)(oJob)
Await ScheduleJob(Of Jobs.FileIndexJob)(oJobConfig)
Case Else
Logger.Warn("Job for TypeId [{0}] is not implemented!", oJob.TypeId)
End Select
Next
End Sub
Private Function BuildJobConfig(Of TJob As IJob)(pJob As JobDefinition) As JobConfig
Return New JobConfig With {
.Name = pJob.Name,
.Enabled = pJob.Active,
.Arguments = New Dictionary(Of String, String) From {
{"Name", pJob.Name}
},
.CronSchedule = pJob.CronSchedule
}
End Function
Dim oJob = JobBuilder.Create(Of T)().
WithIdentity(oJobName).
UsingJobData(oJobData).
Build()
Private Async Function ScheduleJob(Of T As IJob)(pJobConfig As JobConfig) As Task
If Await Scheduler.CheckExists(New JobKey(GetJobName(pJobConfig))) Then
Logger.Debug("Job already exists, rescheduling..")
Await DoRescheduleJob(Of T)(pJobConfig)
Else
Logger.Debug("Job does not exist, scheduling..")
Await DoScheduleJob(Of T)(pJobConfig)
End If
End Function
Dim oTrigger = TriggerBuilder.Create().
WithIdentity(oTriggerName).
StartNow().
WithCronSchedule(pJobConfig.CronSchedule).
Build()
''' <summary>
''' Updates the Trigger and Schedule for a given JobConfig
''' </summary>
''' <typeparam name="T"></typeparam>
''' <param name="pJobConfig"></param>
''' <returns></returns>
Private Async Function DoRescheduleJob(Of T As IJob)(pJobConfig As JobConfig) As Task
Dim oJobKey As New JobKey(pJobConfig.Name)
Dim oTriggerKey As New TriggerKey(GetTriggerName(pJobConfig))
Dim oTrigger = BuildTrigger(pJobConfig)
Await Scheduler.RescheduleJob(oTriggerKey, oTrigger)
End Function
Private Async Function DoScheduleJob(Of T As IJob)(pJobConfig As JobConfig) As Task
Dim oJobName As String = GetJobName(pJobConfig)
Dim oTriggerName As String = GetTriggerName(pJobConfig)
Dim oJobData As JobDataMap = BuildJobData(pJobConfig)
Dim oJob As IJobDetail = BuildJob(Of T)(pJobConfig, oJobData)
Dim oTrigger As ITrigger = BuildTrigger(pJobConfig)
If pJobConfig.Enabled Then
Await Scheduler.ScheduleJob(oJob, oTrigger)
Logger.Info("Job {0} scheduled.", oJobName)
Logger.Info("Job [{0}] scheduled.", oJobName)
Else
Logger.Info("Job {0} is disabled.", oJobName)
Logger.Info("Job [{0}] is disabled.", oJobName)
End If
If pJobConfig.StartWithoutDelay Then
Dim oDebugJob = JobBuilder.Create(Of T)().
WithIdentity(oJobName & "-DEBUG").
UsingJobData(oJobData).
Build()
Dim oDebugTrigger = TriggerBuilder.Create().
WithIdentity(oTriggerName & "-DEBUG").
Dim oNoDelayTrigger = TriggerBuilder.Create().
WithIdentity(oTriggerName & "-NO-DELAY").
StartAt(DateBuilder.FutureDate(10, IntervalUnit.Second)).
Build()
Logger.Info("Job {0} will start in 10 Seconds.", oJobName)
Await Scheduler.ScheduleJob(oDebugJob, oDebugTrigger)
Await Scheduler.ScheduleJob(oJob, oNoDelayTrigger)
End If
End Function
Private Function BuildJobData(pJobConfig As JobConfig) As JobDataMap
Return New JobDataMap From {
{Constants.Scheduler.JOB_CONFIG_LOGCONFIG, LogConfig},
{Constants.Scheduler.JOB_CONFIG_ARGUMENTS, pJobConfig.Arguments},
{Constants.Scheduler.JOB_CONFIG_DATABASE, Database},
{Constants.Scheduler.JOB_CONFIG_STATE, State}
}
End Function
Private Function BuildJob(Of T As IJob)(pJobConfig As JobConfig, pJobData As JobDataMap) As IJobDetail
Dim oJobName = GetJobName(pJobConfig)
Return JobBuilder.Create(Of T)().
WithIdentity(pJobConfig.Name).
UsingJobData(pJobData).
Build()
End Function
Private Function BuildTrigger(pJobConfig As JobConfig) As ITrigger
Dim oTriggerName As String = GetTriggerName(pJobConfig)
Return TriggerBuilder.Create().
WithIdentity(oTriggerName).
WithCronSchedule(pJobConfig.CronSchedule).
StartNow().
Build()
End Function
Private Function GetJobName(pJobConfig As JobConfig) As String
Return pJobConfig.Name
End Function
Private Function GetTriggerName(pJobConfig As JobConfig) As String
Dim oJobName As String = pJobConfig.Name
Return $"{oJobName}-TRIGGER"
End Function
End Class
End Namespace

View File

@@ -0,0 +1,65 @@
Imports DigitalData.Modules.Base
Imports DigitalData.Modules.Logging
Imports ECM.JobRunner.Common
Public Class JobStatus
Inherits BaseClass
Public ReadOnly Entries As New List(Of StatusItem)
Public Sub New(pLogConfig As LogConfig)
MyBase.New(pLogConfig)
End Sub
Public Sub Start(pJob As Quartz.IJobExecutionContext)
Dim oStatus = GetJobStatus(pJob)
Logger.Info("Starting Job [{0}]", oStatus.Id)
If oStatus IsNot Nothing Then
oStatus.Name = pJob.JobDetail.Key.Name
oStatus.StartTime = Date.Now
oStatus.Executing = True
End If
End Sub
Public Sub Update(pJob As Quartz.IJobExecutionContext, pCurrent As Integer, pTotal As Integer)
Dim oStatus = GetJobStatus(pJob)
Logger.Debug("Updating Job [{0}] with Status [{1}/{2}]", oStatus.Id, pCurrent, pTotal)
If oStatus IsNot Nothing Then
oStatus.ProgressTotal = pTotal
oStatus.ProgressCurrent = pCurrent
oStatus.ExecutionTime = pJob.JobRunTime
End If
End Sub
Public Sub Complete(pJob As Quartz.IJobExecutionContext)
Dim oStatus = GetJobStatus(pJob)
Logger.Info("Completing Job [{0}]", oStatus.Id)
If oStatus IsNot Nothing Then
oStatus.ProgressCurrent = oStatus.ProgressTotal
oStatus.ExecutionTime = pJob.JobRunTime
oStatus.Executing = False
oStatus.CompleteTime = Date.Now
End If
End Sub
Private Function GetJobStatus(pJob As Quartz.IJobExecutionContext) As StatusItem
Dim oJobId = GetJobId(pJob)
Dim oExists = Entries.Where(Function(e) e.Id = oJobId).Any()
If Not oExists Then
Entries.Add(New StatusItem With {.Id = oJobId})
End If
Return Entries.Where(Function(e) e.Id = oJobId).SingleOrDefault()
End Function
Private Function GetJobId(pJob As Quartz.IJobExecutionContext) As String
Return pJob.JobDetail.Key.ToString() & pJob.FireTimeUtc.ToString("u")
End Function
End Class

View File

@@ -1,5 +1,6 @@
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Logging
Imports ECM.JobRunner.Common
Imports Quartz
Namespace Scheduler.Jobs
@@ -7,15 +8,30 @@ Namespace Scheduler.Jobs
Friend LogConfig As LogConfig
Friend Logger As Logger
Friend Database As MSSQLServer
Friend State As State
Private ctx As IJobExecutionContext
Public Function InitializeJob(context As IJobExecutionContext) As Dictionary(Of String, String)
ctx = context
Dim oJobData = context.MergedJobDataMap
LogConfig = oJobData.Item(Constants.Scheduler.JOB_CONFIG_LOGCONFIG)
Database = oJobData.Item(Constants.Scheduler.JOB_CONFIG_DATABASE)
State = oJobData.Item(Constants.Scheduler.JOB_CONFIG_STATE)
Logger = LogConfig.GetLogger()
State.JobStatus.Start(ctx)
Return oJobData.Item(Constants.Scheduler.JOB_CONFIG_ARGUMENTS)
End Function
Public Sub UpdateProgress(pCurrentValue As Integer, pTotalValue As Integer)
State.JobStatus.Update(ctx, pCurrentValue, pTotalValue)
End Sub
Public Sub CompleteJob()
State.JobStatus.Complete(ctx)
End Sub
End Class
End Namespace

View File

@@ -19,6 +19,7 @@ Namespace Scheduler.Jobs
context.Result = oResult
CompleteJob()
Return Task.FromResult(True)
End Function
End Class

View File

@@ -13,12 +13,19 @@ Namespace Scheduler.Jobs
Logger.Info("Running File Import [{0}]", oName)
Dim oMax = 100
For index = 1 To oMax
UpdateProgress(index, oMax)
Threading.Thread.Sleep(100)
Next
Dim oResult = New JobResult() With {
.Description = $"File Import Job [{oName}] completed!"
}
context.Result = oResult
CompleteJob()
Return Task.FromResult(True)
End Function
End Class

View File

@@ -18,6 +18,7 @@ Namespace Scheduler.Jobs
context.Result = oResult
CompleteJob()
Return Task.FromResult(True)
End Function
End Class

View File

@@ -52,6 +52,7 @@ Public Class Service
WCF.JobRunner.State = State
WCF.JobRunner.LogConfig = LogConfig
WCF.JobRunner.Database = Database
WCF.JobRunner.Scheduler = Scheduler
' start the service
Dim oAddresses() As Uri = {Binding.GetAddress(Config.Host, Config.Port, Config.Name)}

View File

@@ -12,6 +12,8 @@ Public Class State
Private ReadOnly Database As MSSQLServer
Public ReadOnly JobHistory As JobHistory
Public ReadOnly JobStatus As JobStatus
Public ReadOnly Property JobTypes As New List(Of JobType)
Public ReadOnly Property JobDefinitions As New List(Of JobDefinition)
@@ -19,6 +21,7 @@ Public Class State
MyBase.New(pLogConfig)
Database = pDatabase
JobHistory = New JobHistory(pLogConfig)
JobStatus = New JobStatus(pLogConfig)
_JobTypes = GetJobTypes()
_JobDefinitions = GetJobDefinitions(_JobTypes)
@@ -68,6 +71,8 @@ Public Class State
Dim oJobs As New List(Of JobDefinition)
Try
Logger.Info("Loading Jobs..")
Dim oSQL As String = "SELECT * FROM TBECM_JR_JOB"
Dim oTable As DataTable = Database.GetDatatable(oSQL)

View File

@@ -1,5 +1,8 @@
Imports System.ServiceModel
Imports ECM.JobRunner.Common
Imports ECM.JobRunner.Windows.UpdateJob
Imports ECM.JobRunner.Windows.GetJobStatus
Imports ECM.JobRunner.Windows.GetJobConfig
Namespace WCF
<ServiceContract(Name:="IEDMIService", [Namespace]:="http://DigitalData.Services.EDMIService")>
@@ -9,10 +12,13 @@ Namespace WCF
Function GetHeartbeat() As Date
<OperationContract>
Function GetJobHistory() As GetJobHistory.GetJobHistoryResponse
Function GetJobStatus() As GetJobStatusResponse
<OperationContract>
Function GetJobConfig() As GetJobConfig.GetJobConfigResponse
Function UpdateJob(pData As UpdateJobRequest) As UpdateJobResponse
<OperationContract>
Function GetJobConfig() As GetJobConfigResponse
End Interface
End Namespace

View File

@@ -3,7 +3,7 @@ Imports System.ServiceModel.Description
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Logging
Imports DigitalData.Modules.Messaging.WCF
Imports ECM.JobRunner.Common
Imports ECM.JobRunner.Windows.Scheduler
Namespace WCF
@@ -13,6 +13,7 @@ Namespace WCF
Public Shared State As State
Public Shared LogConfig As LogConfig
Public Shared Database As MSSQLServer
Public Shared Scheduler As JobScheduler
''' <summary>
''' See: https://stackoverflow.com/questions/42327988/addserviceendpoint-throws-key-is-null
@@ -35,8 +36,8 @@ Namespace WCF
Return Now
End Function
Public Function GetJobHistory() As GetJobHistory.GetJobHistoryResponse Implements IJobRunner.GetJobHistory
Dim oMethod As New GetJobHistory.GetJobHistoryMethod(LogConfig, Database, State)
Public Function GetJobHistory() As GetJobStatus.GetJobStatusResponse Implements IJobRunner.GetJobStatus
Dim oMethod As New GetJobStatus.GetJobStatusMethod(LogConfig, Database, State, Scheduler)
Return oMethod.Run()
End Function
@@ -44,6 +45,11 @@ Namespace WCF
Dim oMethod As New GetJobConfig.GetJobConfigMethod(LogConfig, Database, State)
Return oMethod.Run()
End Function
Public Function UpdateJob(pData As UpdateJob.UpdateJobRequest) As UpdateJob.UpdateJobResponse Implements IJobRunner.UpdateJob
Dim oMethod As New UpdateJob.UpdateJobMethod(LogConfig, Database, State, Scheduler)
Return oMethod.Run(pData)
End Function
End Class
End Namespace

View File

@@ -18,8 +18,6 @@ Public Class GetJobConfig
.JobDefinitions = State.JobDefinitions
}
End Function
End Class
Public Class GetJobConfigResponse

View File

@@ -1,25 +0,0 @@
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Logging
Imports ECM.JobRunner.Common
Imports System.Runtime.Serialization
Public Class GetJobHistory
Public Class GetJobHistoryMethod
Inherits Base.BaseMethod
Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pState As State)
MyBase.New(pLogConfig, pDatabase, pState)
End Sub
Public Function Run() As GetJobHistoryResponse
Return New GetJobHistoryResponse With {.Items = State.JobHistory.Entries}
End Function
End Class
Public Class GetJobHistoryResponse
Inherits Base.BaseResponse
<DataMember>
Public Property Items As List(Of HistoryItem)
End Class
End Class

View File

@@ -0,0 +1,37 @@
Imports System.Collections.ObjectModel
Imports System.Runtime.Serialization
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Logging
Imports ECM.JobRunner.Common
Imports ECM.JobRunner.Windows.Scheduler
Public Class GetJobStatus
Public Class GetJobStatusMethod
Inherits Base.BaseMethod
Private ReadOnly Scheduler As JobScheduler
Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pState As State, pScheduler As JobScheduler)
MyBase.New(pLogConfig, pDatabase, pState)
Scheduler = pScheduler
End Sub
Public Function Run() As GetJobStatusResponse
Return New GetJobStatusResponse With {
.HistoryItems = State.JobHistory.Entries,
.StatusItems = State.JobStatus.Entries
}
End Function
End Class
Public Class GetJobStatusResponse
Inherits Base.BaseResponse
<DataMember>
Public Property HistoryItems As List(Of HistoryItem)
<DataMember>
Public Property StatusItems As List(Of StatusItem)
End Class
End Class

View File

@@ -0,0 +1,93 @@
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Logging
Imports ECM.JobRunner.Common
Imports ECM.JobRunner.Windows.Scheduler
Public Class UpdateJob
Public Class UpdateJobMethod
Inherits Base.BaseMethod
Private ReadOnly Scheduler As JobScheduler
Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pState As State, pScheduler As JobScheduler)
MyBase.New(pLogConfig, pDatabase, pState)
Scheduler = pScheduler
End Sub
Public Function Run(pData As UpdateJobRequest) As UpdateJobResponse
Dim oResponse As Boolean = False
Select Case pData.Action
Case UpdateJobRequest.UpdateJobAction.Update
oResponse = DoUpdateJob(pData)
Case UpdateJobRequest.UpdateJobAction.Create
oResponse = DoCreateJob(pData)
Case UpdateJobRequest.UpdateJobAction.Delete
oResponse = DoDeleteJob(pData)
End Select
If oResponse Then
Scheduler.Reload()
End If
Return New UpdateJobResponse With {.OK = oResponse}
End Function
Private Function DoUpdateJob(pData As UpdateJobRequest) As Boolean
Dim oJob = pData.Job
Dim oSQL As String = "UPDATE TBECM_JR_JOB SET TITLE = @TITLE, QUARTZ_DEF = @CRON, JOB_TYPE_ID = @TYPE_ID, ACTIVE = @ACTIVE WHERE GUID = @GUID"
Dim oCommand As New SqlClient.SqlCommand(oSQL)
oCommand.Parameters.Add("TITLE", SqlDbType.NVarChar, 250).Value = oJob.Name
oCommand.Parameters.Add("CRON", SqlDbType.NVarChar, 250).Value = oJob.CronSchedule
oCommand.Parameters.Add("TYPE_ID", SqlDbType.Int).Value = oJob.TypeId
oCommand.Parameters.Add("ACTIVE", SqlDbType.Bit).Value = oJob.Active
oCommand.Parameters.Add("GUID", SqlDbType.Int).Value = oJob.Id
Return Database.ExecuteNonQuery(oCommand)
End Function
Private Function DoCreateJob(pData As UpdateJobRequest) As Boolean
Dim oJob = pData.Job
Dim oSQL As String = "INSERT INTO TBECM_JR_JOB (TITLE, QUARTZ_DEF, JOB_TYPE_ID, ACTIVE) VALUES (@TITLE, @CRON, @TYPE_ID, @ACTIVE)"
Dim oCommand As New SqlClient.SqlCommand(oSQL)
oCommand.Parameters.Add("TITLE", SqlDbType.NVarChar, 250).Value = oJob.Name
oCommand.Parameters.Add("CRON", SqlDbType.NVarChar, 250).Value = oJob.CronSchedule
oCommand.Parameters.Add("TYPE_ID", SqlDbType.Int).Value = oJob.TypeId
oCommand.Parameters.Add("ACTIVE", SqlDbType.Bit).Value = oJob.Active
Return Database.ExecuteNonQuery(oCommand)
End Function
Private Function DoDeleteJob(pData As UpdateJobRequest) As Boolean
Dim oJob = pData.Job
Dim oSQL As String = "DELETE FROM TBECM_JR_JOB WHERE GUID = @GUID"
Dim oCommand As New SqlClient.SqlCommand(oSQL)
oCommand.Parameters.Add("GUID", SqlDbType.Int).Value = oJob.Id
Return Database.ExecuteNonQuery(oCommand)
End Function
End Class
Public Class UpdateJobRequest
Public Enum UpdateJobAction
Create
Update
Delete
End Enum
Public Action As UpdateJobAction
Public Job As JobRunnerReference.JobDefinition
End Class
Public Class UpdateJobResponse
Inherits Base.BaseResponse
End Class
End Class