EDMIService: Add Quartz Scheduler, Add Caching for Datatables

Service.EDMIService/App.config

@@ -59,6 +59,10 @@
         <assemblyIdentity name="FirebirdSql.Data.FirebirdClient" publicKeyToken="3750abcc3150b00c" culture="neutral" />
         <bindingRedirect oldVersion="0.0.0.0-7.5.0.0" newVersion="7.5.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Memory" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.0.1.1" newVersion="4.0.1.1" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
   <startup>

Service.EDMIService/EDMIService.vb

@@ -19,6 +19,7 @@ Public Class EDMIService
     Public Shared Filesystem As Filesystem.File
     Public Shared EDMIArchive As EDMI.File.Archive
     Public Shared GlobalState As GlobalState
+    Public Shared Scheduler As Scheduler
 
     Private ReadOnly _logger As Logger
 
@@ -64,6 +65,28 @@ Public Class EDMIService
     End Function
 #End Region
 
+#Region "Database"
+    Public Function ReturnDatatableFromCache(Name As String) As TableResult Implements IEDMIService.ReturnDatatableFromCache
+        Try
+            Dim oSql = ""
+            _logger.Info($"ReturnDatatableFromCache, SQL: {oSql}")
+
+            Dim oDataset As DataSet = Scheduler.DataSet
+            Dim oDataTable As DataTable = Nothing
+            If oDataset.Tables.Contains(Name) Then
+                oDataTable = oDataset.Tables.Item(Name)
+            Else
+                Throw New ApplicationException($"DataTable {Name} does not exist")
+            End If
+
+            Return New TableResult(oDataTable)
+        Catch ex As Exception
+            _logger.Error(ex)
+            Return New TableResult(ex.Message)
+        End Try
+    End Function
+#End Region
+
 #Region "=== Database (MSSQL IDB) ==="
     Public Function ReturnDatatable_MSSQL_IDB(SQL As String) As TableResult Implements IEDMIService.ReturnDatatable_MSSQL_IDB
         Try

Service.EDMIService/EDMIService.vbproj

@@ -67,16 +67,39 @@
       <HintPath>..\packages\FirebirdSql.Data.FirebirdClient.7.5.0\lib\net452\FirebirdSql.Data.FirebirdClient.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.CSharp" />
+    <Reference Include="Microsoft.Extensions.Logging.Abstractions, Version=2.1.1.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Extensions.Logging.Abstractions.2.1.1\lib\netstandard2.0\Microsoft.Extensions.Logging.Abstractions.dll</HintPath>
+    </Reference>
     <Reference Include="NLog, Version=4.0.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
       <HintPath>..\packages\NLog.4.7.5\lib\net45\NLog.dll</HintPath>
     </Reference>
+    <Reference Include="Quartz, Version=3.2.3.0, Culture=neutral, PublicKeyToken=f6b8c98a402cc8a4, processorArchitecture=MSIL">
+      <HintPath>..\packages\Quartz.3.2.3\lib\net461\Quartz.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
+    <Reference Include="System.Buffers, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Buffers.4.5.1\lib\net461\System.Buffers.dll</HintPath>
+    </Reference>
     <Reference Include="System.ComponentModel.DataAnnotations" />
     <Reference Include="System.Configuration" />
     <Reference Include="System.Configuration.Install" />
     <Reference Include="System.Data" />
     <Reference Include="System.Deployment" />
+    <Reference Include="System.Diagnostics.DiagnosticSource, Version=4.0.5.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Diagnostics.DiagnosticSource.4.7.1\lib\net46\System.Diagnostics.DiagnosticSource.dll</HintPath>
+    </Reference>
     <Reference Include="System.IO.Compression" />
+    <Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Numerics" />
+    <Reference Include="System.Numerics.Vectors, Version=4.1.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.3\lib\net461\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Runtime.Remoting" />
     <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.ServiceModel" />
     <Reference Include="System.ServiceProcess" />
@@ -108,6 +131,9 @@
     <Compile Include="Exceptions.vb" />
     <Compile Include="Results\DatabaseResult.vb" />
     <Compile Include="EDMIService.vb" />
+    <Compile Include="Scheduler.vb" />
+    <Compile Include="Scheduler\DatatableJob.vb" />
+    <Compile Include="Scheduler\JobListener.vb" />
     <Compile Include="WindowsService.vb">
       <SubType>Component</SubType>
     </Compile>

Service.EDMIService/GlobalState.vb

@@ -4,21 +4,23 @@ Imports DigitalData.Modules.Logging
 Public Class GlobalState
     Private ReadOnly _LogConfig As LogConfig
     Private ReadOnly _Logger As Logger
-    Private ReadOnly _MSSQL As MSSQLServer
+    Private ReadOnly _MSSQL_IDB As MSSQLServer
+    Private ReadOnly _MSSQL_ECM As MSSQLServer
 
     Public Property ObjectStores As New List(Of ObjectStore)
 
-    Public Sub New(LogConfig As LogConfig, MSSQL_IDB As MSSQLServer)
+    Public Sub New(LogConfig As LogConfig, MSSQL_IDB As MSSQLServer, MSSQL_ECM As MSSQLServer)
         _LogConfig = LogConfig
         _Logger = LogConfig.GetLogger()
-        _MSSQL = MSSQL_IDB
+        _MSSQL_IDB = MSSQL_IDB
+        _MSSQL_ECM = MSSQL_ECM
     End Sub
 
     Public Sub LoadObjectStores()
         _Logger.Debug("Loading Object Stores")
         Try
             Dim oSQL As String = "SELECT * FROM VWIDB_OBJECTSTORE"
-            Dim oDatatable As DataTable = _MSSQL.GetDatatable(oSQL)
+            Dim oDatatable As DataTable = _MSSQL_IDB.GetDatatable(oSQL)
 
             ObjectStores.Clear()
 
@@ -36,7 +38,6 @@ Public Class GlobalState
             Next
         Catch ex As Exception
             _Logger.Error(ex)
-            Throw ex
         End Try
     End Sub
 

Service.EDMIService/IEDMIService.vb

@@ -11,6 +11,11 @@ Interface IEDMIService
     Function Heartbeat() As Boolean
 #End Region
 
+#Region "Database"
+    <OperationContract>
+    Function ReturnDatatableFromCache(Name As String) As TableResult
+#End Region
+
 #Region "Database (Firebird)"
     <OperationContract>
     Function ReturnDatatable_Firebird(SQL As String) As TableResult

Service.EDMIService/Scheduler.vb

@@ -0,0 +1,164 @@
+Imports System.Collections.Specialized
+Imports DigitalData.Modules.Database
+Imports DigitalData.Modules.Logging
+Imports Quartz
+Imports Quartz.Impl
+Imports Quartz.Impl.Matchers
+Imports Quartz.Logging
+
+Public Class Scheduler
+    Private _Factory As StdSchedulerFactory
+    Private _MSSQL As MSSQLServer
+    Private _Scheduler As IScheduler
+    Private _LogConfig As LogConfig
+    Private _Logger As DigitalData.Modules.Logging.Logger
+    Private _JobListener As JobListener
+
+    Private _Props = New NameValueCollection From {
+        {"quartz.serializer.type", "binary"}
+    }
+
+    Private Const JOB_GROUP As String = "DatatableJobs"
+
+    Public ReadOnly Property DataSet As DataSet
+        Get
+            Return _JobListener.Dataset
+        End Get
+    End Property
+
+    Public Sub New(LogConfig As LogConfig, MSSQL_ECM As MSSQLServer)
+        _LogConfig = LogConfig
+        _Logger = LogConfig.GetLogger()
+        _Factory = New StdSchedulerFactory(_Props)
+        _MSSQL = MSSQL_ECM
+
+        Dim oDataSet As New DataSet()
+        _JobListener = New JobListener(LogConfig, oDataSet)
+
+        Quartz.Logging.LogProvider.SetCurrentLogProvider(New LogProvider(_Logger))
+    End Sub
+
+    Public Async Sub Start()
+        ' Get new scheduler
+        _Scheduler = Await _Factory.GetScheduler()
+
+        ' configure it
+        _Scheduler.ListenerManager.AddJobListener(_JobListener,
+                                                  GroupMatcher(Of JobKey).GroupEquals(JOB_GROUP))
+
+        ' start it
+        Await _Scheduler.Start()
+
+        Dim oCronjobs As DataTable = Await GetCronJobs()
+
+        Try
+            If oCronjobs IsNot Nothing Then
+                _Logger.Debug("Loaded {0} cron jobs", oCronjobs.Rows.Count)
+
+                For Each oRow As DataRow In oCronjobs.Rows
+                    Dim oDefinition As String = oRow.Item("CRON_DEFINITION")
+                    Dim oTitle As String = oRow.Item("TITLE")
+                    Dim oGuid As Integer = oRow.Item("GUID")
+                    Dim oCronDetails As DataTable = Await GetCronJobDetails(oGuid)
+
+                    If oCronDetails IsNot Nothing Then
+                        _Logger.Debug("Loaded job [{0}]", oTitle)
+                        _Logger.Debug("Job details: {0}", oCronDetails.Rows.Count)
+                        _Logger.Debug("Job definition: {0}", oDefinition)
+
+                        For Each oRowDetail In oCronDetails.Rows
+                            Dim oTrigger As ITrigger
+                            Dim oJob As IJobDetail
+
+                            oTrigger = TriggerBuilder.Create().
+                                WithIdentity(oTitle).
+                                WithCronSchedule(oDefinition).
+                                StartNow().
+                                Build()
+
+                            oJob = JobBuilder.Create(Of DatatableJob)().
+                                WithIdentity(oGuid, JOB_GROUP).
+                                UsingJobData(New JobDataMap() From {
+                                    {"LogConfig", _LogConfig},
+                                    {"MSSQL", _MSSQL},
+                                    {"CronJobId", oGuid},
+                                    {"CronJobTitle", oTitle},
+                                    {"CronJobDetails", oRowDetail}
+                                }).
+                                Build()
+
+                            Await _Scheduler.ScheduleJob(oJob, oTrigger)
+                            _Logger.Debug("Scheduled a new job for Cron Job Id [{0}]", oGuid)
+                        Next
+                    Else
+                        _Logger.Warn("CronJob Details for CronJob [{0}] could not be fetched!", oGuid)
+                    End If
+                Next
+            Else
+                _Logger.Warn("CronJobs could not be fetched!")
+            End If
+        Catch ex As Exception
+            _Logger.Error(ex)
+            _Logger.Warn("Unexpected Error while setting up scheduler: " & ex.Message)
+        End Try
+    End Sub
+
+    Public Async Function GetCronJobs() As Task(Of DataTable)
+        Try
+            Dim oSQL As String = "SELECT * FROM TBAPPSERV_CRON_JOB WHERE ACTIVE = 1"
+            Dim oDatatable As DataTable = Await _MSSQL.GetDatatableAsync(oSQL)
+
+            Return oDatatable
+        Catch ex As Exception
+            Return Nothing
+        End Try
+    End Function
+
+    Public Async Function GetCronJobDetails(CronJobId As Integer) As Task(Of DataTable)
+        Try
+            Dim oSQL As String = $"SELECT * FROM TBAPPSERV_CRON_DETAIL WHERE CRON_ID = {CronJobId}"
+            Dim oDatatable As DataTable = Await _MSSQL.GetDatatableAsync(oSQL)
+
+            Return oDatatable
+        Catch ex As Exception
+            Return Nothing
+        End Try
+    End Function
+
+    Public Async Sub [Stop]()
+        Await _Scheduler.Shutdown()
+    End Sub
+
+    Private Class LogProvider
+        Implements ILogProvider
+
+        Private _Logger As Modules.Logging.Logger
+
+        Public Sub New(Logger As DigitalData.Modules.Logging.Logger)
+            MyBase.New()
+            _Logger = Logger
+        End Sub
+
+        Public Function OpenNestedContext(message As String) As IDisposable Implements ILogProvider.OpenNestedContext
+            Throw New NotImplementedException()
+        End Function
+
+        Public Function OpenMappedContext(key As String, value As Object, Optional destructure As Boolean = False) As IDisposable Implements ILogProvider.OpenMappedContext
+            Throw New NotImplementedException()
+        End Function
+
+        Private Function GetLogger(name As String) As Logging.Logger Implements ILogProvider.GetLogger
+            Return Function(level, func, exception, parameters)
+                       If exception IsNot Nothing Then
+                           _Logger.Error(exception)
+                       ElseIf level >= LogLevel.Debug AndAlso func IsNot Nothing Then
+                           _Logger.Debug(func(), parameters)
+                       ElseIf level >= LogLevel.Info AndAlso func IsNot Nothing Then
+                           _Logger.Info(func(), parameters)
+                       End If
+
+                       Return True
+                   End Function
+        End Function
+    End Class
+End Class

Service.EDMIService/Scheduler/DatatableJob.vb

@@ -0,0 +1,40 @@
+Imports DigitalData.Modules.Database
+Imports DigitalData.Modules.Logging
+Imports Quartz
+
+Public Class DatatableJob
+    Implements IJob
+
+    Public Function Execute(context As IJobExecutionContext) As Task Implements IJob.Execute
+        Dim oJobData = context.MergedJobDataMap
+        Dim oLogConfig As LogConfig = oJobData.Item("LogConfig")
+        Dim oCronJobTitle As String = oJobData.Item("CronJobTitle")
+        Dim oDetailRow As DataRow = oJobData.Item("CronJobDetails")
+        Dim oMSSQL As MSSQLServer = oJobData.Item("MSSQL")
+
+        Dim oConnectionId As Integer = oDetailRow.Item("CON_ID")
+        Dim oTitle As String = oDetailRow.Item("TITLE")
+        Dim oDatatableName As String = oDetailRow.Item("DT_NAME")
+        Dim oSQL As String = oDetailRow.Item("COMMAND")
+
+        Dim oLogger As Logger = oLogConfig.GetLogger()
+        oLogger.Debug("Running Command-Job [{0}]", oTitle)
+        oLogger.Debug("Datatable Name: {0}", oDatatableName)
+        oLogger.Debug("Connection Id: {0}", oConnectionId)
+
+        Dim oConnectionString = oMSSQL.Get_ConnectionStringforID(oConnectionId)
+
+        Try
+            Dim oResult = oMSSQL.GetDatatableWithConnection(oSQL, oConnectionString)
+            oResult.TableName = oDatatableName
+            oLogger.Debug("Result Datatable contains [{0}] rows", oResult.Rows.Count)
+
+            ' Das Ergebnis speichern
+            context.Result = oResult
+        Catch ex As Exception
+            oLogger.Warn("Unhandled exception while executing SQL for Datatable {}")
+        End Try
+
+        Return Task.FromResult(True)
+    End Function
+End Class

Service.EDMIService/Scheduler/JobListener.vb

@@ -0,0 +1,49 @@
+Imports System.Threading
+Imports DigitalData.Modules.Logging
+Imports Quartz
+Imports Quartz.Listener
+
+Public Class JobListener
+    Inherits JobListenerSupport
+    Public Overrides ReadOnly Property Name As String = "JobListener"
+    Public Property Dataset As DataSet
+
+    Private ReadOnly _Logger As Logger
+    Private ReadOnly _LogConfig As LogConfig
+
+    Public Sub New(LogConfig As LogConfig, ResultDataSet As DataSet)
+        MyBase.New()
+
+        _LogConfig = LogConfig
+        _Logger = LogConfig.GetLogger()
+        Dataset = ResultDataSet
+    End Sub
+
+    Public Overrides Function JobWasExecuted(context As IJobExecutionContext, jobException As JobExecutionException, Optional cancellationToken As CancellationToken = Nothing) As Task
+        Try
+            Dim oDataTable As DataTable = context.Result
+            Dim oDetailRow As DataRow = context.MergedJobDataMap.Item("CronJobDetails")
+            Dim oDatatableName As String = oDetailRow.Item("DT_NAME")
+            Dim oDatatableNameTemp As String = oDatatableName & "-TEMP"
+
+            If Dataset.Tables.Contains(oDatatableName) Then
+                _Logger.Debug("DataTable exists, renaming and replacing")
+                ' Rename the new table, add TEMP suffix
+                oDataTable.TableName = oDatatableNameTemp
+                ' Add the new table to the dataset
+                Dataset.Tables.Add(oDataTable)
+                ' Remove the old table
+                Dataset.Tables.Remove(oDatatableName)
+                ' Rename the new table
+                Dataset.Tables.Item(oDatatableNameTemp).TableName = oDatatableName
+            Else
+                _Logger.Debug("DataTable does not exist, adding")
+                Dataset.Tables.Add(oDataTable)
+            End If
+        Catch ex As Exception
+            _Logger.Error(ex)
+        End Try
+
+        Return MyBase.JobWasExecuted(context, jobException, cancellationToken)
+    End Function
+End Class

Service.EDMIService/WindowsService.vb

@@ -23,6 +23,7 @@ Public Class WindowsService
     Private _Archive As EDMI.File.Archive
     Private _Filesystem As Filesystem.File
     Private _Global As GlobalState
+    Private _Scheduler As Scheduler
 
     Public Sub New()
         ServiceName = SERVICE_NAME
@@ -37,6 +38,7 @@ Public Class WindowsService
             Dim oServicePath As String = AppDomain.CurrentDomain.BaseDirectory
 
             _LogConfig = New LogConfig(LogConfig.PathType.CustomPath, oServicePath)
+            _LogConfig.Debug = True
             _Logger = _LogConfig.GetLogger()
             _Logger.Info("Service {0} is starting...", SERVICE_DISPLAY_NAME)
             _Logger.Info("ServiceDirectory: {0}", oServicePath)
@@ -56,11 +58,15 @@ Public Class WindowsService
 
             _Archive = New EDMI.File.Archive(_LogConfig)
             _Filesystem = New Filesystem.File(_LogConfig)
-            _Global = New GlobalState(_LogConfig, _MSSQL_IDB)
+            _Global = New GlobalState(_LogConfig, _MSSQL_IDB, _MSSQL_ECM)
+            _Scheduler = New Scheduler(_LogConfig, _MSSQL_ECM)
 
             _Logger.Debug("Loading Objectstores")
             _Global.LoadObjectStores()
 
+            _Logger.Debug("Starting Scheduler")
+            _Scheduler.Start()
+
             _Logger.Debug("Preparing WCF ServiceHost")
             EDMIService.MSSQL_ECM = _MSSQL_ECM
             EDMIService.MSSQL_IDB = _MSSQL_IDB
@@ -70,6 +76,7 @@ Public Class WindowsService
             EDMIService.EDMIArchive = _Archive
             EDMIService.Filesystem = _Filesystem
             EDMIService.GlobalState = _Global
+            EDMIService.Scheduler = _Scheduler
 
             _Logger.Debug("Starting WCF ServiceHost")
             _ServiceHost = New ServiceHost(GetType(EDMIService))
@@ -121,6 +128,8 @@ Public Class WindowsService
             _ServiceHost.Close()
             _ServiceHost = Nothing
         End If
+
+        _Scheduler.Stop()
     End Sub
 End Class
 

Service.EDMIService/packages.config

@@ -1,5 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="FirebirdSql.Data.FirebirdClient" version="7.5.0" targetFramework="net472" />
+  <package id="Microsoft.Extensions.Logging.Abstractions" version="2.1.1" targetFramework="net472" />
   <package id="NLog" version="4.7.5" targetFramework="net472" />
+  <package id="Quartz" version="3.2.3" targetFramework="net472" />
+  <package id="System.Buffers" version="4.5.1" targetFramework="net472" />
+  <package id="System.Diagnostics.DiagnosticSource" version="4.7.1" targetFramework="net472" />
+  <package id="System.Memory" version="4.5.4" targetFramework="net472" />
+  <package id="System.Numerics.Vectors" version="4.5.0" targetFramework="net472" />
+  <package id="System.Runtime.CompilerServices.Unsafe" version="4.5.3" targetFramework="net472" />
 </packages>