EDMIService: Refactor service so that cron job details a read from database on every job run

Service.EDMIService/EDMIService.vb

@@ -1,6 +1,5 @@
 Imports DigitalData.Modules.Database
 Imports DigitalData.Modules.Logging
-Imports DigitalData.Modules.Filesystem
 Imports DigitalData.Modules.Language
 Imports DigitalData.Modules
 Imports System.IO
@@ -8,7 +7,6 @@ Imports System.ServiceModel
 Imports System.Data.SqlClient
 Imports DigitalData.Services.EDMIService.Results
 Imports System.ServiceModel.Description
-Imports System.ServiceModel.Channels
 
 <ServiceBehavior(InstanceContextMode:=InstanceContextMode.PerSession)>
 Public Class EDMIService

Service.EDMIService/Scheduler.vb

@@ -59,48 +59,28 @@ Public Class Scheduler
                     Dim oDefinition As String = oRow.Item("CRON_DEFINITION")
                     Dim oTitle As String = oRow.Item("TITLE")
                     Dim oGuid As Integer = oRow.Item("GUID")
-                    Dim oCronDetails As DataTable = Await GetCronJobDetails(oGuid)
 
-                    If oCronDetails IsNot Nothing Then
+                    Dim oTrigger As ITrigger
-                        _Logger.Debug("Loaded job [{0}]", oTitle)
+                    Dim oJob As IJobDetail
-                        _Logger.Debug("Job details: {0}", oCronDetails.Rows.Count)
+                    Dim oIdentity As String = $"CRON-JOB-{oGuid}"
-                        _Logger.Debug("Job definition: {0}", oDefinition)
 
-                        For Each oRowDetail As DataRow In oCronDetails.Rows
+                    oTrigger = TriggerBuilder.Create().
-                            Dim oTrigger As ITrigger
+                        WithIdentity(oIdentity, JOB_GROUP).
-                            Dim oJob As IJobDetail
+                        WithCronSchedule(oDefinition).
-                            Dim oDataTableName As String = oRowDetail.Item("DT_NAME")
+                        Build()
-                            Dim oRunOnStartup As Boolean = oRowDetail.Item("RUN_ON_STARTUP")
-                            Dim oJobIdentity As String = $"{oGuid}~{oDataTableName}"
 
-                            Dim oBaseTrigger = TriggerBuilder.Create().
+                    oJob = JobBuilder.Create(Of DatatableJob)().
-                                WithIdentity(oJobIdentity, JOB_GROUP).
+                        WithIdentity(oIdentity, JOB_GROUP).
-                                WithCronSchedule(oDefinition)
+                        UsingJobData(New JobDataMap From {
+                            {"LogConfig", _LogConfig},
+                            {"MSSQL", _MSSQL},
+                            {"CronJobId", oGuid},
+                            {"CronJobTitle", oTitle}
+                        }).
+                        Build()
 
-                            ' Run directly at startup if configured
+                    Await _Scheduler.ScheduleJob(oJob, oTrigger)
-                            If oRunOnStartup Then
+                    _Logger.Debug("Scheduled a new job for Cron Job [{0}]", oTitle)
-                                oBaseTrigger = oBaseTrigger.StartNow()
-                            End If
-
-                            oTrigger = oBaseTrigger.Build()
-
-                            oJob = JobBuilder.Create(Of DatatableJob)().
-                                WithIdentity(oJobIdentity, JOB_GROUP).
-                                UsingJobData(New JobDataMap() From {
-                                    {"LogConfig", _LogConfig},
-                                    {"MSSQL", _MSSQL},
-                                    {"CronJobId", oGuid},
-                                    {"CronJobTitle", oTitle},
-                                    {"CronJobDetails", oRowDetail}
-                                }).
-                                Build()
-
-                            Await _Scheduler.ScheduleJob(oJob, oTrigger)
-                            _Logger.Debug("Scheduled a new job for Cron Job Id [{0}]", oGuid)
-                        Next
-                    Else
-                        _Logger.Warn("CronJob Details for CronJob [{0}] could not be fetched!", oGuid)
-                    End If
                 Next
             Else
                 _Logger.Warn("CronJobs could not be fetched!")

Service.EDMIService/Scheduler/DatatableJob.vb

@@ -9,58 +9,127 @@ Imports Quartz
 Public Class DatatableJob
     Implements IJob
 
+    Private _MSSQL As MSSQLServer
+
     Public Function Execute(context As IJobExecutionContext) As Task Implements IJob.Execute
         Dim oJobData = context.MergedJobDataMap
         Dim oLogConfig As LogConfig = oJobData.Item("LogConfig")
+        Dim oCronJobId As Integer = oJobData.Item("CronJobId")
+        Dim oCronJobTitle As String = oJobData.Item("CronJobTitle")
         Dim oLogger As Logger = oLogConfig.GetLogger()
-        Dim oDetailRow As DataRow = oJobData.Item("CronJobDetails")
+
-        Dim oDatatableName As String = NotNull(oDetailRow.Item("DT_NAME"), String.Empty)
+        Dim oResult As New JobResult()
+
+        oLogger.Info("Running Datatable Job [{0}]", oCronJobTitle)
 
         Try
-            Dim oCronJobTitle As String = oJobData.Item("CronJobTitle")
             Dim oMSSQL As MSSQLServer = oJobData.Item("MSSQL")
+            Dim oCronSQL As String = $"SELECT * FROM TBAPPSERV_CRON_DETAIL WHERE CRON_ID = {oCronJobId}"
+            Dim oCronDetails As DataTable = oMSSQL.GetDatatable(oCronSQL)
 
-            Dim oConnectionId As Integer = NotNull(oDetailRow.Item("CON_ID"), String.Empty)
+            For Each oRow As DataRow In oCronDetails.Rows
-            Dim oTitle As String = NotNull(oDetailRow.Item("TITLE"), String.Empty)
+                Dim oConnectionId As Integer = NotNull(oRow.Item("CON_ID"), String.Empty)
-            Dim oSQL As String = NotNull(oDetailRow.Item("COMMAND"), String.Empty)
+                Dim oTitle As String = NotNull(oRow.Item("TITLE"), String.Empty)
+                Dim oSQL As String = NotNull(oRow.Item("COMMAND"), String.Empty)
+                Dim oDatatableName As String = NotNull(oRow.Item("DT_NAME"), String.Empty)
 
-            oLogger.Debug("Running Command-Job [{0}]", oTitle)
+                oLogger.Debug("Running Command-Job [{0}]", oTitle)
-            oLogger.Debug("Datatable Name: {0}", oDatatableName)
+                oLogger.Debug("Datatable Name: {0}", oDatatableName)
-            oLogger.Debug("Connection Id: {0}", oConnectionId)
+                oLogger.Debug("Connection Id: {0}", oConnectionId)
 
-            Dim oConnectionString = oMSSQL.Get_ConnectionStringforID(oConnectionId)
+                Dim oConnectionString = oMSSQL.Get_ConnectionStringforID(oConnectionId)
+                Dim oTable = oMSSQL.GetDatatableWithConnection(oSQL, oConnectionString)
+                oTable.TableName = oDatatableName
+                oLogger.Debug("Result Datatable [{0}] contains [{1}] rows", oTable.TableName, oTable.Rows.Count)
 
-            Dim oTable = oMSSQL.GetDatatableWithConnection(oSQL, oConnectionString)
+                Dim oResultTable = New JobResult.ResultTable() With {
-            oTable.TableName = oDatatableName
+                    .Table = oTable,
-            oLogger.Debug("Result Datatable [{0}] contains [{1}] rows", oTable.TableName, oTable.Rows.Count)
+                    .DetailRow = oRow
+                }
 
-            Dim oResult = New JobResult() With {
+                Dim oChildTableNAme As String = NotNull(oRow.Item("CHILD_DT_NAME"), String.Empty)
-                .Table = oTable
-            }
 
-            Dim oChildTableNAme As String = NotNull(oDetailRow.Item("CHILD_DT_NAME"), String.Empty)
+                If oChildTableNAme <> String.Empty Then
+                    Dim oParentColumn As String = NotNull(oRow.Item("DT_COLUMN"), String.Empty)
+                    Dim oChildColumn As String = NotNull(oRow.Item("CHILD_DT_COLUMN"), String.Empty)
+                    oLogger.Debug("Child Datatable [{0}] defined, Relation: Parent [{1}] -> Child [{2}]", oChildTableNAme, oParentColumn, oChildColumn)
 
-            If oChildTableNAme <> String.Empty Then
+                    Dim oChildTable As DataTable = oMSSQL.GetDatatableWithConnection($"SELECT * FROM {oChildTableNAme}", oConnectionString)
-                Dim oParentColumn As String = NotNull(oDetailRow.Item("DT_COLUMN"), String.Empty)
+                    oChildTable.TableName = oChildTableNAme
-                Dim oChildColumn As String = NotNull(oDetailRow.Item("CHILD_DT_COLUMN"), String.Empty)
+                    oLogger.Debug("Child Datatable [{0}] contains [{1}] rows", oChildTable.TableName, oChildTable.Rows.Count)
-                oLogger.Debug("Child Datatable [{0}] defined, Relation: Parent [{1}] -> Child [{2}]", oChildTableName, oParentColumn, oChildColumn)
 
-                Dim oChildTable As DataTable = oMSSQL.GetDatatableWithConnection($"SELECT * FROM {oChildTableName}", oConnectionString)
+                    oResultTable.ChildTable = oChildTable
-                oChildTable.TableName = oChildTableName
+                    oResultTable.ChildRelationColumn = oChildColumn
-                oLogger.Debug("Child Datatable [{0}] contains [{1}] rows", oChildTable.TableName, oChildTable.Rows.Count)
+                    oResultTable.TableRelationColumn = oParentColumn
+                End If
 
-                oResult.ChildTable = oChildTable
+                oLogger.Info("Fetched Datatable [{0}]", oDatatableName)
-                oResult.ChildRelationColumn = oChildColumn
+                oResult.Tables.Add(oResultTable)
-                oResult.TableRelationColumn = oParentColumn
+            Next
-            End If
 
-            ' Das Ergebnis speichern
+            '  Das Ergebnis speichern
             context.Result = oResult
+
         Catch ex As Exception
             oLogger.Error(ex)
-            oLogger.Warn("Unhandled exception while executing SQL for Datatable {0}", oDatatableName)
+            oLogger.Warn("Unhandled exception while executing SQL for Datatable {0}", oCronJobTitle)
         End Try
 
         Return Task.FromResult(True)
+
+#Region "OLD"
+        'Dim oJobData = context.MergedJobDataMap
+        'Dim oLogConfig As LogConfig = oJobData.Item("LogConfig")
+        'Dim oLogger As Logger = oLogConfig.GetLogger()
+        'Dim oDetailRow As DataRow = oJobData.Item("CronJobDetails")
+        'Dim oDatatableName As String = NotNull(oDetailRow.Item("DT_NAME"), String.Empty)
+
+        'Try
+        '    Dim oCronJobTitle As String = oJobData.Item("CronJobTitle")
+        '    Dim oMSSQL As MSSQLServer = oJobData.Item("MSSQL")
+
+        '    Dim oConnectionId As Integer = NotNull(oDetailRow.Item("CON_ID"), String.Empty)
+        '    Dim oTitle As String = NotNull(oDetailRow.Item("TITLE"), String.Empty)
+        '    Dim oSQL As String = NotNull(oDetailRow.Item("COMMAND"), String.Empty)
+
+        '    oLogger.Debug("Running Command-Job [{0}]", oTitle)
+        '    oLogger.Debug("Datatable Name: {0}", oDatatableName)
+        '    oLogger.Debug("Connection Id: {0}", oConnectionId)
+
+        '    Dim oConnectionString = oMSSQL.Get_ConnectionStringforID(oConnectionId)
+
+        '    Dim oTable = oMSSQL.GetDatatableWithConnection(oSQL, oConnectionString)
+        '    oTable.TableName = oDatatableName
+        '    oLogger.Debug("Result Datatable [{0}] contains [{1}] rows", oTable.TableName, oTable.Rows.Count)
+
+        '    Dim oResult = New JobResult() With {
+        '        .Table = oTable
+        '    }
+
+        '    Dim oChildTableNAme As String = NotNull(oDetailRow.Item("CHILD_DT_NAME"), String.Empty)
+
+        '    If oChildTableNAme <> String.Empty Then
+        '        Dim oParentColumn As String = NotNull(oDetailRow.Item("DT_COLUMN"), String.Empty)
+        '        Dim oChildColumn As String = NotNull(oDetailRow.Item("CHILD_DT_COLUMN"), String.Empty)
+        '        oLogger.Debug("Child Datatable [{0}] defined, Relation: Parent [{1}] -> Child [{2}]", oChildTableName, oParentColumn, oChildColumn)
+
+        '        Dim oChildTable As DataTable = oMSSQL.GetDatatableWithConnection($"SELECT * FROM {oChildTableName}", oConnectionString)
+        '        oChildTable.TableName = oChildTableName
+        '        oLogger.Debug("Child Datatable [{0}] contains [{1}] rows", oChildTable.TableName, oChildTable.Rows.Count)
+
+        '        oResult.ChildTable = oChildTable
+        '        oResult.ChildRelationColumn = oChildColumn
+        '        oResult.TableRelationColumn = oParentColumn
+        '    End If
+
+        '    ' Das Ergebnis speichern
+        '    context.Result = oResult
+        'Catch ex As Exception
+        '    oLogger.Error(ex)
+        '    oLogger.Warn("Unhandled exception while executing SQL for Datatable {0}", oDatatableName)
+        'End Try
+
+        'Return Task.FromResult(True)
+#End Region
     End Function
 End Class

Service.EDMIService/Scheduler/JobListener.vb

@@ -26,6 +26,24 @@ Public Class JobListener
         Dataset = ResultDataSet
     End Sub
 
+    Public Overrides Function JobWasExecuted(context As IJobExecutionContext, jobException As JobExecutionException, Optional cancellationToken As CancellationToken = Nothing) As Task
+        _Logger.Info("Job [{0}] was executed successfully. Saving Data.", context.JobDetail.Key)
+
+        Dim oResult As JobResult = context.Result
+
+        For Each oTableResult As JobResult.ResultTable In oResult.Tables
+            Try
+                _Logger.Debug("Saving Datatable [{0}]", oTableResult.Table)
+                SaveDataTables(oTableResult)
+
+            Catch ex As Exception
+                _Logger.Error("Error while executing SaveDataTables for {0}", oTableResult.Table.TableName)
+            End Try
+        Next
+
+        Return MyBase.JobWasExecuted(context, jobException, cancellationToken)
+    End Function
+
     Public Sub ReplaceExistingTable(Name As String, Table As DataTable, DataSet As DataSet, Optional ChildTable As DataTable = Nothing)
         Try
             Dim oDatatableNameTemp As String = Name & "-TEMP"
@@ -140,16 +158,17 @@ Public Class JobListener
         End Try
     End Sub
 
-    Public Sub SaveDataTables(Result As JobResult, DetailRow As DataRow)
+    Public Sub SaveDataTables(Result As JobResult.ResultTable)
         Try
             Dim oTable As DataTable = Result.Table
 
-            Dim oName As String = DetailRow.Item("DT_NAME")
+            Dim oName As String = Result.DetailRow.Item("DT_NAME")
-            Dim oDetailId As Integer = DetailRow.Item("GUID")
+            Dim oDetailId As Integer = Result.DetailRow.Item("GUID")
             Dim oDatatableNameTemp As String = oName & "-TEMP"
 
             ' Used for debugging relations and constraints
-            'ListTables(Dataset)
+            _Logger.Debug("Dataset BEFORE saving datatables")
+            ListTables(Dataset)
 
             If Dataset.Tables.Contains(oName) Then
                 ' Replace existing table
@@ -179,6 +198,10 @@ Public Class JobListener
                 _Logger.Debug(oRelation.RelationName)
             Next
 
+            ' Used for debugging relations and constraints
+            _Logger.Debug("Dataset AFTER saving datatables")
+            ListTables(Dataset)
+
             _MSSQL.ExecuteNonQuery($"INSERT INTO TBAPPSERV_CRON_DETAIL_HISTORY (DETAIL_ID) VALUES ({oDetailId})")
         Catch ex As Exception
             _Logger.Warn("Unexpected error in JobListener: {0}", ex.Message)
@@ -205,11 +228,4 @@ Public Class JobListener
             oIndex += 1
         Next
     End Sub
-
-    Public Overrides Function JobWasExecuted(context As IJobExecutionContext, jobException As JobExecutionException, Optional cancellationToken As CancellationToken = Nothing) As Task
-        Dim oDetailRow As DataRow = context.MergedJobDataMap.Item("CronJobDetails")
-        SaveDataTables(context.Result, oDetailRow)
-
-        Return MyBase.JobWasExecuted(context, jobException, cancellationToken)
-    End Function
 End Class

Service.EDMIService/Scheduler/JobResult.vb

@@ -1,6 +1,12 @@
 Public Class JobResult
-    Public Table As DataTable
+    Public Tables As New List(Of ResultTable)
-    Public ChildTable As DataTable
+
-    Public TableRelationColumn As String
+    Public Class ResultTable
-    Public ChildRelationColumn As String
+        Public Table As DataTable
+        Public ChildTable As DataTable
+        Public TableRelationColumn As String
+        Public ChildRelationColumn As String
+
+        Public DetailRow As DataRow
+    End Class
 End Class

Service.JobRunner/App.config

@@ -29,7 +29,7 @@
                 <value>False</value>
             </setting>
             <setting name="ADSYNC_CONFIG" serializeAs="String">
-                <value>False|0 0 0/1 * * ?|RootPath::LDAP://DIGITALDATA,UserFilter::(&amp;(samaccountname=@SAMACCOUNTNAME)),GroupFilter(&amp;(samaccountname=*))</value>
+                <value>False|0 0 0/1 * * ?|RootPath::LDAP://DIGITALDATA,UserFilter::(&amp;(samaccountname=@SAMACCOUNTNAME)),GroupFilter::(&amp;(samaccountname=*))</value>
             </setting>
             <setting name="TEST_CONFIG" serializeAs="String">
                 <value>True|10/0 * * * * ?|Foo::Bar</value>