Merge branch 'master' of http://dd-vmp07-com04:3000/AppStd/Monorepo

Modules.Database/MSSQLServer.vb

@@ -49,41 +49,41 @@ Public Class MSSQLServer
 
         Return oConnectionStringBuilder.ToString
     End Function
-    Public Function Get_ConnectionStringforID(pID As Integer)
+    Public Function Get_ConnectionStringforID(pConnectionId As Integer)
         Dim connectionString As String = ""
         Try
-            'Me.TBCONNECTIONTableAdapter.FillByID(Me.DD_DMSLiteDataSet.TBCONNECTION, id)
+            Dim oDTConnection As DataTable = GetDatatable($"SELECT * FROM TBDD_CONNECTION WHERE GUID = {pConnectionId}")
-            Dim oDTConnection As DataTable = GetDatatable("SELECT * FROM TBDD_CONNECTION WHERE GUID = " & pID)
             If oDTConnection.Rows.Count = 1 Then
-                Select Case oDTConnection.Rows(0).Item("SQL_PROVIDER").ToString.ToUpper
+                Dim oRow As DataRow = oDTConnection.Rows(0)
+                Dim oProvider = oRow.Item("SQL_PROVIDER").ToString.ToUpper
+                Dim oServer = oRow.Item("SERVER")
+                Dim oDatabase = oRow.Item("DATENBANK")
+                Dim oUser = oRow.Item("USERNAME")
+                Dim oPassword = oRow.Item("PASSWORD")
+
+                Select Case oProvider
                     Case "MS-SQL"
-                        If oDTConnection.Rows(0).Item("USERNAME") = "WINAUTH" Then
+                        If oUser = "WINAUTH" Then
-                            connectionString = "Server=" & oDTConnection.Rows(0).Item("SERVER") & ";Database=" & oDTConnection.Rows(0).Item("DATENBANK") & ";Trusted_Connection=True;"
+                            connectionString = $"Server={oServer};Database={oDatabase};Trusted_Connection=True;"
                         Else
-                            connectionString = "Server=" & oDTConnection.Rows(0).Item("SERVER") & ";Database=" & oDTConnection.Rows(0).Item("DATENBANK") & ";User Id=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("PASSWORD") & ";"
+                            connectionString = $"Server={oServer};Database={oDatabase};User Id={oUser};Password={oPassword};"
                         End If
-                        ' connectionString = "Server=" & DTConnection.Rows(0).Item("SERVER") & ";Database=" & DTConnection.Rows(0).Item("DATENBANK") & ";User Id=" & DTConnection.Rows(0).Item("USERNAME") & ";Password=" & DTConnection.Rows(0).Item("PASSWORD") & ";"
                     Case "Oracle"
-                        If oDTConnection.Rows(0).Item("BEMERKUNG").ToString.Contains("without tnsnames") Then
+                        If oRow.Item("BEMERKUNG").ToString.Contains("without tnsnames") Then
-                            connectionString = "Data Source=(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=" & oDTConnection.Rows(0).Item("SERVER") & ")(PORT=1521)))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=" &
+                            connectionString = $"Data Source=(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST={oServer})(PORT=1521)))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME={oDatabase})));User Id={oUser};Password={oPassword};"
-                               oDTConnection.Rows(0).Item("DATENBANK") & ")));User Id=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("PASSWORD") & ";"
                         Else
-                            connectionString = "Data Source=" & oDTConnection.Rows(0).Item("SERVER") & ";Persist Security Info=True;User Id=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("PASSWORD") & ";Unicode=True"
+                            connectionString = $"Data Source={oServer};Persist Security Info=True;User Id={oUser};Password={oPassword};Unicode=True"
                         End If
-                        'Case "ODBC"
-                        '    Dim conn As New OdbcConnection("dsn=" & DTConnection.Rows(0).Item("SERVER") & ";uid=" & DTConnection.Rows(0).Item("USERNAME") & ";pwd=" + DTConnection.Rows(0).Item("PASSWORD"))
-                        '    connectionString = conn.ConnectionString
                     Case Else
-                        _Logger.Info(" - ConnectionType nicht integriert")
+                        _Logger.Warn("Provider {0} nicht unterstützt!", oProvider)
 
                 End Select
             Else
-                _Logger.Info(" No entry for Connection-ID: " & pID.ToString)
+                _Logger.Info("No entry for Connection-ID: " & pConnectionId.ToString)
             End If
-
         Catch ex As Exception
             _Logger.Error(ex)
-            _Logger.Info(" - Error in bei Get ConnectionString - Fehler: " & vbNewLine & ex.Message)
+            _Logger.Info("Error in bei Get_ConnectionStringforID")
         End Try
         Return connectionString
     End Function
@@ -94,6 +94,7 @@ Public Class MSSQLServer
 
     Private Function TestCanConnect(ConnectionString As String) As Boolean
         Try
+            _Logger.Debug("Testing connection to [{0}]", ConnectionString)
             Dim oConnection As New SqlConnection(ConnectionString)
             oConnection.Open()
             oConnection.Close()
@@ -195,8 +196,7 @@ Public Class MSSQLServer
                 End Using
             End Using
         Catch ex As Exception
-            _Logger.Warn($"GetDatatableWithConnection failed SQLCommand [{SqlCommand}] - ERROR: {ex.Message}")
+            _Logger.Warn($"GetDatatableWithConnection failed with SQLCommand [{SqlCommand}] and ConnectionString [{MaskConnectionString(ConnectionString)}]")
-
             Return Nothing
         End Try
     End Function

Modules.Database/My Project/AssemblyInfo.vb

@@ -12,7 +12,7 @@ Imports System.Runtime.InteropServices
 <Assembly: AssemblyDescription("")>
 <Assembly: AssemblyCompany("Digital Data")>
 <Assembly: AssemblyProduct("Modules.Database")>
-<Assembly: AssemblyCopyright("Copyright ©  2018")>
+<Assembly: AssemblyCopyright("Copyright © 2021")>
 <Assembly: AssemblyTrademark("")>
 
 <Assembly: ComVisible(False)>
@@ -31,5 +31,5 @@ Imports System.Runtime.InteropServices
 ' übernehmen, indem Sie "*" eingeben:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("1.2.0.1")>
+<Assembly: AssemblyVersion("1.2.2.0")>
-<Assembly: AssemblyFileVersion("1.2.0.1")>
+<Assembly: AssemblyFileVersion("1.2.2.0")>

Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb

@@ -1,27 +1,6 @@
-Imports System.Collections.Generic
+Public Class GraphQLConfig
-
-Public Class GraphQLConfig
     Public Property BaseUrl As String = ""
     Public Property Email As String = ""
     Public Property Password As String = ""
     Public Property CertificateFingerprint As String = ""
-    Public Property Queries As New List(Of Query)
-
-    Public Class Query
-        Public Property Name As String
-        Public Property ConnectionString As String = ""
-        Public Property ClearBeforeFill As Boolean = False
-        Public Property QueryString As String = ""
-        Public Property OperationName As String = ""
-        Public Property DestinationTable As String = ""
-
-        Public Property MappingBasePath As String = ""
-        Public Property MappingFields As New List(Of FieldMapping)
-    End Class
-
-    Public Class FieldMapping
-        Public SourcePath As String = ""
-        Public DestinationColumn As String = ""
-        Public Value As String = ""
-    End Class
 End Class

Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb

@@ -1,4 +1,6 @@
-Imports System.IO
+Option Explicit On
+
+Imports System.IO
 Imports DigitalData.Modules.Interfaces
 Imports DigitalData.Modules.Jobs
 Imports DigitalData.Modules.Config
@@ -7,13 +9,15 @@ Imports Newtonsoft.Json.Linq
 Imports System.Collections.Generic
 Imports System.Linq
 Imports System.Text.RegularExpressions
+Imports DigitalData.Modules.Database
+Imports System.Data
 
 Public Class GraphQLJob
     Inherits JobBase
     Implements IJob(Of GraphQLArgs)
 
-    Public Sub New(LogConfig As LogConfig)
+    Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer)
-        MyBase.New(LogConfig, Nothing, Nothing)
+        MyBase.New(LogConfig, Nothing, MSSQL)
     End Sub
 
     Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start
@@ -34,14 +38,50 @@ Public Class GraphQLJob
             ' save cookie for future requests
             oInterface.SaveCookies(oLoginResponse.Cookies.Item(0))
 
-            _Logger.Debug("Getting the data")
+            _Logger.Debug("Loading Queries")
 
-            For Each oQuery As GraphQLConfig.Query In oConfigManager.Config.Queries
+            ' Load query data from TBCUST_JOBRUNNER_QUERY
-                Dim oCurrentQuery = oQuery
+            Dim oQueryTable As DataTable = _MSSQL.GetDatatable("SELECT * FROM TBCUST_JOBRUNNER_QUERY ORDER BY OPERATION_NAME, CLEAR_BEFORE_FILL ASC")
-                Dim oDatabase As New Database.MSSQLServer(_LogConfig, oQuery.ConnectionString)
+            Dim oQueryList As New List(Of GraphQL.Query)
 
-                ' get the data
+            ' Save query data to business objects
-                Dim oDataResponse = oInterface.GetData(oCurrentQuery.QueryString, oCurrentQuery.OperationName)
+            For Each oRow As DataRow In oQueryTable.Rows
+                Dim oQuery As New GraphQL.Query With {
+                    .Id = oRow.Item("GUID"),
+                    .Name = oRow.Item("TITLE"),
+                    .ClearBeforeFill = oRow.Item("CLEAR_BEFORE_FILL"),
+                    .ConnectionId = oRow.Item("CON_ID"), ' TODO: Connection String?
+                    .DestinationTable = oRow.Item("DESTINATION_TABLE"),
+                    .OperationName = oRow.Item("OPERATION_NAME"),
+                    .MappingBasePath = oRow.Item("MAPPING_BASE_PATH"),
+                    .QueryString = oRow.Item("QUERY_STRING"),
+                    .QueryConstraint = oRow.Item("QUERY_CONSTRAINT")
+                }
+                oQueryList.Add(oQuery)
+            Next
+
+            _Logger.Debug("Getting the data from GraphQL")
+
+            For Each oQuery As GraphQL.Query In oQueryList
+                Try
+                    Dim oConnectionId As Integer = oQuery.ConnectionId
+                    Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId)
+
+                    Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString)
+
+                    _Logger.Info("Resetting data for query [{0}] with constraint [{1}]", oQuery.Name, oQuery.QueryConstraint)
+
+                    ' Reset all records to status = 0
+                    Dim oResetSQL = $"UPDATE {oQuery.DestinationTable} SET STATUS = 0"
+                    If oQuery.QueryConstraint <> String.Empty Then
+                        oResetSQL &= $" WHERE {oQuery.QueryConstraint}"
+                    End If
+                    _MSSQL.ExecuteNonQuery(oResetSQL)
+
+                    _Logger.Info("Getting data for query [{0}]", oQuery.Name)
+
+                    ' get the data from GraphQL
+                    Dim oDataResponse = oInterface.GetData(oQuery.QueryString, oQuery.OperationName)
                     Dim oResult As String
 
                     ' write data to string
@@ -51,9 +91,49 @@ Public Class GraphQLJob
                         End Using
                     End Using
 
-                If IsNothing(HandleResponse(oResult, oCurrentQuery, oDatabase)) Then
+                    ' Fill the query object with field mapping data from TBCUST_JOBRUNNER_QUERY_MAPPING
+                    Dim oSQL As String = "SELECT t2.* FROM TBCUST_JOBRUNNER_QUERY_MAPPING t
+                                          JOIN TBCUST_JOBRUNNER_MAPPING t2 ON t.MAPPING_ID = t2.GUID
+                                          WHERE t.QUERY_ID = {0}"
+                    Dim oMappingTable As DataTable = _MSSQL.GetDatatable(String.Format(oSQL, oQuery.Id))
+
+                    For Each oMapping As DataRow In oMappingTable.Rows
+                        oQuery.MappingFields.Add(New GraphQL.FieldMapping With {
+                            .DestinationColumn = oMapping.Item("DestinationColumn"),
+                            .SourcePath = oMapping.Item("SourcePath")
+                        })
+                    Next
+
+                    ' Handle the response from GraphQL and insert Data
+                    Dim oQueryHandleResult = HandleResponse(oResult, oQuery, oDatabase)
+
+                    If IsNothing(oQueryHandleResult) Then
                         Continue For
                     End If
+
+                    ' Finally delete all old records, when:
+                    ' - ClearBeforeFill is true, which should be the last query for a given Operation
+                    ' - QueryConstraint is not empty, which means the records produced by this query can be selected easily
+                    If oQuery.ClearBeforeFill = True Or oQuery.QueryConstraint <> String.Empty Then
+                        Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 0"
+                        If oQuery.QueryConstraint <> String.Empty Then
+                            oDeleteSQL &= $" AND {oQuery.QueryConstraint}"
+                        End If
+                        _MSSQL.ExecuteNonQuery(oDeleteSQL)
+                    End If
+
+                Catch ex As Exception
+                    _Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", oQuery.Name, oQuery.OperationName)
+                    _Logger.Error(ex)
+
+                    ' If a crash happens, delete all records which were inserted in this run,
+                    ' thus going back to the previous state
+                    Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 1"
+                    If oQuery.QueryConstraint <> String.Empty Then
+                        oDeleteSQL &= $" AND {oQuery.QueryConstraint}"
+                    End If
+                    _MSSQL.ExecuteNonQuery(oDeleteSQL)
+                End Try
             Next
 
             ' logout
@@ -65,10 +145,9 @@ Public Class GraphQLJob
         End Try
     End Sub
 
-    Private Function HandleResponse(JsonString As String, QueryData As GraphQLConfig.Query, DB As Database.MSSQLServer)
+    Private Function HandleResponse(JsonString As String, QueryData As GraphQL.Query, DB As Database.MSSQLServer) As GraphQL.Query
         Dim oObj As JObject = JObject.Parse(JsonString)
-        Dim oResultList = oObj.SelectToken(QueryData.MappingBasePath)
+        Dim oResultList As JToken = oObj.SelectToken(QueryData.MappingBasePath)
-        Dim oMappings = QueryData.MappingFields
 
         If oResultList Is Nothing Then
             _Logger.Warn("HandleResponse: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name)
@@ -77,25 +156,26 @@ Public Class GraphQLJob
 
         _Logger.Info("Processing Queue [{0}] with [{1}] Items", QueryData.Name, oResultList.Count)
 
-        If QueryData.ClearBeforeFill Then
+        'If QueryData.ClearBeforeFill Then
-            _Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable)
+        '    _Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable)
-            Try
+        '    _Logger.Info("Clear Command: [{0}]", QueryData.ClearCommand)
-                DB.ExecuteNonQuery($"DELETE FROM {QueryData.DestinationTable}")
+        '    Try
-            Catch ex As Exception
+        '        DB.ExecuteNonQuery(QueryData.ClearCommand)
-                _Logger.Error(ex)
+        '    Catch ex As Exception
-            End Try
+        '        _Logger.Error(ex)
-        End If
+        '    End Try
+        'End If
 
-        For Each oResultItem In oResultList
+        For Each oResultItem As JToken In oResultList
             Try
                 Dim oValues As New List(Of String)
                 Dim oKeys As New List(Of String)
 
-                For Each oMapping In oMappings
+                For Each oMapping In QueryData.MappingFields
                     Dim oToken = oResultItem.SelectToken(oMapping.SourcePath)
 
                     If oToken Is Nothing Then
-                        _Logger.Warn("HandleResponse: Could not find SourcePath: {0}", oMapping.SourcePath)
+                        _Logger.Warn("HandleResponse: Could not find value at SourcePath: {0}", oMapping.SourcePath)
                     End If
 
                     oValues.Add(oToken.ToString())

Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb

@@ -0,0 +1,24 @@
+Imports System.Collections.Generic
+
+Namespace GraphQL
+    Public Class Query
+        Public Property Id As Integer
+        Public Property Name As String
+        Public Property ConnectionId As String = ""
+        Public Property ClearBeforeFill As Boolean = False
+        Public Property ClearCommand As String = ""
+        Public Property QueryString As String = ""
+        Public Property QueryConstraint As String = ""
+        Public Property OperationName As String = ""
+        Public Property DestinationTable As String = ""
+
+        Public Property MappingBasePath As String = ""
+        Public Property MappingFields As New List(Of FieldMapping)
+    End Class
+
+    Public Class FieldMapping
+        Public SourcePath As String = ""
+        Public DestinationColumn As String = ""
+        Public Value As String = ""
+    End Class
+End Namespace

Modules.Jobs/Jobs.vbproj

@@ -91,6 +91,7 @@
     <Compile Include="EDMI\GraphQL\GraphQLArgs.vb" />
     <Compile Include="EDMI\GraphQL\GraphQLConfig.vb" />
     <Compile Include="EDMI\GraphQL\GraphQLJob.vb" />
+    <Compile Include="EDMI\GraphQL\GraphQLQuery.vb" />
     <Compile Include="EDMI\ZUGFeRD\EmailData.vb" />
     <Compile Include="EDMI\ZUGFeRD\EmailFunctions.vb" />
     <Compile Include="EDMI\ZUGFeRD\EmailStrings.vb" />

Modules.Jobs/My Project/AssemblyInfo.vb

@@ -12,7 +12,7 @@ Imports System.Runtime.InteropServices
 <Assembly: AssemblyDescription("")>
 <Assembly: AssemblyCompany("Digital Data")>
 <Assembly: AssemblyProduct("Modules.Jobs")>
-<Assembly: AssemblyCopyright("Copyright © 2020")>
+<Assembly: AssemblyCopyright("Copyright © 2021")>
 <Assembly: AssemblyTrademark("")>
 
 <Assembly: ComVisible(False)>
@@ -30,5 +30,5 @@ Imports System.Runtime.InteropServices
 ' Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern 
 ' übernehmen, indem Sie "*" eingeben:
 
-<Assembly: AssemblyVersion("1.3.2.0")>
+<Assembly: AssemblyVersion("1.4.1.0")>
-<Assembly: AssemblyFileVersion("1.3.2.0")>
+<Assembly: AssemblyFileVersion("1.4.1.0")>

Service.JobRunner/Jobs/GraphQLJob.vb

@@ -9,12 +9,13 @@ Public Class GraphQLJob
     Public Function Execute(context As IJobExecutionContext) As Task Implements Quartz.IJob.Execute
         Dim oJobData = context.MergedJobDataMap
         Dim oLogConfig As LogConfig = oJobData.Item("LogConfig")
+        Dim oMSSQL As MSSQLServer = oJobData.Item("MSSQL")
         Dim oArgs As Dictionary(Of String, String) = oJobData.Item("Args")
         Dim oJobArgs = New GraphQLArgs With {
             .QueryConfigPath = oArgs.Item("QueryConfigFolder")
         }
 
-        Dim oGraphQLJob As New Modules.Jobs.GraphQLJob(oLogConfig)
+        Dim oGraphQLJob As New Modules.Jobs.GraphQLJob(oLogConfig, oMSSQL)
         oGraphQLJob.Start(oJobArgs)
 
         Return Task.FromResult(True)