diff --git a/Modules.Database/MSSQLServer.vb b/Modules.Database/MSSQLServer.vb index 93742d74..841968ef 100644 --- a/Modules.Database/MSSQLServer.vb +++ b/Modules.Database/MSSQLServer.vb @@ -49,41 +49,41 @@ Public Class MSSQLServer Return oConnectionStringBuilder.ToString End Function - Public Function Get_ConnectionStringforID(pID As Integer) + Public Function Get_ConnectionStringforID(pConnectionId As Integer) Dim connectionString As String = "" Try - 'Me.TBCONNECTIONTableAdapter.FillByID(Me.DD_DMSLiteDataSet.TBCONNECTION, id) - Dim oDTConnection As DataTable = GetDatatable("SELECT * FROM TBDD_CONNECTION WHERE GUID = " & pID) + Dim oDTConnection As DataTable = GetDatatable($"SELECT * FROM TBDD_CONNECTION WHERE GUID = {pConnectionId}") If oDTConnection.Rows.Count = 1 Then - Select Case oDTConnection.Rows(0).Item("SQL_PROVIDER").ToString.ToUpper + Dim oRow As DataRow = oDTConnection.Rows(0) + Dim oProvider = oRow.Item("SQL_PROVIDER").ToString.ToUpper + Dim oServer = oRow.Item("SERVER") + Dim oDatabase = oRow.Item("DATENBANK") + Dim oUser = oRow.Item("USERNAME") + Dim oPassword = oRow.Item("PASSWORD") + + Select Case oProvider Case "MS-SQL" - If oDTConnection.Rows(0).Item("USERNAME") = "WINAUTH" Then - connectionString = "Server=" & oDTConnection.Rows(0).Item("SERVER") & ";Database=" & oDTConnection.Rows(0).Item("DATENBANK") & ";Trusted_Connection=True;" + If oUser = "WINAUTH" Then + connectionString = $"Server={oServer};Database={oDatabase};Trusted_Connection=True;" Else - connectionString = "Server=" & oDTConnection.Rows(0).Item("SERVER") & ";Database=" & oDTConnection.Rows(0).Item("DATENBANK") & ";User Id=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("PASSWORD") & ";" + connectionString = $"Server={oServer};Database={oDatabase};User Id={oUser};Password={oPassword};" End If - ' connectionString = "Server=" & DTConnection.Rows(0).Item("SERVER") & ";Database=" & DTConnection.Rows(0).Item("DATENBANK") & ";User Id=" & DTConnection.Rows(0).Item("USERNAME") & ";Password=" & DTConnection.Rows(0).Item("PASSWORD") & ";" Case "Oracle" - If oDTConnection.Rows(0).Item("BEMERKUNG").ToString.Contains("without tnsnames") Then - connectionString = "Data Source=(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=" & oDTConnection.Rows(0).Item("SERVER") & ")(PORT=1521)))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=" & - oDTConnection.Rows(0).Item("DATENBANK") & ")));User Id=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("PASSWORD") & ";" + If oRow.Item("BEMERKUNG").ToString.Contains("without tnsnames") Then + connectionString = $"Data Source=(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST={oServer})(PORT=1521)))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME={oDatabase})));User Id={oUser};Password={oPassword};" Else - connectionString = "Data Source=" & oDTConnection.Rows(0).Item("SERVER") & ";Persist Security Info=True;User Id=" & oDTConnection.Rows(0).Item("USERNAME") & ";Password=" & oDTConnection.Rows(0).Item("PASSWORD") & ";Unicode=True" + connectionString = $"Data Source={oServer};Persist Security Info=True;User Id={oUser};Password={oPassword};Unicode=True" End If - 'Case "ODBC" - ' Dim conn As New OdbcConnection("dsn=" & DTConnection.Rows(0).Item("SERVER") & ";uid=" & DTConnection.Rows(0).Item("USERNAME") & ";pwd=" + DTConnection.Rows(0).Item("PASSWORD")) - ' connectionString = conn.ConnectionString Case Else - _Logger.Info(" - ConnectionType nicht integriert") + _Logger.Warn("Provider {0} nicht unterstützt!", oProvider) End Select Else - _Logger.Info(" No entry for Connection-ID: " & pID.ToString) + _Logger.Info("No entry for Connection-ID: " & pConnectionId.ToString) End If - Catch ex As Exception _Logger.Error(ex) - _Logger.Info(" - Error in bei Get ConnectionString - Fehler: " & vbNewLine & ex.Message) + _Logger.Info("Error in bei Get_ConnectionStringforID") End Try Return connectionString End Function @@ -94,6 +94,7 @@ Public Class MSSQLServer Private Function TestCanConnect(ConnectionString As String) As Boolean Try + _Logger.Debug("Testing connection to [{0}]", ConnectionString) Dim oConnection As New SqlConnection(ConnectionString) oConnection.Open() oConnection.Close() @@ -195,8 +196,7 @@ Public Class MSSQLServer End Using End Using Catch ex As Exception - _Logger.Warn($"GetDatatableWithConnection failed SQLCommand [{SqlCommand}] - ERROR: {ex.Message}") - + _Logger.Warn($"GetDatatableWithConnection failed with SQLCommand [{SqlCommand}] and ConnectionString [{MaskConnectionString(ConnectionString)}]") Return Nothing End Try End Function diff --git a/Modules.Database/My Project/AssemblyInfo.vb b/Modules.Database/My Project/AssemblyInfo.vb index 52c637a4..695166e6 100644 --- a/Modules.Database/My Project/AssemblyInfo.vb +++ b/Modules.Database/My Project/AssemblyInfo.vb @@ -12,7 +12,7 @@ Imports System.Runtime.InteropServices - + @@ -31,5 +31,5 @@ Imports System.Runtime.InteropServices ' übernehmen, indem Sie "*" eingeben: ' - - + + diff --git a/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb b/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb index 383e58ff..e9ca33aa 100644 --- a/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb +++ b/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb @@ -1,27 +1,6 @@ -Imports System.Collections.Generic - -Public Class GraphQLConfig +Public Class GraphQLConfig Public Property BaseUrl As String = "" Public Property Email As String = "" Public Property Password As String = "" Public Property CertificateFingerprint As String = "" - Public Property Queries As New List(Of Query) - - Public Class Query - Public Property Name As String - Public Property ConnectionString As String = "" - Public Property ClearBeforeFill As Boolean = False - Public Property QueryString As String = "" - Public Property OperationName As String = "" - Public Property DestinationTable As String = "" - - Public Property MappingBasePath As String = "" - Public Property MappingFields As New List(Of FieldMapping) - End Class - - Public Class FieldMapping - Public SourcePath As String = "" - Public DestinationColumn As String = "" - Public Value As String = "" - End Class End Class diff --git a/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb b/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb index 2cac8caf..741464df 100644 --- a/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb +++ b/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb @@ -1,4 +1,6 @@ -Imports System.IO +Option Explicit On + +Imports System.IO Imports DigitalData.Modules.Interfaces Imports DigitalData.Modules.Jobs Imports DigitalData.Modules.Config @@ -7,13 +9,15 @@ Imports Newtonsoft.Json.Linq Imports System.Collections.Generic Imports System.Linq Imports System.Text.RegularExpressions +Imports DigitalData.Modules.Database +Imports System.Data Public Class GraphQLJob Inherits JobBase Implements IJob(Of GraphQLArgs) - Public Sub New(LogConfig As LogConfig) - MyBase.New(LogConfig, Nothing, Nothing) + Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer) + MyBase.New(LogConfig, Nothing, MSSQL) End Sub Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start @@ -34,26 +38,102 @@ Public Class GraphQLJob ' save cookie for future requests oInterface.SaveCookies(oLoginResponse.Cookies.Item(0)) - _Logger.Debug("Getting the data") + _Logger.Debug("Loading Queries") - For Each oQuery As GraphQLConfig.Query In oConfigManager.Config.Queries - Dim oCurrentQuery = oQuery - Dim oDatabase As New Database.MSSQLServer(_LogConfig, oQuery.ConnectionString) + ' Load query data from TBCUST_JOBRUNNER_QUERY + Dim oQueryTable As DataTable = _MSSQL.GetDatatable("SELECT * FROM TBCUST_JOBRUNNER_QUERY ORDER BY OPERATION_NAME, CLEAR_BEFORE_FILL ASC") + Dim oQueryList As New List(Of GraphQL.Query) - ' get the data - Dim oDataResponse = oInterface.GetData(oCurrentQuery.QueryString, oCurrentQuery.OperationName) - Dim oResult As String + ' Save query data to business objects + For Each oRow As DataRow In oQueryTable.Rows + Dim oQuery As New GraphQL.Query With { + .Id = oRow.Item("GUID"), + .Name = oRow.Item("TITLE"), + .ClearBeforeFill = oRow.Item("CLEAR_BEFORE_FILL"), + .ConnectionId = oRow.Item("CON_ID"), ' TODO: Connection String? + .DestinationTable = oRow.Item("DESTINATION_TABLE"), + .OperationName = oRow.Item("OPERATION_NAME"), + .MappingBasePath = oRow.Item("MAPPING_BASE_PATH"), + .QueryString = oRow.Item("QUERY_STRING"), + .QueryConstraint = oRow.Item("QUERY_CONSTRAINT") + } + oQueryList.Add(oQuery) + Next - ' write data to string - Using oStream = oDataResponse.GetResponseStream() - Using oReader As New StreamReader(oStream) - oResult = oReader.ReadToEnd() + _Logger.Debug("Getting the data from GraphQL") + + For Each oQuery As GraphQL.Query In oQueryList + Try + Dim oConnectionId As Integer = oQuery.ConnectionId + Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId) + + Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString) + + _Logger.Info("Resetting data for query [{0}] with constraint [{1}]", oQuery.Name, oQuery.QueryConstraint) + + ' Reset all records to status = 0 + Dim oResetSQL = $"UPDATE {oQuery.DestinationTable} SET STATUS = 0" + If oQuery.QueryConstraint <> String.Empty Then + oResetSQL &= $" WHERE {oQuery.QueryConstraint}" + End If + _MSSQL.ExecuteNonQuery(oResetSQL) + + _Logger.Info("Getting data for query [{0}]", oQuery.Name) + + ' get the data from GraphQL + Dim oDataResponse = oInterface.GetData(oQuery.QueryString, oQuery.OperationName) + Dim oResult As String + + ' write data to string + Using oStream = oDataResponse.GetResponseStream() + Using oReader As New StreamReader(oStream) + oResult = oReader.ReadToEnd() + End Using End Using - End Using - If IsNothing(HandleResponse(oResult, oCurrentQuery, oDatabase)) Then - Continue For - End If + ' Fill the query object with field mapping data from TBCUST_JOBRUNNER_QUERY_MAPPING + Dim oSQL As String = "SELECT t2.* FROM TBCUST_JOBRUNNER_QUERY_MAPPING t + JOIN TBCUST_JOBRUNNER_MAPPING t2 ON t.MAPPING_ID = t2.GUID + WHERE t.QUERY_ID = {0}" + Dim oMappingTable As DataTable = _MSSQL.GetDatatable(String.Format(oSQL, oQuery.Id)) + + For Each oMapping As DataRow In oMappingTable.Rows + oQuery.MappingFields.Add(New GraphQL.FieldMapping With { + .DestinationColumn = oMapping.Item("DestinationColumn"), + .SourcePath = oMapping.Item("SourcePath") + }) + Next + + ' Handle the response from GraphQL and insert Data + Dim oQueryHandleResult = HandleResponse(oResult, oQuery, oDatabase) + + If IsNothing(oQueryHandleResult) Then + Continue For + End If + + ' Finally delete all old records, when: + ' - ClearBeforeFill is true, which should be the last query for a given Operation + ' - QueryConstraint is not empty, which means the records produced by this query can be selected easily + If oQuery.ClearBeforeFill = True Or oQuery.QueryConstraint <> String.Empty Then + Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 0" + If oQuery.QueryConstraint <> String.Empty Then + oDeleteSQL &= $" AND {oQuery.QueryConstraint}" + End If + _MSSQL.ExecuteNonQuery(oDeleteSQL) + End If + + Catch ex As Exception + _Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", oQuery.Name, oQuery.OperationName) + _Logger.Error(ex) + + ' If a crash happens, delete all records which were inserted in this run, + ' thus going back to the previous state + Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 1" + If oQuery.QueryConstraint <> String.Empty Then + oDeleteSQL &= $" AND {oQuery.QueryConstraint}" + End If + _MSSQL.ExecuteNonQuery(oDeleteSQL) + End Try Next ' logout @@ -65,10 +145,9 @@ Public Class GraphQLJob End Try End Sub - Private Function HandleResponse(JsonString As String, QueryData As GraphQLConfig.Query, DB As Database.MSSQLServer) + Private Function HandleResponse(JsonString As String, QueryData As GraphQL.Query, DB As Database.MSSQLServer) As GraphQL.Query Dim oObj As JObject = JObject.Parse(JsonString) - Dim oResultList = oObj.SelectToken(QueryData.MappingBasePath) - Dim oMappings = QueryData.MappingFields + Dim oResultList As JToken = oObj.SelectToken(QueryData.MappingBasePath) If oResultList Is Nothing Then _Logger.Warn("HandleResponse: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name) @@ -77,25 +156,26 @@ Public Class GraphQLJob _Logger.Info("Processing Queue [{0}] with [{1}] Items", QueryData.Name, oResultList.Count) - If QueryData.ClearBeforeFill Then - _Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable) - Try - DB.ExecuteNonQuery($"DELETE FROM {QueryData.DestinationTable}") - Catch ex As Exception - _Logger.Error(ex) - End Try - End If + 'If QueryData.ClearBeforeFill Then + ' _Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable) + ' _Logger.Info("Clear Command: [{0}]", QueryData.ClearCommand) + ' Try + ' DB.ExecuteNonQuery(QueryData.ClearCommand) + ' Catch ex As Exception + ' _Logger.Error(ex) + ' End Try + 'End If - For Each oResultItem In oResultList + For Each oResultItem As JToken In oResultList Try Dim oValues As New List(Of String) Dim oKeys As New List(Of String) - For Each oMapping In oMappings + For Each oMapping In QueryData.MappingFields Dim oToken = oResultItem.SelectToken(oMapping.SourcePath) If oToken Is Nothing Then - _Logger.Warn("HandleResponse: Could not find SourcePath: {0}", oMapping.SourcePath) + _Logger.Warn("HandleResponse: Could not find value at SourcePath: {0}", oMapping.SourcePath) End If oValues.Add(oToken.ToString()) diff --git a/Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb b/Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb new file mode 100644 index 00000000..408bd5f9 --- /dev/null +++ b/Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb @@ -0,0 +1,24 @@ +Imports System.Collections.Generic + +Namespace GraphQL + Public Class Query + Public Property Id As Integer + Public Property Name As String + Public Property ConnectionId As String = "" + Public Property ClearBeforeFill As Boolean = False + Public Property ClearCommand As String = "" + Public Property QueryString As String = "" + Public Property QueryConstraint As String = "" + Public Property OperationName As String = "" + Public Property DestinationTable As String = "" + + Public Property MappingBasePath As String = "" + Public Property MappingFields As New List(Of FieldMapping) + End Class + + Public Class FieldMapping + Public SourcePath As String = "" + Public DestinationColumn As String = "" + Public Value As String = "" + End Class +End Namespace diff --git a/Modules.Jobs/Jobs.vbproj b/Modules.Jobs/Jobs.vbproj index d439a7e2..756bc825 100644 --- a/Modules.Jobs/Jobs.vbproj +++ b/Modules.Jobs/Jobs.vbproj @@ -91,6 +91,7 @@ + diff --git a/Modules.Jobs/My Project/AssemblyInfo.vb b/Modules.Jobs/My Project/AssemblyInfo.vb index f9270abc..b162d696 100644 --- a/Modules.Jobs/My Project/AssemblyInfo.vb +++ b/Modules.Jobs/My Project/AssemblyInfo.vb @@ -12,7 +12,7 @@ Imports System.Runtime.InteropServices - + @@ -30,5 +30,5 @@ Imports System.Runtime.InteropServices ' Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern ' übernehmen, indem Sie "*" eingeben: - - + + diff --git a/Service.JobRunner/Jobs/GraphQLJob.vb b/Service.JobRunner/Jobs/GraphQLJob.vb index d2ebf3dc..0b3ed649 100644 --- a/Service.JobRunner/Jobs/GraphQLJob.vb +++ b/Service.JobRunner/Jobs/GraphQLJob.vb @@ -9,12 +9,13 @@ Public Class GraphQLJob Public Function Execute(context As IJobExecutionContext) As Task Implements Quartz.IJob.Execute Dim oJobData = context.MergedJobDataMap Dim oLogConfig As LogConfig = oJobData.Item("LogConfig") + Dim oMSSQL As MSSQLServer = oJobData.Item("MSSQL") Dim oArgs As Dictionary(Of String, String) = oJobData.Item("Args") Dim oJobArgs = New GraphQLArgs With { .QueryConfigPath = oArgs.Item("QueryConfigFolder") } - Dim oGraphQLJob As New Modules.Jobs.GraphQLJob(oLogConfig) + Dim oGraphQLJob As New Modules.Jobs.GraphQLJob(oLogConfig, oMSSQL) oGraphQLJob.Start(oJobArgs) Return Task.FromResult(True)