JobRunner: Load Jobrunner GraphQL Queries via Database
This commit is contained in:
Jonathan Jenne
parent
fb410d6308
commit
2d08ab2c22
@ -1,27 +1,6 @@
|
||||
Imports System.Collections.Generic
|
||||
|
||||
Public Class GraphQLConfig
|
||||
Public Class GraphQLConfig
|
||||
Public Property BaseUrl As String = ""
|
||||
Public Property Email As String = ""
|
||||
Public Property Password As String = ""
|
||||
Public Property CertificateFingerprint As String = ""
|
||||
Public Property Queries As New List(Of Query)
|
||||
|
||||
Public Class Query
|
||||
Public Property Name As String
|
||||
Public Property ConnectionString As String = ""
|
||||
Public Property ClearBeforeFill As Boolean = False
|
||||
Public Property QueryString As String = ""
|
||||
Public Property OperationName As String = ""
|
||||
Public Property DestinationTable As String = ""
|
||||
|
||||
Public Property MappingBasePath As String = ""
|
||||
Public Property MappingFields As New List(Of FieldMapping)
|
||||
End Class
|
||||
|
||||
Public Class FieldMapping
|
||||
Public SourcePath As String = ""
|
||||
Public DestinationColumn As String = ""
|
||||
Public Value As String = ""
|
||||
End Class
|
||||
End Class
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
Imports System.IO
|
||||
Option Explicit On
|
||||
|
||||
Imports System.IO
|
||||
Imports DigitalData.Modules.Interfaces
|
||||
Imports DigitalData.Modules.Jobs
|
||||
Imports DigitalData.Modules.Config
|
||||
@ -7,13 +9,15 @@ Imports Newtonsoft.Json.Linq
|
||||
Imports System.Collections.Generic
|
||||
Imports System.Linq
|
||||
Imports System.Text.RegularExpressions
|
||||
Imports DigitalData.Modules.Database
|
||||
Imports System.Data
|
||||
|
||||
Public Class GraphQLJob
|
||||
Inherits JobBase
|
||||
Implements IJob(Of GraphQLArgs)
|
||||
|
||||
Public Sub New(LogConfig As LogConfig)
|
||||
MyBase.New(LogConfig, Nothing, Nothing)
|
||||
Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer)
|
||||
MyBase.New(LogConfig, Nothing, MSSQL)
|
||||
End Sub
|
||||
|
||||
Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start
|
||||
@ -34,14 +38,50 @@ Public Class GraphQLJob
|
||||
' save cookie for future requests
|
||||
oInterface.SaveCookies(oLoginResponse.Cookies.Item(0))
|
||||
|
||||
_Logger.Debug("Getting the data")
|
||||
_Logger.Debug("Loading Queries")
|
||||
|
||||
For Each oQuery As GraphQLConfig.Query In oConfigManager.Config.Queries
|
||||
Dim oCurrentQuery = oQuery
|
||||
Dim oDatabase As New Database.MSSQLServer(_LogConfig, oQuery.ConnectionString)
|
||||
' Load query data from TBCUST_JOBRUNNER_QUERY
|
||||
Dim oQueryTable As DataTable = _MSSQL.GetDatatable("SELECT * FROM TBCUST_JOBRUNNER_QUERY ORDER BY OPERATION_NAME, CLEAR_BEFORE_FILL ASC")
|
||||
Dim oQueryList As New List(Of GraphQL.Query)
|
||||
|
||||
' get the data
|
||||
Dim oDataResponse = oInterface.GetData(oCurrentQuery.QueryString, oCurrentQuery.OperationName)
|
||||
' Save query data to business objects
|
||||
For Each oRow As DataRow In oQueryTable.Rows
|
||||
Dim oQuery As New GraphQL.Query With {
|
||||
.Id = oRow.Item("GUID"),
|
||||
.Name = oRow.Item("TITLE"),
|
||||
.ClearBeforeFill = oRow.Item("CLEAR_BEFORE_FILL"),
|
||||
.ConnectionId = oRow.Item("CON_ID"), ' TODO: Connection String?
|
||||
.DestinationTable = oRow.Item("DESTINATION_TABLE"),
|
||||
.OperationName = oRow.Item("OPERATION_NAME"),
|
||||
.MappingBasePath = oRow.Item("MAPPING_BASE_PATH"),
|
||||
.QueryString = oRow.Item("QUERY_STRING"),
|
||||
.QueryConstraint = oRow.Item("QUERY_CONSTRAINT")
|
||||
}
|
||||
oQueryList.Add(oQuery)
|
||||
Next
|
||||
|
||||
_Logger.Debug("Getting the data from GraphQL")
|
||||
|
||||
For Each oQuery As GraphQL.Query In oQueryList
|
||||
Try
|
||||
Dim oConnectionId As Integer = oQuery.ConnectionId
|
||||
Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId)
|
||||
|
||||
Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString)
|
||||
|
||||
_Logger.Info("Resetting data for query [{0}] with constraint [{1}]", oQuery.Name, oQuery.QueryConstraint)
|
||||
|
||||
' Reset all records to status = 0
|
||||
Dim oResetSQL = $"UPDATE {oQuery.DestinationTable} SET STATUS = 0"
|
||||
If oQuery.QueryConstraint <> String.Empty Then
|
||||
oResetSQL &= $" WHERE {oQuery.QueryConstraint}"
|
||||
End If
|
||||
_MSSQL.ExecuteNonQuery(oResetSQL)
|
||||
|
||||
_Logger.Info("Getting data for query [{0}]", oQuery.Name)
|
||||
|
||||
' get the data from GraphQL
|
||||
Dim oDataResponse = oInterface.GetData(oQuery.QueryString, oQuery.OperationName)
|
||||
Dim oResult As String
|
||||
|
||||
' write data to string
|
||||
@ -51,9 +91,49 @@ Public Class GraphQLJob
|
||||
End Using
|
||||
End Using
|
||||
|
||||
If IsNothing(HandleResponse(oResult, oCurrentQuery, oDatabase)) Then
|
||||
' Fill the query object with field mapping data from TBCUST_JOBRUNNER_QUERY_MAPPING
|
||||
Dim oSQL As String = "SELECT t2.* FROM TBCUST_JOBRUNNER_QUERY_MAPPING t
|
||||
JOIN TBCUST_JOBRUNNER_MAPPING t2 ON t.MAPPING_ID = t2.GUID
|
||||
WHERE t.QUERY_ID = {0}"
|
||||
Dim oMappingTable As DataTable = _MSSQL.GetDatatable(String.Format(oSQL, oQuery.Id))
|
||||
|
||||
For Each oMapping As DataRow In oMappingTable.Rows
|
||||
oQuery.MappingFields.Add(New GraphQL.FieldMapping With {
|
||||
.DestinationColumn = oMapping.Item("DestinationColumn"),
|
||||
.SourcePath = oMapping.Item("SourcePath")
|
||||
})
|
||||
Next
|
||||
|
||||
' Handle the response from GraphQL and insert Data
|
||||
Dim oQueryHandleResult = HandleResponse(oResult, oQuery, oDatabase)
|
||||
|
||||
If IsNothing(oQueryHandleResult) Then
|
||||
Continue For
|
||||
End If
|
||||
|
||||
' Finally delete all old records, when:
|
||||
' - ClearBeforeFill is true, which should be the last query for a given Operation
|
||||
' - QueryConstraint is not empty, which means the records produced by this query can be selected easily
|
||||
If oQuery.ClearBeforeFill = True Or oQuery.QueryConstraint <> String.Empty Then
|
||||
Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 0"
|
||||
If oQuery.QueryConstraint <> String.Empty Then
|
||||
oDeleteSQL &= $" AND {oQuery.QueryConstraint}"
|
||||
End If
|
||||
_MSSQL.ExecuteNonQuery(oDeleteSQL)
|
||||
End If
|
||||
|
||||
Catch ex As Exception
|
||||
_Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", oQuery.Name, oQuery.OperationName)
|
||||
_Logger.Error(ex)
|
||||
|
||||
' If a crash happens, delete all records which were inserted in this run,
|
||||
' thus going back to the previous state
|
||||
Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 1"
|
||||
If oQuery.QueryConstraint <> String.Empty Then
|
||||
oDeleteSQL &= $" AND {oQuery.QueryConstraint}"
|
||||
End If
|
||||
_MSSQL.ExecuteNonQuery(oDeleteSQL)
|
||||
End Try
|
||||
Next
|
||||
|
||||
' logout
|
||||
@ -65,10 +145,9 @@ Public Class GraphQLJob
|
||||
End Try
|
||||
End Sub
|
||||
|
||||
Private Function HandleResponse(JsonString As String, QueryData As GraphQLConfig.Query, DB As Database.MSSQLServer)
|
||||
Private Function HandleResponse(JsonString As String, QueryData As GraphQL.Query, DB As Database.MSSQLServer) As GraphQL.Query
|
||||
Dim oObj As JObject = JObject.Parse(JsonString)
|
||||
Dim oResultList = oObj.SelectToken(QueryData.MappingBasePath)
|
||||
Dim oMappings = QueryData.MappingFields
|
||||
Dim oResultList As JToken = oObj.SelectToken(QueryData.MappingBasePath)
|
||||
|
||||
If oResultList Is Nothing Then
|
||||
_Logger.Warn("HandleResponse: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name)
|
||||
@ -77,25 +156,26 @@ Public Class GraphQLJob
|
||||
|
||||
_Logger.Info("Processing Queue [{0}] with [{1}] Items", QueryData.Name, oResultList.Count)
|
||||
|
||||
If QueryData.ClearBeforeFill Then
|
||||
_Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable)
|
||||
Try
|
||||
DB.ExecuteNonQuery($"DELETE FROM {QueryData.DestinationTable}")
|
||||
Catch ex As Exception
|
||||
_Logger.Error(ex)
|
||||
End Try
|
||||
End If
|
||||
'If QueryData.ClearBeforeFill Then
|
||||
' _Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable)
|
||||
' _Logger.Info("Clear Command: [{0}]", QueryData.ClearCommand)
|
||||
' Try
|
||||
' DB.ExecuteNonQuery(QueryData.ClearCommand)
|
||||
' Catch ex As Exception
|
||||
' _Logger.Error(ex)
|
||||
' End Try
|
||||
'End If
|
||||
|
||||
For Each oResultItem In oResultList
|
||||
For Each oResultItem As JToken In oResultList
|
||||
Try
|
||||
Dim oValues As New List(Of String)
|
||||
Dim oKeys As New List(Of String)
|
||||
|
||||
For Each oMapping In oMappings
|
||||
For Each oMapping In QueryData.MappingFields
|
||||
Dim oToken = oResultItem.SelectToken(oMapping.SourcePath)
|
||||
|
||||
If oToken Is Nothing Then
|
||||
_Logger.Warn("HandleResponse: Could not find SourcePath: {0}", oMapping.SourcePath)
|
||||
_Logger.Warn("HandleResponse: Could not find value at SourcePath: {0}", oMapping.SourcePath)
|
||||
End If
|
||||
|
||||
oValues.Add(oToken.ToString())
|
||||
|
||||
24
Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb
Normal file
24
Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb
Normal file
@ -0,0 +1,24 @@
|
||||
Imports System.Collections.Generic
|
||||
|
||||
Namespace GraphQL
|
||||
Public Class Query
|
||||
Public Property Id As Integer
|
||||
Public Property Name As String
|
||||
Public Property ConnectionId As String = ""
|
||||
Public Property ClearBeforeFill As Boolean = False
|
||||
Public Property ClearCommand As String = ""
|
||||
Public Property QueryString As String = ""
|
||||
Public Property QueryConstraint As String = ""
|
||||
Public Property OperationName As String = ""
|
||||
Public Property DestinationTable As String = ""
|
||||
|
||||
Public Property MappingBasePath As String = ""
|
||||
Public Property MappingFields As New List(Of FieldMapping)
|
||||
End Class
|
||||
|
||||
Public Class FieldMapping
|
||||
Public SourcePath As String = ""
|
||||
Public DestinationColumn As String = ""
|
||||
Public Value As String = ""
|
||||
End Class
|
||||
End Namespace
|
||||
@ -91,6 +91,7 @@
|
||||
<Compile Include="EDMI\GraphQL\GraphQLArgs.vb" />
|
||||
<Compile Include="EDMI\GraphQL\GraphQLConfig.vb" />
|
||||
<Compile Include="EDMI\GraphQL\GraphQLJob.vb" />
|
||||
<Compile Include="EDMI\GraphQL\GraphQLQuery.vb" />
|
||||
<Compile Include="EDMI\ZUGFeRD\EmailData.vb" />
|
||||
<Compile Include="EDMI\ZUGFeRD\EmailFunctions.vb" />
|
||||
<Compile Include="EDMI\ZUGFeRD\EmailStrings.vb" />
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user