AppStd/Modules
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Jobs: Rewrite GraphQL Job

Browse Source
This commit is contained in:
Jonathan Jenne
2024-01-23 13:33:44 +01:00
parent 9f9feff2fd
commit e85379dcf5
6 changed files with 286 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
Jobs/GraphQL/GraphQLJob.vb
View File

@@ -4,6 +4,7 @@ Imports System.Collections.Generic
Imports System.Data
Imports System.IO
Imports System.Linq
Imports System.Reflection
Imports DigitalData.Modules.Base
Imports DigitalData.Modules.Config
Imports DigitalData.Modules.Database
@@ -17,6 +18,8 @@ Public Class GraphQLJob
Implements IJob(Of GraphQLArgs)
Private _GraphQL As GraphQLInterface = Nothing
Private _Model As GraphQLModel
Private _Writer As GraphQLWriter
Private Const PLACEHOLDER_STATIC = "STATIC:"
Private Const JOB_NAME = "GraphQL Job"
@@ -34,6 +37,9 @@ Public Class GraphQLJob
_GraphQL = New GraphQLInterface(_LogConfig, .BaseUrl, .Email, .Password, .CertificateFingerprint)
End With
_Model = New GraphQLModel(_LogConfig, _MSSQL)
_Writer = New GraphQLWriter(_LogConfig, _MSSQL)
' Login to get cookie
_Logger.Debug("Logging in")
Dim oLoginResponse = _GraphQL.Login()
@@ -43,41 +49,7 @@ Public Class GraphQLJob
_Logger.Debug("Loading Queries")
' Load query data from TBCUST_JOBRUNNER_QUERY
Dim oQueryTable As DataTable = _MSSQL.GetDatatable("SELECT * FROM TBCUST_JOBRUNNER_QUERY ORDER BY SEQUENCE")
Dim oQueryList As New List(Of Query)
' Save query data to business objects
For Each oRow As DataRow In oQueryTable.Rows
Dim oQuery As New Query With {
.Id = oRow.Item("GUID"),
.Name = oRow.Item("TITLE"),
.ClearBeforeFill = oRow.ItemEx("CLEAR_BEFORE_FILL", False),
.ConnectionId = oRow.ItemEx("CON_ID", 1), ' TODO: Connection String?
.DestinationTable = oRow.ItemEx("DESTINATION_TABLE", String.Empty),
.OperationName = oRow.ItemEx("OPERATION_NAME", String.Empty),
.MappingBasePath = oRow.ItemEx("MAPPING_BASE_PATH", String.Empty),
.QueryString = oRow.ItemEx("QUERY_STRING", String.Empty)
}
If oQuery.DestinationTable = String.Empty Then
_Logger.Warn("Value [DestinationTable] could not be read. Configuration incomplete.")
End If
If oQuery.OperationName = String.Empty Then
_Logger.Warn("Value [OperationName] could not be read. Configuration incomplete.")
End If
If oQuery.MappingBasePath = String.Empty Then
_Logger.Warn("Value [MappingBasePath] could not be read. Configuration incomplete.")
End If
If oQuery.QueryString = String.Empty Then
_Logger.Warn("Value [QueryString] could not be read. Configuration incomplete.")
End If
oQueryList.Add(oQuery)
Next
Dim oQueryList = _Model.GetQueryList()
_Logger.Debug("Running [{0}] queries.", oQueryList.Count)
@@ -132,8 +104,8 @@ Public Class GraphQLJob
End If
End If
' Reset all records to status = 0
If pQuery.ClearBeforeFill = False Then
' Reset all records to status = 0
If pQuery.ClearBeforeFill = False Then
_Logger.Info("Resetting data for Query [{0}]", pQuery.Name)
If UpdateWithStatus(pQuery, 0) = False Then
Throw New ApplicationException($"Error while resetting status of current Records for Query [{pQuery.Name}]")
@@ -141,31 +113,22 @@ Public Class GraphQLJob
End If
' get the data from GraphQL
_Logger.Info("Getting data..", pQuery.Name)
_Logger.Info("Getting data from GraphQL..", pQuery.Name)
Dim oDataResponse = _GraphQL.GetData(pQuery.QueryString, pQuery.OperationName)
Dim oResult As String
Dim oJsonResult As String
' write data to string
Using oStream = oDataResponse.GetResponseStream()
Using oReader As New StreamReader(oStream)
oResult = oReader.ReadToEnd()
oJsonResult = oReader.ReadToEnd()
End Using
End Using
' Fill the query object with field mapping data from TBCUST_JOBRUNNER_QUERY_MAPPING
Dim oSQL As String = "SELECT t2.* FROM TBCUST_JOBRUNNER_QUERY_MAPPING t
JOIN TBCUST_JOBRUNNER_MAPPING t2 ON t.MAPPING_ID = t2.GUID
WHERE t.QUERY_ID = {0}"
Dim oMappingTable As DataTable = _MSSQL.GetDatatable(String.Format(oSQL, pQuery.Id))
For Each oMapping As DataRow In oMappingTable.Rows
pQuery.MappingFields.Add(New GraphQL.FieldMapping With {
.DestinationColumn = oMapping.Item("DestinationColumn"),
.SourcePath = oMapping.Item("SourcePath")
})
Next
_Logger.Debug("Writing JSON data to database..")
' Handle the response from GraphQL and insert Data
Dim oWriteDataResult As GraphQL.Query = WriteNewQueryData(oResult, pQuery, oDatabase)
'Dim oWriteDataResult As Query = WriteNewQueryData(oResult, pQuery, oDatabase)
Dim oWriteDataResult As Query = _Writer.WriteNewQueryData(oJsonResult, pQuery, JOB_NAME)
If IsNothing(oWriteDataResult) Then
Throw New ApplicationException($"Error while handling Result of Query [{pQuery.Name}]")
@@ -241,72 +204,6 @@ Public Class GraphQLJob
Return _MSSQL.ExecuteNonQuery(oResetSQL)
End Function
Private Function WriteNewQueryData(JsonString As String, QueryData As GraphQL.Query, DB As Database.MSSQLServer) As GraphQL.Query
Dim oObj As JObject = JObject.Parse(JsonString)
Dim oResultList As JToken
If _GraphQL.ReadJSONPathFragmented(oObj, QueryData.MappingBasePath) = False Then
_Logger.Warn("There is an error in the MappingBasePath [{1}] configuration of query [{0}]", QueryData.Name, QueryData.MappingBasePath)
End If
Try
oResultList = oObj.SelectToken(QueryData.MappingBasePath, errorWhenNoMatch:=True)
Catch ex As Exception
_Logger.Warn("WriteNewQueryData: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name)
_Logger.Error(ex)
Return Nothing
End Try
If oResultList Is Nothing Then
_Logger.Warn("WriteNewQueryData: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name)
Return Nothing
End If
_Logger.Info("WriteNewQueryData: Processing Queue [{0}] with [{1}] Items", QueryData.Name, oResultList.Count)
For Each oResultItem As JToken In oResultList
Try
' ADDED_WHO, ADDED_QUERY_ID are system fields which are used to correctly fill
' and delete rows in the destination table without touching rows from other queries
Dim oKeys As New List(Of String) From {"ADDED_WHO", "ADDED_QUERY_ID", "STATUS"}
Dim oValues As New List(Of String) From {JOB_NAME, QueryData.Id, "1"}
For Each oMapping In QueryData.MappingFields
Dim oValue As String = String.Empty
If oMapping.SourcePath.StartsWith(PLACEHOLDER_STATIC) Then
oValue = oMapping.SourcePath.Replace(PLACEHOLDER_STATIC, String.Empty)
Else
Dim oToken = oResultItem.SelectToken(oMapping.SourcePath)
If oToken Is Nothing Then
_Logger.Warn("WriteNewQueryData: Could not find value at SourcePath: {0}", oMapping.SourcePath)
oValue = String.Empty
Else
oValue = oToken.ToString
End If
End If
oValues.Add(oValue)
oKeys.Add(oMapping.DestinationColumn)
Next
Dim oColumnString = String.Join(",", oKeys.ToArray)
Dim oValueList = oValues.Select(Function(Value) $"'{Value.EscapeForSQL}'").ToList()
Dim oValueString = String.Join(",", oValueList)
Dim oSQL As String = $"INSERT INTO {QueryData.DestinationTable} ({oColumnString}) VALUES ({oValueString})"
DB.ExecuteNonQuery(oSQL)
Catch ex As Exception
_Logger.Error(ex)
End Try
Next
Return QueryData
End Function
Public Function ShouldStart(Arguments As GraphQLArgs) As Boolean Implements IJob(Of GraphQLArgs).ShouldStart
Return Arguments.Enabled
End Function