From 2d08ab2c22cc3b9454856e81b3be552955002d80 Mon Sep 17 00:00:00 2001 From: Jonathan Jenne Date: Wed, 10 Mar 2021 13:19:01 +0100 Subject: [PATCH] JobRunner: Load Jobrunner GraphQL Queries via Database --- Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb | 23 +--- Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb | 146 ++++++++++++++++----- Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb | 24 ++++ Modules.Jobs/Jobs.vbproj | 1 + 4 files changed, 139 insertions(+), 55 deletions(-) create mode 100644 Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb diff --git a/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb b/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb index 383e58ff..e9ca33aa 100644 --- a/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb +++ b/Modules.Jobs/EDMI/GraphQL/GraphQLConfig.vb @@ -1,27 +1,6 @@ -Imports System.Collections.Generic - -Public Class GraphQLConfig +Public Class GraphQLConfig Public Property BaseUrl As String = "" Public Property Email As String = "" Public Property Password As String = "" Public Property CertificateFingerprint As String = "" - Public Property Queries As New List(Of Query) - - Public Class Query - Public Property Name As String - Public Property ConnectionString As String = "" - Public Property ClearBeforeFill As Boolean = False - Public Property QueryString As String = "" - Public Property OperationName As String = "" - Public Property DestinationTable As String = "" - - Public Property MappingBasePath As String = "" - Public Property MappingFields As New List(Of FieldMapping) - End Class - - Public Class FieldMapping - Public SourcePath As String = "" - Public DestinationColumn As String = "" - Public Value As String = "" - End Class End Class diff --git a/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb b/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb index 2cac8caf..741464df 100644 --- a/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb +++ b/Modules.Jobs/EDMI/GraphQL/GraphQLJob.vb @@ -1,4 +1,6 @@ -Imports System.IO +Option Explicit On + +Imports System.IO Imports DigitalData.Modules.Interfaces Imports DigitalData.Modules.Jobs Imports DigitalData.Modules.Config @@ -7,13 +9,15 @@ Imports Newtonsoft.Json.Linq Imports System.Collections.Generic Imports System.Linq Imports System.Text.RegularExpressions +Imports DigitalData.Modules.Database +Imports System.Data Public Class GraphQLJob Inherits JobBase Implements IJob(Of GraphQLArgs) - Public Sub New(LogConfig As LogConfig) - MyBase.New(LogConfig, Nothing, Nothing) + Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer) + MyBase.New(LogConfig, Nothing, MSSQL) End Sub Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start @@ -34,26 +38,102 @@ Public Class GraphQLJob ' save cookie for future requests oInterface.SaveCookies(oLoginResponse.Cookies.Item(0)) - _Logger.Debug("Getting the data") + _Logger.Debug("Loading Queries") + + ' Load query data from TBCUST_JOBRUNNER_QUERY + Dim oQueryTable As DataTable = _MSSQL.GetDatatable("SELECT * FROM TBCUST_JOBRUNNER_QUERY ORDER BY OPERATION_NAME, CLEAR_BEFORE_FILL ASC") + Dim oQueryList As New List(Of GraphQL.Query) + + ' Save query data to business objects + For Each oRow As DataRow In oQueryTable.Rows + Dim oQuery As New GraphQL.Query With { + .Id = oRow.Item("GUID"), + .Name = oRow.Item("TITLE"), + .ClearBeforeFill = oRow.Item("CLEAR_BEFORE_FILL"), + .ConnectionId = oRow.Item("CON_ID"), ' TODO: Connection String? + .DestinationTable = oRow.Item("DESTINATION_TABLE"), + .OperationName = oRow.Item("OPERATION_NAME"), + .MappingBasePath = oRow.Item("MAPPING_BASE_PATH"), + .QueryString = oRow.Item("QUERY_STRING"), + .QueryConstraint = oRow.Item("QUERY_CONSTRAINT") + } + oQueryList.Add(oQuery) + Next + + _Logger.Debug("Getting the data from GraphQL") + + For Each oQuery As GraphQL.Query In oQueryList + Try + Dim oConnectionId As Integer = oQuery.ConnectionId + Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId) + + Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString) + + _Logger.Info("Resetting data for query [{0}] with constraint [{1}]", oQuery.Name, oQuery.QueryConstraint) - For Each oQuery As GraphQLConfig.Query In oConfigManager.Config.Queries - Dim oCurrentQuery = oQuery - Dim oDatabase As New Database.MSSQLServer(_LogConfig, oQuery.ConnectionString) + ' Reset all records to status = 0 + Dim oResetSQL = $"UPDATE {oQuery.DestinationTable} SET STATUS = 0" + If oQuery.QueryConstraint <> String.Empty Then + oResetSQL &= $" WHERE {oQuery.QueryConstraint}" + End If + _MSSQL.ExecuteNonQuery(oResetSQL) + + _Logger.Info("Getting data for query [{0}]", oQuery.Name) - ' get the data - Dim oDataResponse = oInterface.GetData(oCurrentQuery.QueryString, oCurrentQuery.OperationName) - Dim oResult As String + ' get the data from GraphQL + Dim oDataResponse = oInterface.GetData(oQuery.QueryString, oQuery.OperationName) + Dim oResult As String - ' write data to string - Using oStream = oDataResponse.GetResponseStream() - Using oReader As New StreamReader(oStream) - oResult = oReader.ReadToEnd() + ' write data to string + Using oStream = oDataResponse.GetResponseStream() + Using oReader As New StreamReader(oStream) + oResult = oReader.ReadToEnd() + End Using End Using - End Using - If IsNothing(HandleResponse(oResult, oCurrentQuery, oDatabase)) Then - Continue For - End If + ' Fill the query object with field mapping data from TBCUST_JOBRUNNER_QUERY_MAPPING + Dim oSQL As String = "SELECT t2.* FROM TBCUST_JOBRUNNER_QUERY_MAPPING t + JOIN TBCUST_JOBRUNNER_MAPPING t2 ON t.MAPPING_ID = t2.GUID + WHERE t.QUERY_ID = {0}" + Dim oMappingTable As DataTable = _MSSQL.GetDatatable(String.Format(oSQL, oQuery.Id)) + + For Each oMapping As DataRow In oMappingTable.Rows + oQuery.MappingFields.Add(New GraphQL.FieldMapping With { + .DestinationColumn = oMapping.Item("DestinationColumn"), + .SourcePath = oMapping.Item("SourcePath") + }) + Next + + ' Handle the response from GraphQL and insert Data + Dim oQueryHandleResult = HandleResponse(oResult, oQuery, oDatabase) + + If IsNothing(oQueryHandleResult) Then + Continue For + End If + + ' Finally delete all old records, when: + ' - ClearBeforeFill is true, which should be the last query for a given Operation + ' - QueryConstraint is not empty, which means the records produced by this query can be selected easily + If oQuery.ClearBeforeFill = True Or oQuery.QueryConstraint <> String.Empty Then + Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 0" + If oQuery.QueryConstraint <> String.Empty Then + oDeleteSQL &= $" AND {oQuery.QueryConstraint}" + End If + _MSSQL.ExecuteNonQuery(oDeleteSQL) + End If + + Catch ex As Exception + _Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", oQuery.Name, oQuery.OperationName) + _Logger.Error(ex) + + ' If a crash happens, delete all records which were inserted in this run, + ' thus going back to the previous state + Dim oDeleteSQL = $"DELETE FROM {oQuery.DestinationTable} WHERE STATUS = 1" + If oQuery.QueryConstraint <> String.Empty Then + oDeleteSQL &= $" AND {oQuery.QueryConstraint}" + End If + _MSSQL.ExecuteNonQuery(oDeleteSQL) + End Try Next ' logout @@ -65,10 +145,9 @@ Public Class GraphQLJob End Try End Sub - Private Function HandleResponse(JsonString As String, QueryData As GraphQLConfig.Query, DB As Database.MSSQLServer) + Private Function HandleResponse(JsonString As String, QueryData As GraphQL.Query, DB As Database.MSSQLServer) As GraphQL.Query Dim oObj As JObject = JObject.Parse(JsonString) - Dim oResultList = oObj.SelectToken(QueryData.MappingBasePath) - Dim oMappings = QueryData.MappingFields + Dim oResultList As JToken = oObj.SelectToken(QueryData.MappingBasePath) If oResultList Is Nothing Then _Logger.Warn("HandleResponse: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name) @@ -77,25 +156,26 @@ Public Class GraphQLJob _Logger.Info("Processing Queue [{0}] with [{1}] Items", QueryData.Name, oResultList.Count) - If QueryData.ClearBeforeFill Then - _Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable) - Try - DB.ExecuteNonQuery($"DELETE FROM {QueryData.DestinationTable}") - Catch ex As Exception - _Logger.Error(ex) - End Try - End If - - For Each oResultItem In oResultList + 'If QueryData.ClearBeforeFill Then + ' _Logger.Info("Clearing Table {0} before insert", QueryData.DestinationTable) + ' _Logger.Info("Clear Command: [{0}]", QueryData.ClearCommand) + ' Try + ' DB.ExecuteNonQuery(QueryData.ClearCommand) + ' Catch ex As Exception + ' _Logger.Error(ex) + ' End Try + 'End If + + For Each oResultItem As JToken In oResultList Try Dim oValues As New List(Of String) Dim oKeys As New List(Of String) - For Each oMapping In oMappings + For Each oMapping In QueryData.MappingFields Dim oToken = oResultItem.SelectToken(oMapping.SourcePath) If oToken Is Nothing Then - _Logger.Warn("HandleResponse: Could not find SourcePath: {0}", oMapping.SourcePath) + _Logger.Warn("HandleResponse: Could not find value at SourcePath: {0}", oMapping.SourcePath) End If oValues.Add(oToken.ToString()) diff --git a/Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb b/Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb new file mode 100644 index 00000000..408bd5f9 --- /dev/null +++ b/Modules.Jobs/EDMI/GraphQL/GraphQLQuery.vb @@ -0,0 +1,24 @@ +Imports System.Collections.Generic + +Namespace GraphQL + Public Class Query + Public Property Id As Integer + Public Property Name As String + Public Property ConnectionId As String = "" + Public Property ClearBeforeFill As Boolean = False + Public Property ClearCommand As String = "" + Public Property QueryString As String = "" + Public Property QueryConstraint As String = "" + Public Property OperationName As String = "" + Public Property DestinationTable As String = "" + + Public Property MappingBasePath As String = "" + Public Property MappingFields As New List(Of FieldMapping) + End Class + + Public Class FieldMapping + Public SourcePath As String = "" + Public DestinationColumn As String = "" + Public Value As String = "" + End Class +End Namespace diff --git a/Modules.Jobs/Jobs.vbproj b/Modules.Jobs/Jobs.vbproj index d439a7e2..756bc825 100644 --- a/Modules.Jobs/Jobs.vbproj +++ b/Modules.Jobs/Jobs.vbproj @@ -91,6 +91,7 @@ +