Option Explicit On Imports System.Collections.Generic Imports System.Data Imports System.IO Imports System.Linq Imports System.Reflection Imports DigitalData.Modules.Base Imports DigitalData.Modules.Config Imports DigitalData.Modules.Database Imports DigitalData.Modules.Interfaces Imports DigitalData.Modules.Jobs.GraphQL Imports DigitalData.Modules.Logging Imports Newtonsoft.Json.Linq Public Class GraphQLJob Inherits JobBase Implements IJob(Of GraphQLArgs) Private _GraphQL As GraphQLInterface = Nothing Private _Model As GraphQLModel Private _Writer As GraphQLWriter Private Const PLACEHOLDER_STATIC = "STATIC:" Private Const JOB_NAME = "GraphQL Job" Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer) MyBase.New(LogConfig, Nothing, MSSQL) End Sub Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start Try Dim oConfigPath As String = Args.QueryConfigPath Dim oConfigManager As New ConfigManager(Of GraphQLConfig)(_LogConfig, oConfigPath) With oConfigManager.Config _GraphQL = New GraphQLInterface(_LogConfig, .BaseUrl, .Email, .Password, .CertificateFingerprint) End With _Model = New GraphQLModel(_LogConfig, _MSSQL) _Writer = New GraphQLWriter(_LogConfig, _MSSQL) ' Login to get cookie _Logger.Debug("Logging in") Dim oLoginResponse = _GraphQL.Login() ' save cookie for future requests _GraphQL.SaveCookies(oLoginResponse.Cookies.Item(0)) _Logger.Debug("Loading Queries") Dim oQueryList = _Model.GetQueryList() _Logger.Debug("Running [{0}] queries.", oQueryList.Count) ' run For Each oQuery As Query In oQueryList _Logger.Debug("Running Query [{0}].", oQuery.Name) Dim oQueryResult = RunQuery(oQuery) _Logger.Info("Query [{0}] finished with Result [{1}]", oQuery.Name, oQueryResult) Next ' logout _Logger.Debug("Logging out") Dim oLogoutResponse = _GraphQL.Logout() _Logger.Info("Finished GraphQL Job") Catch ex As Exception _Logger.Warn("Finished GraphQL Job with errors") _Logger.Error(ex) Throw ex End Try End Sub Private Function RunQuery(pQuery As Query) Try _Logger.Info("Executing Query [{0}]", pQuery.Name) Dim oConnectionId As Integer = pQuery.ConnectionId Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId) If oConnectionString = String.Empty Then _Logger.Warn("Could not get Connection String for ConnectionId [{0}]", oConnectionId) End If Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString) 'TODO: ONly set status when clear before fill is false 'TODO: ADDED_WHO which contains the query id which inserted the rows ' Clear Table before inserting If pQuery.ClearBeforeFill = True Then If DeleteHistoryTable(pQuery) = False Then Throw New ApplicationException($"Error while dropping history table before fill for Query [{pQuery.Name}]") End If If CreateHistoryTable(pQuery) = False Then Throw New ApplicationException($"Error while creating history table before fill for Query [{pQuery.Name}]") End If If TruncateTable(pQuery) = False Then Throw New ApplicationException($"Error while truncating table before fill for Query [{pQuery.Name}]") End If End If ' Reset all records to status = 0 If pQuery.ClearBeforeFill = False Then _Logger.Info("Resetting data for Query [{0}]", pQuery.Name) If UpdateWithStatus(pQuery, 0) = False Then Throw New ApplicationException($"Error while resetting status of current Records for Query [{pQuery.Name}]") End If End If ' get the data from GraphQL _Logger.Info("Getting data from GraphQL..", pQuery.Name) Dim oDataResponse = _GraphQL.GetData(pQuery.QueryString, pQuery.OperationName) Dim oJsonResult As String ' write data to string Using oStream = oDataResponse.GetResponseStream() Using oReader As New StreamReader(oStream) oJsonResult = oReader.ReadToEnd() End Using End Using _Logger.Debug("Writing JSON data to database..") ' Handle the response from GraphQL and insert Data 'Dim oWriteDataResult As Query = WriteNewQueryData(oResult, pQuery, oDatabase) Dim oWriteDataResult As Query = _Writer.WriteNewQueryData(oJsonResult, pQuery, JOB_NAME) If IsNothing(oWriteDataResult) Then Throw New ApplicationException($"Error while handling Result of Query [{pQuery.Name}]") End If _Logger.Info("New Data successfully inserted for Query [{0}]", pQuery.Name) ' Finally delete all old records If pQuery.ClearBeforeFill = False Then _Logger.Info("Deleting old records for Query [{0}].", pQuery.Name) If DeleteWithStatus(pQuery, 0) = False Then Throw New ApplicationException($"Error while deleting current Records for Query [{pQuery.Name}]") End If End If Return True Catch ex As Exception _Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", pQuery.Name, pQuery.OperationName) _Logger.Error(ex) ' If a crash happens, delete all records which were inserted in this run, ' thus going back to the previous state _Logger.Info("Failure, deleting new records..", pQuery.Name) If pQuery.ClearBeforeFill = False Then If DeleteWithStatus(pQuery, 1) = False Then Throw New ApplicationException($"Error while deleting new Records for Query [{pQuery.Name}]") End If End If Return False Finally _Logger.Debug("Finished running Query [{0}].", pQuery.Name) End Try End Function Private Function DeleteHistoryTable(pQuery As Query) As Boolean Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY" Dim oDeleteHistorySQL = $" IF (EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = '{oHistoryTableName}')) BEGIN DROP TABLE {oHistoryTableName}; END" Return _MSSQL.ExecuteNonQuery(oDeleteHistorySQL) End Function Private Function TruncateTable(pQuery As Query) Dim oDeleteTableSQL = $"TRUNCATE TABLE {pQuery.DestinationTable}" Return _MSSQL.ExecuteNonQuery(oDeleteTableSQL) End Function Private Function CreateHistoryTable(pQuery As Query) As Boolean Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY" Dim oFillHistorySQL = $"SELECT * INTO {oHistoryTableName} FROM {pQuery.DestinationTable}" Return _MSSQL.ExecuteNonQuery(oFillHistorySQL) End Function Private Function DeleteWithStatus(pQuery As Query, pStatus As Integer) As Boolean Dim oDeleteSQL = $"DELETE FROM {pQuery.DestinationTable} WHERE STATUS = {pStatus} AND ADDED_QUERY_ID = '{pQuery.Id}'" Return _MSSQL.ExecuteNonQuery(oDeleteSQL) End Function Private Function UpdateWithStatus(pQuery As Query, pStatus As Integer) Dim oResetSQL = $"UPDATE {pQuery.DestinationTable} SET STATUS = {pStatus} WHERE ADDED_QUERY_ID = '{pQuery.Id}'" Return _MSSQL.ExecuteNonQuery(oResetSQL) End Function Public Function ShouldStart(Arguments As GraphQLArgs) As Boolean Implements IJob(Of GraphQLArgs).ShouldStart Return Arguments.Enabled End Function End Class