Option Explicit On Imports System.IO Imports DigitalData.Modules.Config Imports DigitalData.Modules.Database Imports DigitalData.Modules.Interfaces Imports DigitalData.Modules.Jobs.GraphQL Imports DigitalData.Modules.Logging Public Class GraphQLJob Inherits JobBase Implements IJob(Of GraphQLArgs) Private _GraphQL As GraphQLInterface = Nothing Private _Model As GraphQLModel Private _Writer As GraphQLWriter Private Const PLACEHOLDER_STATIC = "STATIC:" Private Const JOB_NAME = "GraphQL Job" Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer) MyBase.New(LogConfig, MSSQL) End Sub Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start Try Dim oConfigPath As String = Args.QueryConfigPath Dim oConfigManager As New ConfigManager(Of GraphQLConfig)(_LogConfig, oConfigPath) With oConfigManager.Config _GraphQL = New GraphQLInterface(_LogConfig, .BaseUrl, .Email, .Password, .CertificateFingerprint) End With _Logger.Info($"baseUrl is: {oConfigManager.Config.BaseUrl}") _Model = New GraphQLModel(_LogConfig, _MSSQL) _Writer = New GraphQLWriter(_LogConfig, _MSSQL) ' Login to get cookie _Logger.Debug("Logging in") Dim oLoginResponse = _GraphQL.Login() ' save cookie for future requests _GraphQL.SaveCookies(oLoginResponse.Cookies.Item(0)) _Logger.Debug("Loading Queries") Dim oQueryList = _Model.GetQueryList() _Logger.Debug("Running [{0}] queries.", oQueryList.Count) ' run For Each oQuery As Query In oQueryList _Logger.Debug("Running Query [{0}].", oQuery.Name) Dim oQueryResult = RunQuery(oQuery) _Logger.Info("Query [{0}] finished with Result [{1}]", oQuery.Name, oQueryResult) Next ' logout _Logger.Debug("Logging out") Dim oLogoutResponse = _GraphQL.Logout() _Logger.Info("Finished GraphQL Job") Catch ex As Exception _Logger.Warn("Finished GraphQL Job with errors") _Logger.Error(ex) Throw ex End Try End Sub Private Function RunQuery(pQuery As Query) Try _Logger.Info("Executing Query [{0}]", pQuery.Name) Dim oConnectionId As Integer = pQuery.ConnectionId Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId) If oConnectionString = String.Empty Then _Logger.Warn("Could not get Connection String for ConnectionId [{0}]", oConnectionId) End If Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString) 'TODO: ONly set status when clear before fill is false 'TODO: ADDED_WHO which contains the query id which inserted the rows ' Clear Table before inserting If pQuery.ClearBeforeFill = True Then If DeleteHistoryTable(pQuery) = False Then Throw New ApplicationException($"Error while dropping history table before fill for Query [{pQuery.Name}]") End If If CreateHistoryTable(pQuery) = False Then Throw New ApplicationException($"Error while creating history table before fill for Query [{pQuery.Name}]") End If If TruncateTable(pQuery) = False Then Throw New ApplicationException($"Error while truncating table before fill for Query [{pQuery.Name}]") End If End If ' Reset all records to status = 0 If pQuery.ClearBeforeFill = False Then _Logger.Info("Resetting data for Query [{0}]", pQuery.Name) If UpdateWithStatus(pQuery, 0) = False Then Throw New ApplicationException($"Error while resetting status of current Records for Query [{pQuery.Name}]") End If End If ' get the data from GraphQL _Logger.Info("Getting data from GraphQL..", pQuery.Name) Dim oDataResponse = _GraphQL.GetData(pQuery.QueryString, pQuery.OperationName) Dim oJsonResult As String ' write data to string Using oStream = oDataResponse.GetResponseStream() Using oReader As New StreamReader(oStream) oJsonResult = oReader.ReadToEnd() End Using End Using _Logger.Debug("Writing JSON data to database..") ' Handle the response from GraphQL and insert Data 'Dim oWriteDataResult As Query = WriteNewQueryData(oResult, pQuery, oDatabase) Dim oWriteDataResult As Query = _Writer.WriteNewQueryData(oJsonResult, pQuery, JOB_NAME) If IsNothing(oWriteDataResult) Then Throw New ApplicationException($"Error while handling Result of Query [{pQuery.Name}]") End If _Logger.Info("New Data successfully inserted for Query [{0}]", pQuery.Name) ' Finally delete all old records If pQuery.ClearBeforeFill = False Then _Logger.Info("Deleting old records for Query [{0}].", pQuery.Name) If DeleteWithStatus(pQuery, 0) = False Then Throw New ApplicationException($"Error while deleting current Records for Query [{pQuery.Name}]") End If End If Return True Catch ex As Exception _Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", pQuery.Name, pQuery.OperationName) _Logger.Error(ex) ' If a crash happens, delete all records which were inserted in this run, ' thus going back to the previous state _Logger.Info("Failure, deleting new records..", pQuery.Name) If pQuery.ClearBeforeFill = False Then If DeleteWithStatus(pQuery, 1) = False Then Throw New ApplicationException($"Error while deleting new Records for Query [{pQuery.Name}]") End If End If Return False Finally _Logger.Debug("Finished running Query [{0}].", pQuery.Name) End Try End Function Private Function DeleteHistoryTable(pQuery As Query) As Boolean Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY" Dim oDeleteHistorySQL = $" IF (EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = '{oHistoryTableName}')) BEGIN DROP TABLE {oHistoryTableName}; END" Return _MSSQL.ExecuteNonQuery(oDeleteHistorySQL) End Function Private Function TruncateTable(pQuery As Query) Dim oDeleteTableSQL = $"TRUNCATE TABLE {pQuery.DestinationTable}" Return _MSSQL.ExecuteNonQuery(oDeleteTableSQL) End Function Private Function CreateHistoryTable(pQuery As Query) As Boolean Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY" Dim oFillHistorySQL = $"SELECT * INTO {oHistoryTableName} FROM {pQuery.DestinationTable}" Return _MSSQL.ExecuteNonQuery(oFillHistorySQL) End Function Private Function DeleteWithStatus(pQuery As Query, pStatus As Integer) As Boolean Dim oDeleteSQL = $"DELETE FROM {pQuery.DestinationTable} WHERE STATUS = {pStatus} AND ADDED_QUERY_ID = '{pQuery.Id}'" Return _MSSQL.ExecuteNonQuery(oDeleteSQL) End Function Private Function UpdateWithStatus(pQuery As Query, pStatus As Integer) Dim oResetSQL = $"UPDATE {pQuery.DestinationTable} SET STATUS = {pStatus} WHERE ADDED_QUERY_ID = '{pQuery.Id}'" Return _MSSQL.ExecuteNonQuery(oResetSQL) End Function Public Function ShouldStart(Arguments As GraphQLArgs) As Boolean Implements IJob(Of GraphQLArgs).ShouldStart Return Arguments.Enabled End Function End Class