Modules/Jobs/GraphQL/GraphQLJob.vb

Option Explicit On

Imports System.Collections.Generic
Imports System.Data
Imports System.IO
Imports System.Linq
Imports System.Reflection
Imports DigitalData.Modules.Base
Imports DigitalData.Modules.Config
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Interfaces
Imports DigitalData.Modules.Jobs.GraphQL
Imports DigitalData.Modules.Logging
Imports Newtonsoft.Json.Linq

Public Class GraphQLJob
    Inherits JobBase
    Implements IJob(Of GraphQLArgs)

    Private _GraphQL As GraphQLInterface = Nothing
    Private _Model As GraphQLModel
    Private _Writer As GraphQLWriter

    Private Const PLACEHOLDER_STATIC = "STATIC:"
    Private Const JOB_NAME = "GraphQL Job"

    Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer)
        MyBase.New(LogConfig, Nothing, MSSQL)
    End Sub

    Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start
        Try
            Dim oConfigPath As String = Args.QueryConfigPath
            Dim oConfigManager As New ConfigManager(Of GraphQLConfig)(_LogConfig, oConfigPath)

            With oConfigManager.Config
                _GraphQL = New GraphQLInterface(_LogConfig, .BaseUrl, .Email, .Password, .CertificateFingerprint)
            End With

            _Model = New GraphQLModel(_LogConfig, _MSSQL)
            _Writer = New GraphQLWriter(_LogConfig, _MSSQL)

            ' Login to get cookie
            _Logger.Debug("Logging in")
            Dim oLoginResponse = _GraphQL.Login()

            ' save cookie for future requests
            _GraphQL.SaveCookies(oLoginResponse.Cookies.Item(0))

            _Logger.Debug("Loading Queries")

            Dim oQueryList = _Model.GetQueryList()

            _Logger.Debug("Running [{0}] queries.", oQueryList.Count)

            ' run
            For Each oQuery As Query In oQueryList
                _Logger.Debug("Running Query [{0}].", oQuery.Name)
                Dim oQueryResult = RunQuery(oQuery)
                _Logger.Info("Query [{0}] finished with Result [{1}]", oQuery.Name, oQueryResult)
            Next

            ' logout
            _Logger.Debug("Logging out")
            Dim oLogoutResponse = _GraphQL.Logout()

            _Logger.Info("Finished GraphQL Job")

        Catch ex As Exception
            _Logger.Warn("Finished GraphQL Job with errors")
            _Logger.Error(ex)
            Throw ex
        End Try
    End Sub

    Private Function RunQuery(pQuery As Query)
        Try
            _Logger.Info("Executing Query [{0}]", pQuery.Name)

            Dim oConnectionId As Integer = pQuery.ConnectionId
            Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId)

            If oConnectionString = String.Empty Then
                _Logger.Warn("Could not get Connection String for ConnectionId [{0}]", oConnectionId)
            End If

            Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString)

            'TODO: ONly set status when clear before fill is false
            'TODO: ADDED_WHO which contains the query id which inserted the rows

            ' Clear Table before inserting
            If pQuery.ClearBeforeFill = True Then
                If DeleteHistoryTable(pQuery) = False Then
                    Throw New ApplicationException($"Error while dropping history table before fill for Query [{pQuery.Name}]")
                End If

                If CreateHistoryTable(pQuery) = False Then
                    Throw New ApplicationException($"Error while creating history table before fill for Query [{pQuery.Name}]")
                End If

                If TruncateTable(pQuery) = False Then
                    Throw New ApplicationException($"Error while truncating table before fill for Query [{pQuery.Name}]")
                End If
            End If

            ' Reset all records to status = 0
            If pQuery.ClearBeforeFill = False Then
                _Logger.Info("Resetting data for Query [{0}]", pQuery.Name)
                If UpdateWithStatus(pQuery, 0) = False Then
                    Throw New ApplicationException($"Error while resetting status of current Records for Query [{pQuery.Name}]")
                End If
            End If

            ' get the data from GraphQL
            _Logger.Info("Getting data from GraphQL..", pQuery.Name)
            Dim oDataResponse = _GraphQL.GetData(pQuery.QueryString, pQuery.OperationName)
            Dim oJsonResult As String

            ' write data to string
            Using oStream = oDataResponse.GetResponseStream()
                Using oReader As New StreamReader(oStream)
                    oJsonResult = oReader.ReadToEnd()
                End Using
            End Using

            _Logger.Debug("Writing JSON data to database..")

            ' Handle the response from GraphQL and insert Data
            'Dim oWriteDataResult As Query = WriteNewQueryData(oResult, pQuery, oDatabase)
            Dim oWriteDataResult As Query = _Writer.WriteNewQueryData(oJsonResult, pQuery, JOB_NAME)

            If IsNothing(oWriteDataResult) Then
                Throw New ApplicationException($"Error while handling Result of Query [{pQuery.Name}]")
            End If

            _Logger.Info("New Data successfully inserted for Query [{0}]", pQuery.Name)

            ' Finally delete all old records
            If pQuery.ClearBeforeFill = False Then
                _Logger.Info("Deleting old records for Query [{0}].", pQuery.Name)
                If DeleteWithStatus(pQuery, 0) = False Then
                    Throw New ApplicationException($"Error while deleting current Records for Query [{pQuery.Name}]")
                End If
            End If

            Return True

        Catch ex As Exception
            _Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", pQuery.Name, pQuery.OperationName)
            _Logger.Error(ex)

            ' If a crash happens, delete all records which were inserted in this run,
            ' thus going back to the previous state
            _Logger.Info("Failure, deleting new records..", pQuery.Name)

            If pQuery.ClearBeforeFill = False Then
                If DeleteWithStatus(pQuery, 1) = False Then
                    Throw New ApplicationException($"Error while deleting new Records for Query [{pQuery.Name}]")
                End If
            End If

            Return False

        Finally
            _Logger.Debug("Finished running Query [{0}].", pQuery.Name)
        End Try
    End Function

    Private Function DeleteHistoryTable(pQuery As Query) As Boolean
        Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY"
        Dim oDeleteHistorySQL = $"
            IF (EXISTS (SELECT *
            FROM INFORMATION_SCHEMA.TABLES
            WHERE TABLE_SCHEMA = 'dbo'
            AND TABLE_NAME = '{oHistoryTableName}'))
            BEGIN
            DROP TABLE {oHistoryTableName};
            END"

        Return _MSSQL.ExecuteNonQuery(oDeleteHistorySQL)
    End Function

    Private Function TruncateTable(pQuery As Query)
        Dim oDeleteTableSQL = $"TRUNCATE TABLE {pQuery.DestinationTable}"

        Return _MSSQL.ExecuteNonQuery(oDeleteTableSQL)
    End Function

    Private Function CreateHistoryTable(pQuery As Query) As Boolean
        Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY"
        Dim oFillHistorySQL = $"SELECT * INTO {oHistoryTableName} FROM {pQuery.DestinationTable}"

        Return _MSSQL.ExecuteNonQuery(oFillHistorySQL)
    End Function

    Private Function DeleteWithStatus(pQuery As Query, pStatus As Integer) As Boolean
        Dim oDeleteSQL = $"DELETE FROM {pQuery.DestinationTable} WHERE STATUS = {pStatus} AND ADDED_QUERY_ID = '{pQuery.Id}'"
        Return _MSSQL.ExecuteNonQuery(oDeleteSQL)
    End Function

    Private Function UpdateWithStatus(pQuery As Query, pStatus As Integer)
        Dim oResetSQL = $"UPDATE {pQuery.DestinationTable} SET STATUS = {pStatus} WHERE ADDED_QUERY_ID = '{pQuery.Id}'"
        Return _MSSQL.ExecuteNonQuery(oResetSQL)
    End Function

    Public Function ShouldStart(Arguments As GraphQLArgs) As Boolean Implements IJob(Of GraphQLArgs).ShouldStart
        Return Arguments.Enabled
    End Function
End Class