Modules/Jobs/GraphQL/GraphQLJob.vb

Option Explicit On
Imports System.IO
Imports DigitalData.Modules.Config
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Interfaces
Imports DigitalData.Modules.Jobs.GraphQL
Imports DigitalData.Modules.Logging

Public Class GraphQLJob
    Inherits JobBase
    Implements IJob(Of GraphQLArgs)

    Private _GraphQL As GraphQLInterface = Nothing
    Private _Model As GraphQLModel
    Private _Writer As GraphQLWriter

    Private Const PLACEHOLDER_STATIC = "STATIC:"
    Private Const JOB_NAME = "GraphQL Job"

    Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer)
        MyBase.New(LogConfig, MSSQL)
    End Sub

    Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start
        Try
            Dim oConfigPath As String = Args.QueryConfigPath
            Dim oConfigManager As New ConfigManager(Of GraphQLConfig)(_LogConfig, oConfigPath)

            With oConfigManager.Config
                _GraphQL = New GraphQLInterface(_LogConfig, .BaseUrl, .Email, .Password, .CertificateFingerprint)
            End With
            _Logger.Info($"baseUrl is: {oConfigManager.Config.BaseUrl}")
            _Model = New GraphQLModel(_LogConfig, _MSSQL)
            _Writer = New GraphQLWriter(_LogConfig, _MSSQL)

            ' Login to get cookie
            _Logger.Debug("Logging in")
            Dim oLoginResponse = _GraphQL.Login()

            ' save cookie for future requests
            _GraphQL.SaveCookies(oLoginResponse.Cookies.Item(0))

            _Logger.Debug("Loading Queries")

            Dim oQueryList = _Model.GetQueryList()

            _Logger.Debug("Running [{0}] queries.", oQueryList.Count)

            ' run
            For Each oQuery As Query In oQueryList
                _Logger.Debug("Running Query [{0}].", oQuery.Name)
                Dim oQueryResult = RunQuery(oQuery)
                _Logger.Info("Query [{0}] finished with Result [{1}]", oQuery.Name, oQueryResult)
            Next

            ' logout
            _Logger.Debug("Logging out")
            Dim oLogoutResponse = _GraphQL.Logout()

            _Logger.Info("Finished GraphQL Job")

        Catch ex As Exception
            _Logger.Warn("Finished GraphQL Job with errors")
            _Logger.Error(ex)
            Throw ex
        End Try
    End Sub

    Private Function RunQuery(pQuery As Query)
        Try
            _Logger.Info("Executing Query [{0}]", pQuery.Name)

            Dim oConnectionId As Integer = pQuery.ConnectionId
            Dim oConnectionString = _MSSQL.Get_ConnectionStringforID(oConnectionId)

            If oConnectionString = String.Empty Then
                _Logger.Warn("Could not get Connection String for ConnectionId [{0}]", oConnectionId)
            End If

            Dim oDatabase As New MSSQLServer(_LogConfig, oConnectionString)

            'TODO: ONly set status when clear before fill is false
            'TODO: ADDED_WHO which contains the query id which inserted the rows

            ' Clear Table before inserting
            If pQuery.ClearBeforeFill = True Then
                If DeleteHistoryTable(pQuery) = False Then
                    Throw New ApplicationException($"Error while dropping history table before fill for Query [{pQuery.Name}]")
                End If

                If CreateHistoryTable(pQuery) = False Then
                    Throw New ApplicationException($"Error while creating history table before fill for Query [{pQuery.Name}]")
                End If

                If TruncateTable(pQuery) = False Then
                    Throw New ApplicationException($"Error while truncating table before fill for Query [{pQuery.Name}]")
                End If
            End If

            ' Reset all records to status = 0
            If pQuery.ClearBeforeFill = False Then
                _Logger.Info("Resetting data for Query [{0}]", pQuery.Name)
                If UpdateWithStatus(pQuery, 0) = False Then
                    Throw New ApplicationException($"Error while resetting status of current Records for Query [{pQuery.Name}]")
                End If
            End If

            ' get the data from GraphQL
            _Logger.Info("Getting data from GraphQL..", pQuery.Name)
            Dim oDataResponse = _GraphQL.GetData(pQuery.QueryString, pQuery.OperationName)
            Dim oJsonResult As String

            ' write data to string
            Using oStream = oDataResponse.GetResponseStream()
                Using oReader As New StreamReader(oStream)
                    oJsonResult = oReader.ReadToEnd()
                End Using
            End Using

            _Logger.Debug("Writing JSON data to database..")

            ' Handle the response from GraphQL and insert Data
            'Dim oWriteDataResult As Query = WriteNewQueryData(oResult, pQuery, oDatabase)
            Dim oWriteDataResult As Query = _Writer.WriteNewQueryData(oJsonResult, pQuery, JOB_NAME)

            If IsNothing(oWriteDataResult) Then
                Throw New ApplicationException($"Error while handling Result of Query [{pQuery.Name}]")
            End If

            _Logger.Info("New Data successfully inserted for Query [{0}]", pQuery.Name)

            ' Finally delete all old records
            If pQuery.ClearBeforeFill = False Then
                _Logger.Info("Deleting old records for Query [{0}].", pQuery.Name)
                If DeleteWithStatus(pQuery, 0) = False Then
                    Throw New ApplicationException($"Error while deleting current Records for Query [{pQuery.Name}]")
                End If
            End If

            Return True

        Catch ex As Exception
            _Logger.Warn("Error while getting Data for Name/OperationName [{0}]/[{1}]", pQuery.Name, pQuery.OperationName)
            _Logger.Error(ex)

            ' If a crash happens, delete all records which were inserted in this run,
            ' thus going back to the previous state
            _Logger.Info("Failure, deleting new records..", pQuery.Name)

            If pQuery.ClearBeforeFill = False Then
                If DeleteWithStatus(pQuery, 1) = False Then
                    Throw New ApplicationException($"Error while deleting new Records for Query [{pQuery.Name}]")
                End If
            End If

            Return False

        Finally
            _Logger.Debug("Finished running Query [{0}].", pQuery.Name)
        End Try
    End Function

    Private Function DeleteHistoryTable(pQuery As Query) As Boolean
        Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY"
        Dim oDeleteHistorySQL = $"
            IF (EXISTS (SELECT *
            FROM INFORMATION_SCHEMA.TABLES
            WHERE TABLE_SCHEMA = 'dbo'
            AND TABLE_NAME = '{oHistoryTableName}'))
            BEGIN
            DROP TABLE {oHistoryTableName};
            END"

        Return _MSSQL.ExecuteNonQuery(oDeleteHistorySQL)
    End Function

    Private Function TruncateTable(pQuery As Query)
        Dim oDeleteTableSQL = $"TRUNCATE TABLE {pQuery.DestinationTable}"

        Return _MSSQL.ExecuteNonQuery(oDeleteTableSQL)
    End Function

    Private Function CreateHistoryTable(pQuery As Query) As Boolean
        Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY"
        Dim oFillHistorySQL = $"SELECT * INTO {oHistoryTableName} FROM {pQuery.DestinationTable}"

        Return _MSSQL.ExecuteNonQuery(oFillHistorySQL)
    End Function

    Private Function DeleteWithStatus(pQuery As Query, pStatus As Integer) As Boolean
        Dim oDeleteSQL = $"DELETE FROM {pQuery.DestinationTable} WHERE STATUS = {pStatus} AND ADDED_QUERY_ID = '{pQuery.Id}'"
        Return _MSSQL.ExecuteNonQuery(oDeleteSQL)
    End Function

    Private Function UpdateWithStatus(pQuery As Query, pStatus As Integer)
        Dim oResetSQL = $"UPDATE {pQuery.DestinationTable} SET STATUS = {pStatus} WHERE ADDED_QUERY_ID = '{pQuery.Id}'"
        Return _MSSQL.ExecuteNonQuery(oResetSQL)
    End Function

    Public Function ShouldStart(Arguments As GraphQLArgs) As Boolean Implements IJob(Of GraphQLArgs).ShouldStart
        Return Arguments.Enabled
    End Function
End Class