Imports System.Collections.Generic
Imports System.Data
Imports System.IO
Imports System.Linq
Imports System.Security.Cryptography
Imports DigitalData.Modules.Base
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Interfaces
Imports DigitalData.Modules.Interfaces.Exceptions
Imports DigitalData.Modules.Jobs.Exceptions
Imports DigitalData.Modules.Logging
Imports System.Data.SqlClient

Public Class ImportZUGFeRDFiles
    Implements IJob

    Public Const ZUGFERD_IN = "ZUGFeRD in"
    Public Const ZUGFERD_ERROR = "ZUGFeRD Error"
    Public Const ZUGFERD_SUCCESS = "ZUGFeRD Success"
    Public Const ZUGFERD_EML = "ZUGFeRD Eml"
    Public Const ZUGFERD_REJECTED_EML = "ZUGFeRD Eml Rejected"
    Public Const ZUGFERD_ATTACHMENTS = "ZUGFeRD Attachments"
    Public Const ZUGFERD_NO_ZUGFERD = "Non-ZUGFeRD Files"

    Public HISTORY_ID As Integer

    Private Const DIRECTORY_DONT_MOVE = "DIRECTORY_DONT_MOVE"

    ' List of allowed extensions for PDF/A Attachments
    ' This list should not contain xml so the zugferd xml file will be filtered out
    Private ReadOnly AllowedExtensions As New List(Of String) From {"docx", "doc", "pdf", "xls", "xlsx", "ppt", "pptx", "txt"}

    Private ReadOnly _logger As Logger
    Private ReadOnly _logConfig As LogConfig
    Private ReadOnly _filesystem As Filesystem.File
    Private ReadOnly _mssql As MSSQLServer
    Private ReadOnly _email As ZUGFeRD.EmailFunctions
    Private ReadOnly _file As ZUGFeRD.FileFunctions
    Private ReadOnly _history As ZUGFeRD.HistoryFunctions
    Private ReadOnly _embeds As PDFEmbeds

    Private ReadOnly _gdpictureLicenseKey As String

    Private _zugferd As ZUGFeRDInterface
    Private _EmailOutAccountId As Integer

    Private Class ProcessFileResult
        Public ZugferdFileCount As Integer
        Public MD5Hash As String = Nothing

    End Class

    Private Class DatabaseConnections
        Public Property SQLServerConnection As SqlConnection
        Public Property SQLServerTransaction As SqlTransaction
    End Class

    Public Sub New(LogConfig As LogConfig, Optional MSSQL As MSSQLServer = Nothing)
        _logConfig = LogConfig
        _logger = LogConfig.GetLogger()
        _filesystem = New Filesystem.File(_logConfig)
        _mssql = MSSQL
        _email = New ZUGFeRD.EmailFunctions(LogConfig, _mssql)
        _file = New ZUGFeRD.FileFunctions(LogConfig, _mssql)
        _history = New ZUGFeRD.HistoryFunctions(LogConfig, _mssql)
        _embeds = New PDFEmbeds(LogConfig)

        _logger.Debug("Registering GDPicture License")
        If _mssql IsNot Nothing Then
            Dim oSQL = "SELECT LICENSE FROM TBDD_3RD_PARTY_MODULES WHERE NAME = 'GDPICTURE'"
            _gdpictureLicenseKey = _mssql.GetScalarValue(oSQL)
        Else
            _logger.Warn("GDPicture License could not be registered! MSSQL is not enabled!")
            Throw New ArgumentNullException("MSSQL")
        End If
    End Sub

    Public Sub Start(Arguments As Object) Implements IJob.Start
        Dim oArgs As WorkerArgs = Arguments
        Dim oPropertyExtractor = New PropertyValues(_logConfig)
        Dim oAttachmentExtractor = New PDFEmbeds(_logConfig)

        _EmailOutAccountId = oArgs.EmailOutProfileId

        Dim oOptions As New ZUGFeRDInterface.ZugferdOptions() With {
            .AllowFacturX_Filename = oArgs.AllowFacturX,
            .AllowXRechnung_Filename = oArgs.AllowXRechnung
        }
        _zugferd = New ZUGFeRDInterface(_logConfig, _gdpictureLicenseKey, oOptions)

        _logger.Debug("Starting Job {0}", [GetType].Name)

        Try
            'For Each oPath As String In oArgs.WatchDirectory
            Dim oPath As String = oArgs.WatchDirectory
            Dim oDirInfo As New DirectoryInfo(oPath)

            _logger.Debug($"Start processing directory {oDirInfo.FullName}")

            If oDirInfo.Exists = False Then
                _logger.Warn("Watch directory exists. Exiting.")
                Exit Sub
            End If

            '  Filter out *.lock files
            Dim oFiles As List(Of FileInfo) = oDirInfo.
                    GetFiles().
                    Where(Function(f) Not f.Name.EndsWith(".lock")).
                    ToList()

            If oFiles.Count = 0 Then
                _logger.Debug("No files to process. Exiting.")
                Exit Sub
            End If

            _logger.Info("Found {0} files", oFiles.Count)

            ' Group files by messageId
            Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = _zugferd.FileGroup.GroupFiles(oFiles)

            _logger.Info("Found {0} file groups", oGrouped.Count)

            ' Process each file group together
            For Each oFileGroup In oGrouped
                ' Start a new transaction for each file group.
                ' This way we can rollback database changes for the whole filegroup in case something goes wrong.

                Dim oSQLConnection As SqlConnection = _mssql.GetConnection()
                Dim oSQLTransaction As SqlTransaction = oSQLConnection?.BeginTransaction()

                Dim oConnections As New DatabaseConnections() With {
                    .SQLServerConnection = oSQLConnection,
                    .SQLServerTransaction = oSQLTransaction
                }

                ' Count the amount of ZUGFeRD files
                Dim oZUGFeRDCount As Integer = 0

                ' Set the default Move Directory
                Dim oMoveDirectory As String = oArgs.ErrorDirectory

                ' Flag to save if the whole process was a success.
                ' Will be set only at the end of the function if no error occurred.
                Dim oIsSuccess As Boolean = False

                ' Flag to save if the occurred error (if any) was expected
                ' Used to determine if transactions should be committed or not
                Dim oExpectedError As Boolean = True

                ' Create file lists
                Dim oFileGroupFiles As List(Of FileInfo) = oFileGroup.Value
                Dim oEmailAttachmentFiles As New List(Of FileInfo)
                Dim oEmbeddedAttachmentFiles As New List(Of PDFEmbeds.EmbeddedFile)

                Dim oMessageId As String = oFileGroup.Key
                Dim oMissingProperties As New List(Of String)
                Dim oMD5CheckSum As String = String.Empty

                _logger.Info("Start processing file group {0}", oMessageId)

                ' TODO: Use this refactored function 
                ' ProcessFileGroup(oMessageId, oFileGroupFiles, oConnections, oArgs)

                Try
                    For Each oFile In oFileGroupFiles
                        ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the 
                        '   different versions of ZUGFeRD and the type is unknown at compile-time.
                        ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
                        '   of the extracted xml file.
                        ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
                        '   the new meta data, ie. the type of schema (zugferd version)
                        Dim oDocument As ZUGFeRDInterface.ZugferdResult

                        ' Start a global group counter for each file
                        Dim oGlobalGroupCounter = 0
                        ' Clear missing properties for the new file
                        oMissingProperties = New List(Of String)

                        ' Only pdf files are allowed from here on
                        If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
                            _logger.Debug("Skipping non-pdf file {0}", oFile.Name)
                            oEmailAttachmentFiles.Add(oFile)

                            ' Checking filesize for attachment files
                            If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then
                                _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
                                Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
                            End If

                            Continue For
                        End If

                        _logger.Info("Start processing file {0}", oFile.Name)

                        ' Checking filesize for pdf files
                        If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then
                            _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
                            Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
                        End If

                        Try
                            oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName)

                        Catch ex As ValidationException
                            Throw ex

                        Catch ex As ZUGFeRDExecption
                            Select Case ex.ErrorType
                                Case ZUGFeRDInterface.ErrorType.NoZugferd
                                    _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
                                    oEmailAttachmentFiles.Add(oFile)
                                    Continue For

                                Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
                                    _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
                                    Throw New UnsupportedFerdException(ex.XmlFile)

                                Case ZUGFeRDInterface.ErrorType.NoValidZugferd
                                    _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name)
                                    Throw New InvalidFerdException()

                                Case Else
                                    _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name)
                                    Throw ex
                            End Select
                        End Try

                        ' Check if there are more than one ZUGFeRD files
                        If oZUGFeRDCount = 1 Then
                            Throw New TooMuchFerdsException()
                        End If

                        ' Since extraction went well, increase the amount of ZUGFeRD files
                        oZUGFeRDCount += 1

                        ' Extract all attachments with the extensions specified in `AllowedExtensions`.
                        ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
                        ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`.
                        Dim oAttachments = oAttachmentExtractor.Extract(oFile.FullName, AllowedExtensions)
                        If oAttachments Is Nothing Then
                            _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
                        Else
                            oEmbeddedAttachmentFiles.AddRange(oAttachments)
                        End If

                        ' Check the Checksum and rejection status
                        oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, oArgs.IgnoreRejectionStatus)

                        ' Check the document against the configured property map and return:
                        ' - a List of valid properties 
                        ' - a List of missing properties

                        Dim oPropertyMap = _zugferd.FilterPropertyMap(oArgs.PropertyMap, oDocument.Specification)
                        Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId)

                        _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)

                        If oCheckResult.MissingProperties.Count > 0 Then
                            _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
                            oMissingProperties = oCheckResult.MissingProperties
                            Throw New MissingValueException(oFile)
                        Else
                            _logger.Debug("No missing properties found. Continuing.")

                        End If

                        DeleteExistingPropertyValues(oMessageId, oConnections)

                        Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault()
                        If oFirstProperty IsNot Nothing Then
                            InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With {
                                .MessageId = oMessageId,
                                .Description = "ZUGFeRDSpezifikation",
                                .GroupCounter = 0,
                                .IsRequired = False,
                                .Value = oDocument.Specification,
                                .TableName = oFirstProperty.TableName,
                                .TableColumn = "ZUGFERD_SPECIFICATION"
                            })
                        End If

                        For Each oProperty In oCheckResult.ValidProperties
                            InsertPropertyValue(oMessageId, oConnections, oProperty)
                        Next
                    Next

                    'Check if there are no ZUGFeRD files
                    If oZUGFeRDCount = 0 Then
                        ' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated
                        ' This is the default/initial behaviour.
                        If oArgs.NonZugferdDirectory Is Nothing OrElse oArgs.NonZugferdDirectory = String.Empty Then
                            Throw New NoFerdsException()
                        End If

                        ' Also, if the directory is set but does not exist, still a rejection will be generated.
                        If Not IO.Directory.Exists(oArgs.NonZugferdDirectory) Then
                            Throw New NoFerdsException()
                        End If

                        ' Only if the directory is set and does exist, it will be used and any file groups which
                        ' do NOT CONTAIN ANY ZUGFERD DOCUMENTS, are moved to that directory.
                        Throw New NoFerdsAlternateException()

                    End If

                    'If no errors occurred...
                    'Log the History
                    If oMD5CheckSum <> String.Empty Then
                        _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS")
                    Else
                        _history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)")
                    End If

                    oIsSuccess = True
                    oMoveDirectory = oArgs.SuccessDirectory

                Catch ex As ValidationException
                    _logger.Error(ex)

                    Dim oErrors = ex.ValidationErrors
                    Dim oMessage = "REJECTED - ZUGFeRD yes but formal validation failed!"
                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, oMessage)

                    Dim oErrorList As String = ""
                    For Each oError In oErrors
                        oErrorList += $"<li>Element '{oError.ElementName}' mit Wert '{oError.ElementValue}': {oError.ErrorMessage}</li>"
                    Next

                    Dim oBody = String.Format(EmailStrings.EMAIL_VALIDATION_ERROR, oErrorList)
                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "ValidationException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "ValidationException", "Die Rechnungsvalidierung ist fehlgeschlagen!", "", oSQLTransaction)

                Catch ex As MD5HashException
                    _logger.Error(ex)

                    ' When MD5HashException is thrown, we don't have a MD5Hash yet.
                    ' That 's why we set it to String.Empty here.
                    Dim oMessage = "REJECTED - Already processed (MD5Hash)"
                    _history.Update_HistoryEntry(oMessageId, String.Empty, oMessage)

                    Dim oBody = String.Format(EmailStrings.EMAIL_MD5_ERROR, ex.FileName)
                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MD5HashException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "", oSQLTransaction)

                Catch ex As UnsupportedFerdException
                    _logger.Error(ex)

                    ' When UnsupportedFerdException is thrown, we don't have a MD5Hash yet.
                    ' That 's why we set it to String.Empty here.
                    _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format")

                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                    Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, oEmailData.Subject, ex.XmlFile)

                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnsupportedFerdException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "UnsupportedFerdException", "Nicht unterstütztes Datenformat", "", oSQLTransaction)

                Catch ex As InvalidFerdException
                    _logger.Error(ex)

                    ' When InvalidFerdException is thrown, we don't have a MD5Hash yet.
                    ' That 's why we set it to String.Empty here.
                    _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format")

                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                    Dim oBody = String.Format(EmailStrings.EMAIL_INVALID_DOCUMENT, oEmailData.Subject)

                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "InvalidFerdException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "InvalidFerdException", "Inkorrekte Formate", "", oSQLTransaction)

                Catch ex As TooMuchFerdsException
                    _logger.Error(ex)

                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email")

                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                    Dim oBody = String.Format(EmailStrings.EMAIL_TOO_MUCH_FERDS, oEmailData.Subject)

                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "TooMuchFerdsException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "", oSQLTransaction)

                Catch ex As NoFerdsException
                    _logger.Error(ex)

                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email")

                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                    Dim oBody = String.Format(EmailStrings.EMAIL_NO_FERDS, oEmailData.Subject)

                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "NoFerdsException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "", oSQLTransaction)

                Catch ex As MissingValueException
                    _logger.Error(ex)

                    Dim oMessage As String = ""
                    For Each prop In oMissingProperties
                        oMessage &= $"- {prop}"
                    Next

                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]")

                    Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties)
                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oSQLTransaction)

                Catch ex As FileSizeLimitReachedException
                    _logger.Error(ex)

                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - File size limit reached")

                    Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)

                    Dim oKey = FileSizeLimitReachedException.KEY_FILENAME
                    Dim oFileExceedingThreshold As String = IIf(ex.Data.Contains(oKey), ex.Data.Item(oKey), "")
                    Dim oFileWithoutMessageId = oFileExceedingThreshold.
                        Replace(oMessageId, "").
                        Replace("~", "")

                    Dim oBody = String.Format(EmailStrings.EMAIL_FILE_SIZE_REACHED, oArgs.MaxAttachmentSizeInMegaBytes, oFileWithoutMessageId)

                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileSizeLimitReachedException", _EmailOutAccountId, oArgs.NamePortal)
                    AddRejectedState(oMessageId, "FileSizeLimitReachedException", "Erlaubte Dateigröße überschritten", "", oSQLTransaction)


                Catch ex As NoFerdsAlternateException
                    ' TODO: Maybe dont even log this 'error', since it's not really an error and it might happen *A LOT*
                    _logger.Error(ex)
                    oMoveDirectory = oArgs.NonZugferdDirectory

                Catch ex As OutOfMemoryException
                    _logger.Warn("OutOfMemory Error occurred: {0}", ex.Message)
                    _logger.Error(ex)

                    ' Send Email to Digital Data
                    Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
                    Dim oEmailData As New EmailData With {
                        .From = oArgs.ExceptionEmailAddress,
                        .Subject = $"OutOfMemoryException im ZUGFeRD-Parser @ {oMessageId}"
                    }
                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "OutOfMemoryException", _EmailOutAccountId, oArgs.NamePortal)

                    ' Rollback Transaction
                    oSQLTransaction.Rollback()

                    oMoveDirectory = DIRECTORY_DONT_MOVE

                    oExpectedError = False

                Catch ex As Exception
                    _logger.Warn("Unknown Error occurred: {0}", ex.Message)
                    _logger.Error(ex)

                    ' Send Email to Digital Data
                    Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
                    Dim oEmailData As New EmailData With {
                        .From = oArgs.ExceptionEmailAddress,
                        .Subject = $"UnhandledException im ZUGFeRD-Parser @ {oMessageId}"
                    }
                    _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnhandledException", _EmailOutAccountId, oArgs.NamePortal)

                    ' Rollback Transaction
                    oSQLTransaction.Rollback()

                    oMoveDirectory = DIRECTORY_DONT_MOVE

                    oExpectedError = False

                Finally
                    Try
                        ' If an application error occurred, dont move files so they will be processed again later
                        If oMoveDirectory = DIRECTORY_DONT_MOVE Then
                            _logger.Info("Application Error occurred. Files for message Id {0} will not be moved.", oMessageId)
                        Else
                            ' Move all files of the current group
                            _file.MoveFiles(oArgs, oMessageId, oFileGroupFiles, oEmailAttachmentFiles, oEmbeddedAttachmentFiles, oMoveDirectory, oIsSuccess)
                        End If
                        _logger.Info("Finished processing file group {0}", oMessageId)
                    Catch ex As Exception
                        ' Send Email to Digital Data
                        Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
                        Dim oEmailData As New EmailData With {
                            .From = oArgs.ExceptionEmailAddress,
                            .Subject = $"FileMoveException im ZUGFeRD-Parser @ {oMessageId}"
                        }
                        _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileMoveException", _EmailOutAccountId, oArgs.NamePortal)

                        _logger.Warn("Could not move files!")
                        _logger.Error(ex)
                        Throw ex
                    End Try

                    Try
                        ' If everything went OK or an expected error occurred, 
                        ' finally commit all changes To the Database
                        ' ==================================================================
                        If oIsSuccess Or oExpectedError Then
                            ' Commit Transaction
                            oSQLTransaction.Commit()
                        End If
                    Catch ex As Exception
                        _logger.Error(ex)
                        _logger.Warn("Database Transactions were not committed successfully.")
                    End Try

                    Try
                        oSQLConnection.Close()
                    Catch ex As Exception
                        _logger.Error(ex)
                        _logger.Warn("Database Connections were not closed successfully.")
                    End Try
                End Try

            Next

            _logger.Debug("Finishing Job {0}", Me.GetType.Name)
        Catch ex As Exception
            _logger.Error(ex)
            _logger.Info("Job Failed! See error log for details")
        End Try
    End Sub



#Region "=== REFACTOR"
    Private Function ProcessFileGroup(oMessageId As String, pFiles As List(Of FileInfo), oConnections As DatabaseConnections, pArgs As WorkerArgs)
        Dim oMissingProperties = New List(Of String)
        Dim oEmailAttachmentFiles = New List(Of FileInfo)
        Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile)
        Dim oZUGFeRDCount = 0
        Dim oMD5CheckSum As String = Nothing

        ' Set the default Move Directory
        Dim oMoveDirectory As String = pArgs.ErrorDirectory

        ' Flag to save if the whole process was a success.
        ' Will be set only at the end of the function if no error occurred.
        Dim oIsSuccess As Boolean = False

        ' Flag to save if the occurred error (if any) was expected
        ' Used to determine if transactions should be committed or not
        Dim oExpectedError As Boolean = True

        Try
            For Each oFile In pFiles
                ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the 
                '   different versions of ZUGFeRD and the type is unknown at compile-time.
                ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
                '   of the extracted xml file.
                ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
                '   the new meta data, ie. the type of schema (zugferd version)
                Dim oDocument As ZUGFeRDInterface.ZugferdResult

                ' Start a global group counter for each file
                Dim oGlobalGroupCounter = 0
                ' Clear missing properties for the new file
                oMissingProperties = New List(Of String)

                ' Only pdf files are allowed from here on
                If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
                    _logger.Debug("Skipping non-pdf file {0}", oFile.Name)
                    oEmailAttachmentFiles.Add(oFile)

                    ' Checking filesize for attachment files
                    If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
                        _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
                        Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
                    End If

                    Continue For
                End If

                _logger.Info("Start processing file {0}", oFile.Name)

                ' Checking filesize for pdf files
                If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
                    _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
                    Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
                End If

                Try
                    oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName)

                Catch ex As ValidationException
                    Throw ex

                Catch ex As ZUGFeRDExecption
                    Select Case ex.ErrorType
                        Case ZUGFeRDInterface.ErrorType.NoZugferd
                            _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
                            oEmailAttachmentFiles.Add(oFile)
                            Continue For

                        Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
                            _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
                            Throw New UnsupportedFerdException(ex.XmlFile)

                        Case ZUGFeRDInterface.ErrorType.NoValidZugferd
                            _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name)
                            Throw New InvalidFerdException()

                        Case Else
                            _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name)
                            Throw ex
                    End Select
                End Try

                ' Check if there are more than one ZUGFeRD files
                If oZUGFeRDCount = 1 Then
                    Throw New TooMuchFerdsException()
                End If

                ' Since extraction went well, increase the amount of ZUGFeRD files
                oZUGFeRDCount += 1

                ' Extract all attachments with the extensions specified in `AllowedExtensions`.
                ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
                ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`.
                Dim oAttachments = _embeds.Extract(oFile.FullName, AllowedExtensions)
                If oAttachments Is Nothing Then
                    _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
                Else
                    oEmbeddedAttachmentFiles.AddRange(oAttachments)
                End If

                ' Check the Checksum and rejection status
                oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, pArgs.IgnoreRejectionStatus)

                ' Check the document against the configured property map and return:
                ' - a List of valid properties 
                ' - a List of missing properties

                Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, oDocument.Specification)
                Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId)

                _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)

                If oCheckResult.MissingProperties.Count > 0 Then
                    _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
                    oMissingProperties = oCheckResult.MissingProperties
                    Throw New MissingValueException(oFile)
                Else
                    _logger.Debug("No missing properties found. Continuing.")

                End If

                DeleteExistingPropertyValues(oMessageId, oConnections)

                Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault()
                If oFirstProperty IsNot Nothing Then
                    InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With {
                        .MessageId = oMessageId,
                        .Description = "ZUGFeRDSpezifikation",
                        .GroupCounter = 0,
                        .IsRequired = False,
                        .Value = oDocument.Specification,
                        .TableName = oFirstProperty.TableName,
                        .TableColumn = "ZUGFERD_SPECIFICATION"
                    })
                End If

                For Each oProperty In oCheckResult.ValidProperties
                    InsertPropertyValue(oMessageId, oConnections, oProperty)
                Next
            Next

            'Check if there are no ZUGFeRD files
            If oZUGFeRDCount = 0 Then

                ' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated
                ' This is the default/initial behaviour.
                If pArgs.NonZugferdDirectory Is Nothing OrElse pArgs.NonZugferdDirectory = String.Empty Then
                    Throw New NoFerdsException()
                End If

                ' Also, if the directory is set but does not exist, still a rejection will be generated.
                If Not IO.Directory.Exists(pArgs.NonZugferdDirectory) Then
                    Throw New NoFerdsException()
                End If

                ' Only if the directory is set and does exist, it will be used and any file groups which
                ' do NOT CONTAIN ANY ZUGFERD DOCUMENTS, are moved to that directory.
                Throw New NoFerdsAlternateException()

            End If

            'If no errors occurred...
            'Log the History
            If oMD5CheckSum <> String.Empty Then
                _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS")
            Else
                _history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)")
            End If

            oIsSuccess = True
            oMoveDirectory = pArgs.SuccessDirectory

        Catch ex As ValidationException
            _logger.Error(ex)

            Dim oErrors = ex.ValidationErrors
            Dim oMessage = "REJECTED - ZUGFeRD yes but formal validation failed!"
            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, oMessage)

            Dim oErrorList As String = ""
            For Each oError In oErrors
                oErrorList += $"<li>Element '{oError.ElementName}' mit Wert '{oError.ElementValue}': {oError.ErrorMessage}</li>"
            Next

            Dim oBody = String.Format(EmailStrings.EMAIL_VALIDATION_ERROR, oErrorList)
            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "ValidationException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "ValidationException", "Die Rechnungsvalidierung ist fehlgeschlagen!", "", oConnections.SQLServerTransaction)

        Catch ex As MD5HashException
            _logger.Error(ex)

            ' When MD5HashException is thrown, we don't have a MD5Hash yet.
            ' That 's why we set it to String.Empty here.
            Dim oMessage = "REJECTED - Already processed (MD5Hash)"
            _history.Update_HistoryEntry(oMessageId, String.Empty, oMessage)

            Dim oBody = EmailStrings.EMAIL_MD5_ERROR
            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MD5HashException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "", oConnections.SQLServerTransaction)

        Catch ex As UnsupportedFerdException
            _logger.Error(ex)

            ' When UnsupportedFerdException is thrown, we don't have a MD5Hash yet.
            ' That 's why we set it to String.Empty here.
            _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format")

            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
            Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, oEmailData.Subject, ex.XmlFile)

            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnsupportedFerdException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "UnsupportedFerdException", "Nicht unterstütztes Datenformat", "", oConnections.SQLServerTransaction)

        Catch ex As InvalidFerdException
            _logger.Error(ex)

            ' When InvalidFerdException is thrown, we don't have a MD5Hash yet.
            ' That 's why we set it to String.Empty here.
            _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format")

            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
            Dim oBody = String.Format(EmailStrings.EMAIL_INVALID_DOCUMENT, oEmailData.Subject)

            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "InvalidFerdException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "InvalidFerdException", "Inkorrekte Formate", "", oConnections.SQLServerTransaction)

        Catch ex As TooMuchFerdsException
            _logger.Error(ex)

            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email")

            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
            Dim oBody = String.Format(EmailStrings.EMAIL_TOO_MUCH_FERDS, oEmailData.Subject)

            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "TooMuchFerdsException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "", oConnections.SQLServerTransaction)

        Catch ex As NoFerdsException
            _logger.Error(ex)

            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email")

            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
            Dim oBody = String.Format(EmailStrings.EMAIL_NO_FERDS, oEmailData.Subject)

            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "NoFerdsException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "", oConnections.SQLServerTransaction)

        Catch ex As MissingValueException
            _logger.Error(ex)

            Dim oMessage As String = ""
            For Each prop In oMissingProperties
                oMessage &= $"- {prop}"
            Next

            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]")

            Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties)
            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oConnections.SQLServerTransaction)

        Catch ex As FileSizeLimitReachedException
            _logger.Error(ex)

            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - File size limit reached")

            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)

            Dim oKey = FileSizeLimitReachedException.KEY_FILENAME
            Dim oFileExceedingThreshold As String = IIf(ex.Data.Contains(oKey), ex.Data.Item(oKey), "")
            Dim oFileWithoutMessageId = oFileExceedingThreshold.
                Replace(oMessageId, "").
                Replace("~", "")

            Dim oBody = String.Format(EmailStrings.EMAIL_FILE_SIZE_REACHED, pArgs.MaxAttachmentSizeInMegaBytes, oFileWithoutMessageId)

            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileSizeLimitReachedException", _EmailOutAccountId, pArgs.NamePortal)
            AddRejectedState(oMessageId, "FileSizeLimitReachedException", "Erlaubte Dateigröße überschritten", "", oConnections.SQLServerTransaction)


        Catch ex As NoFerdsAlternateException
            ' TODO: Maybe dont even log this 'error', since it's not really an error and it might happen *A LOT*
            _logger.Error(ex)
            oMoveDirectory = pArgs.NonZugferdDirectory

        Catch ex As OutOfMemoryException
            _logger.Warn("OutOfMemory Error occurred: {0}", ex.Message)
            _logger.Error(ex)

            ' Send Email to Digital Data
            Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
            Dim oEmailData As New EmailData With {
                .From = pArgs.ExceptionEmailAddress,
                .Subject = $"OutOfMemoryException im ZUGFeRD-Parser @ {oMessageId}"
            }
            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "OutOfMemoryException", _EmailOutAccountId, pArgs.NamePortal)

            ' Rollback Transaction
            oConnections.SQLServerTransaction.Rollback()

            oMoveDirectory = DIRECTORY_DONT_MOVE

            oExpectedError = False

        Catch ex As Exception
            _logger.Warn("Unknown Error occurred: {0}", ex.Message)
            _logger.Error(ex)

            ' Send Email to Digital Data
            Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
            Dim oEmailData As New EmailData With {
                .From = pArgs.ExceptionEmailAddress,
                .Subject = $"UnhandledException im ZUGFeRD-Parser @ {oMessageId}"
            }
            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnhandledException", _EmailOutAccountId, pArgs.NamePortal)

            ' Rollback Transaction
            oConnections.SQLServerTransaction.Rollback()

            oMoveDirectory = DIRECTORY_DONT_MOVE

            oExpectedError = False

        Finally
            Try
                ' If an application error occurred, dont move files so they will be processed again later
                If oMoveDirectory = DIRECTORY_DONT_MOVE Then
                    _logger.Info("Application Error occurred. Files for message Id {0} will not be moved.", oMessageId)
                Else
                    ' Move all files of the current group
                    _file.MoveFiles(pArgs, oMessageId, pFiles, oEmailAttachmentFiles, oEmbeddedAttachmentFiles, oMoveDirectory, oIsSuccess)
                End If
                _logger.Info("Finished processing file group {0}", oMessageId)
            Catch ex As Exception
                ' Send Email to Digital Data
                Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
                Dim oEmailData As New EmailData With {
                    .From = pArgs.ExceptionEmailAddress,
                    .Subject = $"FileMoveException im ZUGFeRD-Parser @ {oMessageId}"
                }
                _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileMoveException", _EmailOutAccountId, pArgs.NamePortal)

                _logger.Warn("Could not move files!")
                _logger.Error(ex)
                Throw ex
            End Try

            Try
                ' If everything went OK or an expected error occurred, 
                ' finally commit all changes To the Database
                ' ==================================================================
                If oIsSuccess Or oExpectedError Then
                    ' Commit Transaction
                    oConnections.SQLServerTransaction.Commit()
                End If

                _logger.Info("FileGroup for MessageId [{0}] successfully processed!", oMessageId)
            Catch ex As Exception
                _logger.Error(ex)
                _logger.Warn("Database Transactions were not committed successfully.")
            End Try

            Try
                oConnections.SQLServerConnection.Close()
            Catch ex As Exception
                _logger.Error(ex)
                _logger.Warn("Database Connections were not closed successfully.")
            End Try
        End Try
    End Function

    Private Function ProcessFile(pMessageId As String, oFile As FileInfo, oConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult
        ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the 
        '   different versions of ZUGFeRD and the type is unknown at compile-time.
        ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
        '   of the extracted xml file.
        ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
        '   the new meta data, ie. the type of schema (zugferd version)
        Dim oDocument As ZUGFeRDInterface.ZugferdResult

        Dim oMissingProperties = New List(Of String)
        Dim oEmailAttachmentFiles = New List(Of FileInfo)
        Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile)
        Dim oZugferdFiles As Integer = 0

        ' Start a global group counter for each file
        Dim oGlobalGroupCounter = 0
        ' Clear missing properties for the new file
        oMissingProperties = New List(Of String)

        ' Only pdf files are allowed from here on
        If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
            _logger.Debug("Skipping non-pdf file {0}", oFile.Name)
            oEmailAttachmentFiles.Add(oFile)

            ' Checking filesize for attachment files
            If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
                _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
                Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
            End If

            'TODO: how to handle?
            'Continue For
        End If

        _logger.Info("Start processing file {0}", oFile.Name)

        ' Checking filesize for pdf files
        If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
            _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
            Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
        End If

        Try
            oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName)

        Catch ex As ValidationException
            Throw ex

        Catch ex As ZUGFeRDExecption
            Select Case ex.ErrorType
                Case ZUGFeRDInterface.ErrorType.NoZugferd
                    _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
                    oEmailAttachmentFiles.Add(oFile)

                    'TODO: How to handle?
                    'Continue For

                Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
                    _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
                    Throw New UnsupportedFerdException(ex.XmlFile)

                Case ZUGFeRDInterface.ErrorType.NoValidZugferd
                    _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name)
                    Throw New InvalidFerdException()

                Case Else
                    _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name)
                    Throw ex
            End Select
        End Try

        ' Check if there are more than one ZUGFeRD files
        If oZugferdFiles = 1 Then
            Throw New TooMuchFerdsException()
        End If

        ' Since extraction went well, increase the amount of ZUGFeRD files
        oZugferdFiles += 1

        ' Extract all attachments with the extensions specified in `AllowedExtensions`.
        ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
        ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`.
        Dim oAttachments = _embeds.Extract(oFile.FullName, AllowedExtensions)
        If oAttachments Is Nothing Then
            _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
        Else
            oEmbeddedAttachmentFiles.AddRange(oAttachments)
        End If

        ' Check the Checksum and rejection status
        Dim oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, pMessageId, pArgs.IgnoreRejectionStatus)

        ' Check the document against the configured property map and return:
        ' - a List of valid properties 
        ' - a List of missing properties

        Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, oDocument.Specification)
        Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, pMessageId)

        _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)

        If oCheckResult.MissingProperties.Count > 0 Then
            _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
            oMissingProperties = oCheckResult.MissingProperties
            Throw New MissingValueException(oFile)
        Else
            _logger.Debug("No missing properties found. Continuing.")

        End If

        DeleteExistingPropertyValues(pMessageId, oConnections)

        Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault()
        If oFirstProperty IsNot Nothing Then
            InsertPropertyValue(pMessageId, oConnections, New PropertyValues.ValidProperty() With {
                .MessageId = pMessageId,
                .Description = "ZUGFeRDSpezifikation",
                .GroupCounter = 0,
                .IsRequired = False,
                .Value = oDocument.Specification,
                .TableName = oFirstProperty.TableName,
                .TableColumn = "ZUGFERD_SPECIFICATION"
            })
        End If

        For Each oProperty In oCheckResult.ValidProperties
            InsertPropertyValue(pMessageId, oConnections, oProperty)
        Next

        Return New ProcessFileResult With {
            .ZugferdFileCount = oZugferdFiles
        }
    End Function
#End Region


    Private Sub DeleteExistingPropertyValues(pMessageId As String, pConnections As DatabaseConnections)
        Dim oDelSQL = $"DELETE FROM TBEDMI_ITEM_VALUE where REFERENCE_GUID = '{pMessageId}'"
        Dim oStep As String

        oStep = "TBEDMI_ITEM_VALUE Delete MessageID Items"
        Try
            _mssql.ExecuteNonQueryWithConnectionObject(oDelSQL, pConnections.SQLServerConnection, MSSQLServer.TransactionMode.ExternalTransaction, pConnections.SQLServerTransaction)
        Catch ex As Exception
            _logger.Warn("Step [{0}] with SQL [{1}] was not successful.", oStep, oDelSQL)
        End Try
    End Sub

    Private Sub InsertPropertyValue(pMessageId As String, pConnections As DatabaseConnections, pProperty As PropertyValues.ValidProperty)
        Dim oGroupCounterValue = pProperty.GroupCounter

        ' If GroupCounter is -1, it means this is a default property that can only occur once.
        ' Set the actual inserted value to 0
        If oGroupCounterValue = -1 Then
            oGroupCounterValue = 0
        End If

        Dim oCommand = $"INSERT INTO {pProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER, SPEC_NAME, IS_REQUIRED) VALUES 
                                            ('{pMessageId}', '{pProperty.Description}', '{pProperty.Value.Replace("'", "''")}', {oGroupCounterValue},'{pProperty.TableColumn}','{pProperty.IsRequired}')"
        _logger.Debug("Mapping Property [{0}]  with value [{1}], Will be inserted into table [{2}]", pProperty.TableColumn, pProperty.Value.Replace("'", "''"), pProperty.TableName)
        ' Insert into SQL Server

        Dim oResult = _mssql.ExecuteNonQueryWithConnectionObject(oCommand, pConnections.SQLServerConnection, MSSQLServer.TransactionMode.ExternalTransaction, pConnections.SQLServerTransaction)
        If oResult = False Then
            _logger.Warn($"SQL Command [{oCommand}] was not successful. Check the log.")
        End If
    End Sub

    Private Function CreateMD5(ByVal Filename As String) As String
        Try
            Dim oMD5 As New MD5CryptoServiceProvider
            Dim oHash As Byte()
            Dim oHashString As String
            Dim oResult As String = ""

            Using oFileStream As New FileStream(Filename, FileMode.Open, FileAccess.Read, FileShare.Read, 8192)
                oHash = oMD5.ComputeHash(oFileStream)
                oHashString = BitConverter.ToString(oHash)
            End Using

            oResult = oHashString.Replace("-", "")
            Return oResult
        Catch ex As Exception
            _logger.Error(ex)
            Return ""
        End Try
    End Function



    ''' <summary>
    ''' Generates the MD5 Checksum of a file and checks it against the histroy table TBEDM_ZUGFERD_HISTORY_IN
    ''' </summary>
    ''' <param name="pFilePath">The path of the file to be checked</param>
    ''' <param name="pIgnoreRejectionStatus">Should the check take into account the rejection status of the file?</param>
    ''' <returns>The MD5 Checksum of the file, or an empty string, if the Checksum could not be created</returns>
    ''' <exception cref="MD5HashException">Throws, when the file should be rejected, ie. if it already exists in the table</exception>
    Private Function GenerateAndCheck_MD5Sum(pFile As FileInfo, pMessageId As String, pIgnoreRejectionStatus As Boolean) As String
        Dim oMD5CheckSum = CreateMD5(pFile.FullName)

        ' Exit if MD5 could not be created
        If oMD5CheckSum = String.Empty Then
            _logger.Warn("MD5 Checksum is nothing for file [{0}]!", pFile.FullName)
            Return oMD5CheckSum
        End If

        ' Check if Checksum exists in History Table
        'Dim oCheckCommand = $"SELECT * FROM TBEDM_ZUGFERD_HISTORY_IN WHERE GUID = (SELECT MAX(GUID) FROM TBEDM_ZUGFERD_HISTORY_IN WHERE UPPER(MD5HASH) = UPPER('{oMD5CheckSum}'))"
        'Dim oTable As DataTable = _firebird.GetDatatable(oCheckCommand, Firebird.TransactionMode.NoTransaction)

        ' Check if Checksum exists in History Table
        ' TODO: WHAT THE FUCK IS THIS
        Dim oCheckCommand = $"SELECT * FROM TBEMLP_HISTORY WHERE GUID = (SELECT MAX(GUID) FROM TBEMLP_HISTORY WHERE UPPER(MD5HASH) = UPPER('{oMD5CheckSum}'))"
        Dim oTable As DataTable = _mssql.GetDatatable(oCheckCommand, MSSQLServer.TransactionMode.NoTransaction)

        ' If History entries could not be fetched, just return the MD5 Checksum
        If IsNothing(oTable) Then
            _logger.Warn("Be careful: oExistsDT is nothing for file [{0}]!", pFile.FullName)
            Return oMD5CheckSum
        End If

        ' If Checksum does not exist in History entries, just return the MD5 Checksum
        If oTable.Rows.Count = 0 Then
            _logger.Debug("File [{0}] was not found in History!", pFile.FullName)
            Return oMD5CheckSum
        End If

        ' ====================================================
        ' Checksum exists in History entries, reject!
        ' ====================================================

        Dim oRejected As Boolean
        Dim oHistoryId As Integer

        ' Try to read Rejected Status and History Id
        Try
            Dim oRow As DataRow = oTable.Rows.Item(0)
            oRejected = oRow.ItemEx("CUST_REJECTED", False)
            oHistoryId = oRow.Item("GUID")

        Catch ex As Exception
            _logger.Warn("Error while converting REJECTED: " & ex.Message)
            oRejected = False

        End Try

        _logger.Info("File with MessageId [{0}] has already been processed.", pMessageId)

        ' If the file was already rejected, it is allowed to be processed again, 
        ' even if the Checksum exists in the history entries (default case)
        ' Which means, if it was not rejected before, it will be rejected in any case!
        '
        ' This logic can be overwritten by the IgnoreRejectionStatus parameter.
        ' If it is set to true, the file will be rejected if the file exists in the history entries,
        ' regardless of the rejected parameter.

        If oRejected = True And pIgnoreRejectionStatus = True Then
            _logger.Info("ZuGFeRDFile already has been processed, but Rejection Status will be ignored!")
        ElseIf oRejected = False Then
            _logger.Info("ZuGFeRDFile already has been processed. Will be rejected.")
            Throw New MD5HashException($"There is already an identical invoice! - HistoryID [{oHistoryId}] - Case 1", pFile.Name)
        Else
            _logger.Info("ZuGFeRDFile already has been processed. Will be rejected.")
            Throw New MD5HashException($"There is already an identical invoice! - HistoryID [{oHistoryId}] - Case 2", pFile.Name)
        End If

        Return oMD5CheckSum
    End Function

    Private Function MoveAndRenameEmailToRejected(pArgs As WorkerArgs, pMessageId As String) As EmailData
        _logger.Info("Moving Mail with MessageId [{0}] to Rejected folder", pMessageId)
        _logger.Debug("Fetching Email Data")

        Dim oEmailData = _email.GetEmailDataForMessageId(pMessageId)
        _logger.Debug("Email Data fetched!")

        Dim oSource = _email.GetOriginalEmailPath(pArgs.OriginalEmailDirectory, pMessageId)
        _logger.Debug("Original email path: [{0}]", oSource)

        Dim oDateSubDirectoryName As String = Now.ToString("yyyy-MM-dd")
        Dim oDestination As String

        Dim oRejectedDirectory As String = Path.Combine(pArgs.RejectedEmailDirectory, oDateSubDirectoryName)

        ' Create the destination directory if it does not exist
        _logger.Debug("Creating destination directory [{0}]", oRejectedDirectory)
        If Not Directory.Exists(oRejectedDirectory) Then
            Try
                Directory.CreateDirectory(oRejectedDirectory)
            Catch ex As Exception
                _logger.Error(ex)
            End Try
        End If

        If oSource = String.Empty Then
            _logger.Warn("Original Email for [{0}] could not be found. Exiting.", pMessageId)
            Return New EmailData()
        End If

        ' If oEmailData is Nothing, TBEDM_EMAIL_PROFILER_HISTORY for MessageId was not found. 
        ' This only should happen when testing and db-tables are deleted frequently
        If oEmailData Is Nothing Then
            oDestination = _email.GetEmailPathWithSubjectAsName(oRejectedDirectory, pMessageId)
        Else
            oDestination = _email.GetEmailPathWithSubjectAsName(oRejectedDirectory, StringEx.ConvertTextToSlug(oEmailData.Subject))
        End If

        _logger.Debug("Destination for eml file is {0}", oDestination)

        Dim oFinalFileName = _filesystem.GetVersionedFilename(oDestination)

        _logger.Debug("Versioned filename for eml file is {0}", oFinalFileName)

        If oEmailData Is Nothing Then
            _logger.Warn("Could not get Email Data from firebird-database. File {0} will not be moved!", oSource)
            Return New EmailData()
        End If

        '---------------------------

        Try
            _logger.Info("Moving email from {0} to {1}", oSource, oFinalFileName)
            File.Move(oSource, oFinalFileName)
            oEmailData.Attachment = oFinalFileName
        Catch ex As Exception
            _logger.Warn("File {0} could not be moved! Original Filename will be used!", oSource)
            _logger.Error(ex)
            oEmailData.Attachment = oSource
        End Try

        _logger.Info("Email [{0}] moved to rejected folder!", pMessageId)

        Return oEmailData
    End Function

    Private Sub AddRejectedState(oMessageID As String, oTitle As String, oTitle1 As String, oComment As String, Transaction As SqlTransaction)
        Try
            'PRCUST_ADD_HISTORY_STATE: @MessageID VARCHAR(250), @TITLE1 VARCHAR(250), @TITLE2 VARCHAR(250)
            Dim oSQL = $"EXEC PRCUST_ADD_HISTORY_STATE '{oMessageID}','{oTitle}','{oTitle1}','{oComment.Replace("'", "''")}'"
            _mssql.ExecuteNonQuery(oSQL, Transaction)
        Catch ex As Exception
            _logger.Error(ex)
        End Try
    End Sub
End Class