Jobs: Prepare Service Refactor, fix nullref error with emaildata, remove obsolete loops

Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb

@@ -37,12 +37,19 @@ Public Class ImportZUGFeRDFiles
     Private ReadOnly _email As ZUGFeRD.EmailFunctions
     Private ReadOnly _file As ZUGFeRD.FileFunctions
     Private ReadOnly _history As ZUGFeRD.HistoryFunctions
+    Private ReadOnly _embeds As PDFEmbeds
 
     Private ReadOnly _gdpictureLicenseKey As String
 
     Private _zugferd As ZUGFeRDInterface
     Private _EmailOutAccountId As Integer
 
+    Private Class ProcessFileResult
+        Public ZugferdFileCount As Integer
+        Public MD5Hash As String = Nothing
+
+    End Class
+
     Private Class DatabaseConnections
         Public Property SQLServerConnection As SqlConnection
         Public Property SQLServerTransaction As SqlTransaction
@@ -56,6 +63,7 @@ Public Class ImportZUGFeRDFiles
         _email = New ZUGFeRD.EmailFunctions(LogConfig, _mssql)
         _file = New ZUGFeRD.FileFunctions(LogConfig, _mssql)
         _history = New ZUGFeRD.HistoryFunctions(LogConfig, _mssql)
+        _embeds = New PDFEmbeds(LogConfig)
 
         _logger.Debug("Registering GDPicture License")
         If _mssql IsNot Nothing Then
@@ -67,82 +75,6 @@ Public Class ImportZUGFeRDFiles
         End If
     End Sub
 
-    Private Function MoveAndRenameEmailToRejected(pArgs As WorkerArgs, pMessageId As String) As EmailData
-        _logger.Info("Moving Mail with MessageId [{0}] to Rejected folder", pMessageId)
-        _logger.Debug("Fetching Email Data")
-
-        Dim oEmailData = _email.GetEmailDataForMessageId(pMessageId)
-        _logger.Debug("Email Data fetched!")
-
-        Dim oSource = _email.GetOriginalEmailPath(pArgs.OriginalEmailDirectory, pMessageId)
-        _logger.Debug("Original email path: [{0}]", oSource)
-
-        Dim oDateSubDirectoryName As String = Now.ToString("yyyy-MM-dd")
-        Dim oDestination As String
-
-        Dim oRejectedDirectory As String = Path.Combine(pArgs.RejectedEmailDirectory, oDateSubDirectoryName)
-
-        ' Create the destination directory if it does not exist
-        _logger.Debug("Creating destination directory [{0}]", oRejectedDirectory)
-        If Not Directory.Exists(oRejectedDirectory) Then
-            Try
-                Directory.CreateDirectory(oRejectedDirectory)
-            Catch ex As Exception
-                _logger.Error(ex)
-            End Try
-        End If
-
-        If oSource = String.Empty Then
-            _logger.Warn("Original Email for [{0}] could not be found. Exiting.", pMessageId)
-            Return Nothing
-        End If
-
-        ' If oEmailData is Nothing, TBEDM_EMAIL_PROFILER_HISTORY for MessageId was not found. 
-        ' This only should happen when testing and db-tables are deleted frequently
-        If oEmailData Is Nothing Then
-            oDestination = _email.GetEmailPathWithSubjectAsName(oRejectedDirectory, pMessageId)
-        Else
-            oDestination = _email.GetEmailPathWithSubjectAsName(oRejectedDirectory, StringEx.ConvertTextToSlug(oEmailData.Subject))
-        End If
-
-        _logger.Debug("Destination for eml file is {0}", oDestination)
-
-        Dim oFinalFileName = _filesystem.GetVersionedFilename(oDestination)
-
-        _logger.Debug("Versioned filename for eml file is {0}", oFinalFileName)
-
-        If oEmailData Is Nothing Then
-            _logger.Warn("Could not get Email Data from firebird-database. File {0} will not be moved!", oSource)
-            Return Nothing
-        End If
-
-        '---------------------------
-
-        Try
-            _logger.Info("Moving email from {0} to {1}", oSource, oFinalFileName)
-            File.Move(oSource, oFinalFileName)
-            oEmailData.Attachment = oFinalFileName
-        Catch ex As Exception
-            _logger.Warn("File {0} could not be moved! Original Filename will be used!", oSource)
-            _logger.Error(ex)
-            oEmailData.Attachment = oSource
-        End Try
-
-        _logger.Info("Email [{0}] moved to rejected folder!", pMessageId)
-
-        Return oEmailData
-    End Function
-
-    Private Sub AddRejectedState(oMessageID As String, oTitle As String, oTitle1 As String, oComment As String, Transaction As SqlTransaction)
-        Try
-            'PRCUST_ADD_HISTORY_STATE: @MessageID VARCHAR(250), @TITLE1 VARCHAR(250), @TITLE2 VARCHAR(250)
-            Dim oSQL = $"EXEC PRCUST_ADD_HISTORY_STATE '{oMessageID}','{oTitle}','{oTitle1}','{oComment.Replace("'", "''")}'"
-            _mssql.ExecuteNonQuery(oSQL, Transaction)
-        Catch ex As Exception
-            _logger.Error(ex)
-        End Try
-    End Sub
-
     Public Sub Start(Arguments As Object) Implements IJob.Start
         Dim oArgs As WorkerArgs = Arguments
         Dim oPropertyExtractor = New PropertyValues(_logConfig)
@@ -159,27 +91,30 @@ Public Class ImportZUGFeRDFiles
         _logger.Debug("Starting Job {0}", [GetType].Name)
 
         Try
-            For Each oPath As String In oArgs.WatchDirectories
+            'For Each oPath As String In oArgs.WatchDirectory
+            Dim oPath As String = oArgs.WatchDirectory
             Dim oDirInfo As New DirectoryInfo(oPath)
 
             _logger.Debug($"Start processing directory {oDirInfo.FullName}")
 
-                If oDirInfo.Exists Then
+            If oDirInfo.Exists = False Then
+                _logger.Warn("Watch directory exists. Exiting.")
+                Exit Sub
+            End If
+
             '  Filter out *.lock files
             Dim oFiles As List(Of FileInfo) = oDirInfo.
                     GetFiles().
                     Where(Function(f) Not f.Name.EndsWith(".lock")).
                     ToList()
-                    Dim oFileCount = oFiles.Count
-                    Dim oCurrentFileCount = 0
 
-                    If oFileCount = 0 Then
+            If oFiles.Count = 0 Then
-                        _logger.Debug("No files to process.")
+                _logger.Debug("No files to process. Exiting.")
-                        Continue For
+                Exit Sub
-                    Else
-                        _logger.Info("Found {0} files", oFileCount)
             End If
 
+            _logger.Info("Found {0} files", oFiles.Count)
+
             ' Group files by messageId
             Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = _zugferd.FileGroup.GroupFiles(oFiles)
 
@@ -223,6 +158,9 @@ Public Class ImportZUGFeRDFiles
 
                 _logger.Info("Start processing file group {0}", oMessageId)
 
+                ' TODO: Use this refactored function 
+                ' ProcessFileGroup(oMessageId, oFileGroupFiles, oConnections, oArgs)
+
                 Try
                     For Each oFile In oFileGroupFiles
                         ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the 
@@ -237,7 +175,6 @@ Public Class ImportZUGFeRDFiles
                         Dim oGlobalGroupCounter = 0
                         ' Clear missing properties for the new file
                         oMissingProperties = New List(Of String)
-                                oCurrentFileCount += 1
 
                         ' Only pdf files are allowed from here on
                         If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
@@ -307,7 +244,7 @@ Public Class ImportZUGFeRDFiles
                         End If
 
                         ' Check the Checksum and rejection status
-                                oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile.FullName, oMessageId, oArgs.IgnoreRejectionStatus)
+                        oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, oArgs.IgnoreRejectionStatus)
 
                         ' Check the document against the configured property map and return:
                         ' - a List of valid properties 
@@ -369,9 +306,9 @@ Public Class ImportZUGFeRDFiles
                     'If no errors occurred...
                     'Log the History
                     If oMD5CheckSum <> String.Empty Then
-                                _history.Create_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS", oSQLTransaction)
+                        _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS")
                     Else
-                                _history.Create_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)", oSQLTransaction)
+                        _history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)")
                     End If
 
                     oIsSuccess = True
@@ -382,7 +319,7 @@ Public Class ImportZUGFeRDFiles
 
                     Dim oErrors = ex.ValidationErrors
                     Dim oMessage = "REJECTED - ZUGFeRD yes but formal validation failed!"
-                            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, oMessage, oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, oMessage)
 
                     Dim oErrorList As String = ""
                     For Each oError In oErrors
@@ -400,9 +337,9 @@ Public Class ImportZUGFeRDFiles
                     ' When MD5HashException is thrown, we don't have a MD5Hash yet.
                     ' That 's why we set it to String.Empty here.
                     Dim oMessage = "REJECTED - Already processed (MD5Hash)"
-                            _history.Update_HistoryEntry(oMessageId, String.Empty, oMessage, oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, String.Empty, oMessage)
 
-                            Dim oBody = EmailStrings.EMAIL_MD5_ERROR
+                    Dim oBody = String.Format(EmailStrings.EMAIL_MD5_ERROR, ex.FileName)
                     Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                     _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MD5HashException", _EmailOutAccountId, oArgs.NamePortal)
                     AddRejectedState(oMessageId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "", oSQLTransaction)
@@ -412,7 +349,7 @@ Public Class ImportZUGFeRDFiles
 
                     ' When UnsupportedFerdException is thrown, we don't have a MD5Hash yet.
                     ' That 's why we set it to String.Empty here.
-                            _history.Create_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format", oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format")
 
                     Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                     Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, oEmailData.Subject, ex.XmlFile)
@@ -425,7 +362,7 @@ Public Class ImportZUGFeRDFiles
 
                     ' When InvalidFerdException is thrown, we don't have a MD5Hash yet.
                     ' That 's why we set it to String.Empty here.
-                            _history.Create_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format", oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format")
 
                     Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                     Dim oBody = String.Format(EmailStrings.EMAIL_INVALID_DOCUMENT, oEmailData.Subject)
@@ -436,7 +373,7 @@ Public Class ImportZUGFeRDFiles
                 Catch ex As TooMuchFerdsException
                     _logger.Error(ex)
 
-                            _history.Create_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email", oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email")
 
                     Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                     Dim oBody = String.Format(EmailStrings.EMAIL_TOO_MUCH_FERDS, oEmailData.Subject)
@@ -447,7 +384,7 @@ Public Class ImportZUGFeRDFiles
                 Catch ex As NoFerdsException
                     _logger.Error(ex)
 
-                            _history.Create_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email", oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email")
 
                     Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
                     Dim oBody = String.Format(EmailStrings.EMAIL_NO_FERDS, oEmailData.Subject)
@@ -463,7 +400,7 @@ Public Class ImportZUGFeRDFiles
                         oMessage &= $"- {prop}"
                     Next
 
-                            _history.Create_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]", oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]")
 
                     Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties)
                     Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
@@ -473,7 +410,7 @@ Public Class ImportZUGFeRDFiles
                 Catch ex As FileSizeLimitReachedException
                     _logger.Error(ex)
 
-                            _history.Create_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - File size limit reached", oSQLTransaction)
+                    _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - File size limit reached")
 
                     Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
 
@@ -576,8 +513,7 @@ Public Class ImportZUGFeRDFiles
                         _logger.Warn("Database Connections were not closed successfully.")
                     End Try
                 End Try
-                    Next
+
-                End If
             Next
 
             _logger.Debug("Finishing Job {0}", Me.GetType.Name)
@@ -587,6 +523,520 @@ Public Class ImportZUGFeRDFiles
         End Try
     End Sub
 
+
+
+#Region "=== REFACTOR"
+    Private Function ProcessFileGroup(oMessageId As String, pFiles As List(Of FileInfo), oConnections As DatabaseConnections, pArgs As WorkerArgs)
+        Dim oMissingProperties = New List(Of String)
+        Dim oEmailAttachmentFiles = New List(Of FileInfo)
+        Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile)
+        Dim oZUGFeRDCount = 0
+        Dim oMD5CheckSum As String = Nothing
+
+        ' Set the default Move Directory
+        Dim oMoveDirectory As String = pArgs.ErrorDirectory
+
+        ' Flag to save if the whole process was a success.
+        ' Will be set only at the end of the function if no error occurred.
+        Dim oIsSuccess As Boolean = False
+
+        ' Flag to save if the occurred error (if any) was expected
+        ' Used to determine if transactions should be committed or not
+        Dim oExpectedError As Boolean = True
+
+        Try
+            For Each oFile In pFiles
+                ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the 
+                '   different versions of ZUGFeRD and the type is unknown at compile-time.
+                ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
+                '   of the extracted xml file.
+                ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
+                '   the new meta data, ie. the type of schema (zugferd version)
+                Dim oDocument As ZUGFeRDInterface.ZugferdResult
+
+                ' Start a global group counter for each file
+                Dim oGlobalGroupCounter = 0
+                ' Clear missing properties for the new file
+                oMissingProperties = New List(Of String)
+
+                ' Only pdf files are allowed from here on
+                If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
+                    _logger.Debug("Skipping non-pdf file {0}", oFile.Name)
+                    oEmailAttachmentFiles.Add(oFile)
+
+                    ' Checking filesize for attachment files
+                    If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
+                        _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+                        Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+                    End If
+
+                    Continue For
+                End If
+
+                _logger.Info("Start processing file {0}", oFile.Name)
+
+                ' Checking filesize for pdf files
+                If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
+                    _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+                    Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+                End If
+
+                Try
+                    oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName)
+
+                Catch ex As ValidationException
+                    Throw ex
+
+                Catch ex As ZUGFeRDExecption
+                    Select Case ex.ErrorType
+                        Case ZUGFeRDInterface.ErrorType.NoZugferd
+                            _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
+                            oEmailAttachmentFiles.Add(oFile)
+                            Continue For
+
+                        Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
+                            _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
+                            Throw New UnsupportedFerdException(ex.XmlFile)
+
+                        Case ZUGFeRDInterface.ErrorType.NoValidZugferd
+                            _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name)
+                            Throw New InvalidFerdException()
+
+                        Case Else
+                            _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name)
+                            Throw ex
+                    End Select
+                End Try
+
+                ' Check if there are more than one ZUGFeRD files
+                If oZUGFeRDCount = 1 Then
+                    Throw New TooMuchFerdsException()
+                End If
+
+                ' Since extraction went well, increase the amount of ZUGFeRD files
+                oZUGFeRDCount += 1
+
+                ' Extract all attachments with the extensions specified in `AllowedExtensions`.
+                ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
+                ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`.
+                Dim oAttachments = _embeds.Extract(oFile.FullName, AllowedExtensions)
+                If oAttachments Is Nothing Then
+                    _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
+                Else
+                    oEmbeddedAttachmentFiles.AddRange(oAttachments)
+                End If
+
+                ' Check the Checksum and rejection status
+                oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, pArgs.IgnoreRejectionStatus)
+
+                ' Check the document against the configured property map and return:
+                ' - a List of valid properties 
+                ' - a List of missing properties
+
+                Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, oDocument.Specification)
+                Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId)
+
+                _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)
+
+                If oCheckResult.MissingProperties.Count > 0 Then
+                    _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
+                    oMissingProperties = oCheckResult.MissingProperties
+                    Throw New MissingValueException(oFile)
+                Else
+                    _logger.Debug("No missing properties found. Continuing.")
+
+                End If
+
+                DeleteExistingPropertyValues(oMessageId, oConnections)
+
+                Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault()
+                If oFirstProperty IsNot Nothing Then
+                    InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With {
+                        .MessageId = oMessageId,
+                        .Description = "ZUGFeRDSpezifikation",
+                        .GroupCounter = 0,
+                        .IsRequired = False,
+                        .Value = oDocument.Specification,
+                        .TableName = oFirstProperty.TableName,
+                        .TableColumn = "ZUGFERD_SPECIFICATION"
+                    })
+                End If
+
+                For Each oProperty In oCheckResult.ValidProperties
+                    InsertPropertyValue(oMessageId, oConnections, oProperty)
+                Next
+            Next
+
+            'Check if there are no ZUGFeRD files
+            If oZUGFeRDCount = 0 Then
+
+                ' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated
+                ' This is the default/initial behaviour.
+                If pArgs.NonZugferdDirectory Is Nothing OrElse pArgs.NonZugferdDirectory = String.Empty Then
+                    Throw New NoFerdsException()
+                End If
+
+                ' Also, if the directory is set but does not exist, still a rejection will be generated.
+                If Not IO.Directory.Exists(pArgs.NonZugferdDirectory) Then
+                    Throw New NoFerdsException()
+                End If
+
+                ' Only if the directory is set and does exist, it will be used and any file groups which
+                ' do NOT CONTAIN ANY ZUGFERD DOCUMENTS, are moved to that directory.
+                Throw New NoFerdsAlternateException()
+
+            End If
+
+            'If no errors occurred...
+            'Log the History
+            If oMD5CheckSum <> String.Empty Then
+                _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS")
+            Else
+                _history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)")
+            End If
+
+            oIsSuccess = True
+            oMoveDirectory = pArgs.SuccessDirectory
+
+        Catch ex As ValidationException
+            _logger.Error(ex)
+
+            Dim oErrors = ex.ValidationErrors
+            Dim oMessage = "REJECTED - ZUGFeRD yes but formal validation failed!"
+            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, oMessage)
+
+            Dim oErrorList As String = ""
+            For Each oError In oErrors
+                oErrorList += $"<li>Element '{oError.ElementName}' mit Wert '{oError.ElementValue}': {oError.ErrorMessage}</li>"
+            Next
+
+            Dim oBody = String.Format(EmailStrings.EMAIL_VALIDATION_ERROR, oErrorList)
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "ValidationException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "ValidationException", "Die Rechnungsvalidierung ist fehlgeschlagen!", "", oConnections.SQLServerTransaction)
+
+        Catch ex As MD5HashException
+            _logger.Error(ex)
+
+            ' When MD5HashException is thrown, we don't have a MD5Hash yet.
+            ' That 's why we set it to String.Empty here.
+            Dim oMessage = "REJECTED - Already processed (MD5Hash)"
+            _history.Update_HistoryEntry(oMessageId, String.Empty, oMessage)
+
+            Dim oBody = EmailStrings.EMAIL_MD5_ERROR
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MD5HashException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "", oConnections.SQLServerTransaction)
+
+        Catch ex As UnsupportedFerdException
+            _logger.Error(ex)
+
+            ' When UnsupportedFerdException is thrown, we don't have a MD5Hash yet.
+            ' That 's why we set it to String.Empty here.
+            _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format")
+
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+            Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, oEmailData.Subject, ex.XmlFile)
+
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnsupportedFerdException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "UnsupportedFerdException", "Nicht unterstütztes Datenformat", "", oConnections.SQLServerTransaction)
+
+        Catch ex As InvalidFerdException
+            _logger.Error(ex)
+
+            ' When InvalidFerdException is thrown, we don't have a MD5Hash yet.
+            ' That 's why we set it to String.Empty here.
+            _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format")
+
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+            Dim oBody = String.Format(EmailStrings.EMAIL_INVALID_DOCUMENT, oEmailData.Subject)
+
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "InvalidFerdException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "InvalidFerdException", "Inkorrekte Formate", "", oConnections.SQLServerTransaction)
+
+        Catch ex As TooMuchFerdsException
+            _logger.Error(ex)
+
+            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email")
+
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+            Dim oBody = String.Format(EmailStrings.EMAIL_TOO_MUCH_FERDS, oEmailData.Subject)
+
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "TooMuchFerdsException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "", oConnections.SQLServerTransaction)
+
+        Catch ex As NoFerdsException
+            _logger.Error(ex)
+
+            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email")
+
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+            Dim oBody = String.Format(EmailStrings.EMAIL_NO_FERDS, oEmailData.Subject)
+
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "NoFerdsException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "", oConnections.SQLServerTransaction)
+
+        Catch ex As MissingValueException
+            _logger.Error(ex)
+
+            Dim oMessage As String = ""
+            For Each prop In oMissingProperties
+                oMessage &= $"- {prop}"
+            Next
+
+            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]")
+
+            Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties)
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oConnections.SQLServerTransaction)
+
+        Catch ex As FileSizeLimitReachedException
+            _logger.Error(ex)
+
+            _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - File size limit reached")
+
+            Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
+
+            Dim oKey = FileSizeLimitReachedException.KEY_FILENAME
+            Dim oFileExceedingThreshold As String = IIf(ex.Data.Contains(oKey), ex.Data.Item(oKey), "")
+            Dim oFileWithoutMessageId = oFileExceedingThreshold.
+                Replace(oMessageId, "").
+                Replace("~", "")
+
+            Dim oBody = String.Format(EmailStrings.EMAIL_FILE_SIZE_REACHED, pArgs.MaxAttachmentSizeInMegaBytes, oFileWithoutMessageId)
+
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileSizeLimitReachedException", _EmailOutAccountId, pArgs.NamePortal)
+            AddRejectedState(oMessageId, "FileSizeLimitReachedException", "Erlaubte Dateigröße überschritten", "", oConnections.SQLServerTransaction)
+
+
+        Catch ex As NoFerdsAlternateException
+            ' TODO: Maybe dont even log this 'error', since it's not really an error and it might happen *A LOT*
+            _logger.Error(ex)
+            oMoveDirectory = pArgs.NonZugferdDirectory
+
+        Catch ex As OutOfMemoryException
+            _logger.Warn("OutOfMemory Error occurred: {0}", ex.Message)
+            _logger.Error(ex)
+
+            ' Send Email to Digital Data
+            Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
+            Dim oEmailData As New EmailData With {
+                .From = pArgs.ExceptionEmailAddress,
+                .Subject = $"OutOfMemoryException im ZUGFeRD-Parser @ {oMessageId}"
+            }
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "OutOfMemoryException", _EmailOutAccountId, pArgs.NamePortal)
+
+            ' Rollback Transaction
+            oConnections.SQLServerTransaction.Rollback()
+
+            oMoveDirectory = DIRECTORY_DONT_MOVE
+
+            oExpectedError = False
+
+        Catch ex As Exception
+            _logger.Warn("Unknown Error occurred: {0}", ex.Message)
+            _logger.Error(ex)
+
+            ' Send Email to Digital Data
+            Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
+            Dim oEmailData As New EmailData With {
+                .From = pArgs.ExceptionEmailAddress,
+                .Subject = $"UnhandledException im ZUGFeRD-Parser @ {oMessageId}"
+            }
+            _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnhandledException", _EmailOutAccountId, pArgs.NamePortal)
+
+            ' Rollback Transaction
+            oConnections.SQLServerTransaction.Rollback()
+
+            oMoveDirectory = DIRECTORY_DONT_MOVE
+
+            oExpectedError = False
+
+        Finally
+            Try
+                ' If an application error occurred, dont move files so they will be processed again later
+                If oMoveDirectory = DIRECTORY_DONT_MOVE Then
+                    _logger.Info("Application Error occurred. Files for message Id {0} will not be moved.", oMessageId)
+                Else
+                    ' Move all files of the current group
+                    _file.MoveFiles(pArgs, oMessageId, pFiles, oEmailAttachmentFiles, oEmbeddedAttachmentFiles, oMoveDirectory, oIsSuccess)
+                End If
+                _logger.Info("Finished processing file group {0}", oMessageId)
+            Catch ex As Exception
+                ' Send Email to Digital Data
+                Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
+                Dim oEmailData As New EmailData With {
+                    .From = pArgs.ExceptionEmailAddress,
+                    .Subject = $"FileMoveException im ZUGFeRD-Parser @ {oMessageId}"
+                }
+                _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileMoveException", _EmailOutAccountId, pArgs.NamePortal)
+
+                _logger.Warn("Could not move files!")
+                _logger.Error(ex)
+                Throw ex
+            End Try
+
+            Try
+                ' If everything went OK or an expected error occurred, 
+                ' finally commit all changes To the Database
+                ' ==================================================================
+                If oIsSuccess Or oExpectedError Then
+                    ' Commit Transaction
+                    oConnections.SQLServerTransaction.Commit()
+                End If
+
+                _logger.Info("FileGroup for MessageId [{0}] successfully processed!", oMessageId)
+            Catch ex As Exception
+                _logger.Error(ex)
+                _logger.Warn("Database Transactions were not committed successfully.")
+            End Try
+
+            Try
+                oConnections.SQLServerConnection.Close()
+            Catch ex As Exception
+                _logger.Error(ex)
+                _logger.Warn("Database Connections were not closed successfully.")
+            End Try
+        End Try
+    End Function
+
+    Private Function ProcessFile(pMessageId As String, oFile As FileInfo, oConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult
+        ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the 
+        '   different versions of ZUGFeRD and the type is unknown at compile-time.
+        ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
+        '   of the extracted xml file.
+        ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
+        '   the new meta data, ie. the type of schema (zugferd version)
+        Dim oDocument As ZUGFeRDInterface.ZugferdResult
+
+        Dim oMissingProperties = New List(Of String)
+        Dim oEmailAttachmentFiles = New List(Of FileInfo)
+        Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile)
+        Dim oZugferdFiles As Integer = 0
+
+        ' Start a global group counter for each file
+        Dim oGlobalGroupCounter = 0
+        ' Clear missing properties for the new file
+        oMissingProperties = New List(Of String)
+
+        ' Only pdf files are allowed from here on
+        If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
+            _logger.Debug("Skipping non-pdf file {0}", oFile.Name)
+            oEmailAttachmentFiles.Add(oFile)
+
+            ' Checking filesize for attachment files
+            If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
+                _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+                Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+            End If
+
+            'TODO: how to handle?
+            'Continue For
+        End If
+
+        _logger.Info("Start processing file {0}", oFile.Name)
+
+        ' Checking filesize for pdf files
+        If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
+            _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+            Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
+        End If
+
+        Try
+            oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName)
+
+        Catch ex As ValidationException
+            Throw ex
+
+        Catch ex As ZUGFeRDExecption
+            Select Case ex.ErrorType
+                Case ZUGFeRDInterface.ErrorType.NoZugferd
+                    _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
+                    oEmailAttachmentFiles.Add(oFile)
+
+                    'TODO: How to handle?
+                    'Continue For
+
+                Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
+                    _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
+                    Throw New UnsupportedFerdException(ex.XmlFile)
+
+                Case ZUGFeRDInterface.ErrorType.NoValidZugferd
+                    _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name)
+                    Throw New InvalidFerdException()
+
+                Case Else
+                    _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name)
+                    Throw ex
+            End Select
+        End Try
+
+        ' Check if there are more than one ZUGFeRD files
+        If oZugferdFiles = 1 Then
+            Throw New TooMuchFerdsException()
+        End If
+
+        ' Since extraction went well, increase the amount of ZUGFeRD files
+        oZugferdFiles += 1
+
+        ' Extract all attachments with the extensions specified in `AllowedExtensions`.
+        ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
+        ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`.
+        Dim oAttachments = _embeds.Extract(oFile.FullName, AllowedExtensions)
+        If oAttachments Is Nothing Then
+            _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
+        Else
+            oEmbeddedAttachmentFiles.AddRange(oAttachments)
+        End If
+
+        ' Check the Checksum and rejection status
+        Dim oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, pMessageId, pArgs.IgnoreRejectionStatus)
+
+        ' Check the document against the configured property map and return:
+        ' - a List of valid properties 
+        ' - a List of missing properties
+
+        Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, oDocument.Specification)
+        Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, pMessageId)
+
+        _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)
+
+        If oCheckResult.MissingProperties.Count > 0 Then
+            _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
+            oMissingProperties = oCheckResult.MissingProperties
+            Throw New MissingValueException(oFile)
+        Else
+            _logger.Debug("No missing properties found. Continuing.")
+
+        End If
+
+        DeleteExistingPropertyValues(pMessageId, oConnections)
+
+        Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault()
+        If oFirstProperty IsNot Nothing Then
+            InsertPropertyValue(pMessageId, oConnections, New PropertyValues.ValidProperty() With {
+                .MessageId = pMessageId,
+                .Description = "ZUGFeRDSpezifikation",
+                .GroupCounter = 0,
+                .IsRequired = False,
+                .Value = oDocument.Specification,
+                .TableName = oFirstProperty.TableName,
+                .TableColumn = "ZUGFERD_SPECIFICATION"
+            })
+        End If
+
+        For Each oProperty In oCheckResult.ValidProperties
+            InsertPropertyValue(pMessageId, oConnections, oProperty)
+        Next
+
+        Return New ProcessFileResult With {
+            .ZugferdFileCount = oZugferdFiles
+        }
+    End Function
+#End Region
+
+
     Private Sub DeleteExistingPropertyValues(pMessageId As String, pConnections As DatabaseConnections)
         Dim oDelSQL = $"DELETE FROM TBEDMI_ITEM_VALUE where REFERENCE_GUID = '{pMessageId}'"
         Dim oStep As String
@@ -648,12 +1098,12 @@ Public Class ImportZUGFeRDFiles
     ''' <param name="pIgnoreRejectionStatus">Should the check take into account the rejection status of the file?</param>
     ''' <returns>The MD5 Checksum of the file, or an empty string, if the Checksum could not be created</returns>
     ''' <exception cref="MD5HashException">Throws, when the file should be rejected, ie. if it already exists in the table</exception>
-    Private Function GenerateAndCheck_MD5Sum(pFilePath As String, pMessageId As String, pIgnoreRejectionStatus As Boolean) As String
+    Private Function GenerateAndCheck_MD5Sum(pFile As FileInfo, pMessageId As String, pIgnoreRejectionStatus As Boolean) As String
-        Dim oMD5CheckSum = CreateMD5(pFilePath)
+        Dim oMD5CheckSum = CreateMD5(pFile.FullName)
 
         ' Exit if MD5 could not be created
         If oMD5CheckSum = String.Empty Then
-            _logger.Warn("MD5 Checksum is nothing for file [{0}]!", pFilePath)
+            _logger.Warn("MD5 Checksum is nothing for file [{0}]!", pFile.FullName)
             Return oMD5CheckSum
         End If
 
@@ -668,13 +1118,13 @@ Public Class ImportZUGFeRDFiles
 
         ' If History entries could not be fetched, just return the MD5 Checksum
         If IsNothing(oTable) Then
-            _logger.Warn("Be careful: oExistsDT is nothing for file [{0}]!", pFilePath)
+            _logger.Warn("Be careful: oExistsDT is nothing for file [{0}]!", pFile.FullName)
             Return oMD5CheckSum
         End If
 
         ' If Checksum does not exist in History entries, just return the MD5 Checksum
         If oTable.Rows.Count = 0 Then
-            _logger.Debug("File [{0}] was not found in History!", pFilePath)
+            _logger.Debug("File [{0}] was not found in History!", pFile.FullName)
             Return oMD5CheckSum
         End If
 
@@ -711,12 +1161,88 @@ Public Class ImportZUGFeRDFiles
             _logger.Info("ZuGFeRDFile already has been processed, but Rejection Status will be ignored!")
         ElseIf oRejected = False Then
             _logger.Info("ZuGFeRDFile already has been processed. Will be rejected.")
-            Throw New MD5HashException($"There is already an identical invoice! - HistoryID [{oHistoryId}] - Case 1")
+            Throw New MD5HashException($"There is already an identical invoice! - HistoryID [{oHistoryId}] - Case 1", pFile.Name)
         Else
             _logger.Info("ZuGFeRDFile already has been processed. Will be rejected.")
-            Throw New MD5HashException($"There is already an identical invoice! - HistoryID [{oHistoryId}] - Case 2")
+            Throw New MD5HashException($"There is already an identical invoice! - HistoryID [{oHistoryId}] - Case 2", pFile.Name)
         End If
 
         Return oMD5CheckSum
     End Function
+
+    Private Function MoveAndRenameEmailToRejected(pArgs As WorkerArgs, pMessageId As String) As EmailData
+        _logger.Info("Moving Mail with MessageId [{0}] to Rejected folder", pMessageId)
+        _logger.Debug("Fetching Email Data")
+
+        Dim oEmailData = _email.GetEmailDataForMessageId(pMessageId)
+        _logger.Debug("Email Data fetched!")
+
+        Dim oSource = _email.GetOriginalEmailPath(pArgs.OriginalEmailDirectory, pMessageId)
+        _logger.Debug("Original email path: [{0}]", oSource)
+
+        Dim oDateSubDirectoryName As String = Now.ToString("yyyy-MM-dd")
+        Dim oDestination As String
+
+        Dim oRejectedDirectory As String = Path.Combine(pArgs.RejectedEmailDirectory, oDateSubDirectoryName)
+
+        ' Create the destination directory if it does not exist
+        _logger.Debug("Creating destination directory [{0}]", oRejectedDirectory)
+        If Not Directory.Exists(oRejectedDirectory) Then
+            Try
+                Directory.CreateDirectory(oRejectedDirectory)
+            Catch ex As Exception
+                _logger.Error(ex)
+            End Try
+        End If
+
+        If oSource = String.Empty Then
+            _logger.Warn("Original Email for [{0}] could not be found. Exiting.", pMessageId)
+            Return New EmailData()
+        End If
+
+        ' If oEmailData is Nothing, TBEDM_EMAIL_PROFILER_HISTORY for MessageId was not found. 
+        ' This only should happen when testing and db-tables are deleted frequently
+        If oEmailData Is Nothing Then
+            oDestination = _email.GetEmailPathWithSubjectAsName(oRejectedDirectory, pMessageId)
+        Else
+            oDestination = _email.GetEmailPathWithSubjectAsName(oRejectedDirectory, StringEx.ConvertTextToSlug(oEmailData.Subject))
+        End If
+
+        _logger.Debug("Destination for eml file is {0}", oDestination)
+
+        Dim oFinalFileName = _filesystem.GetVersionedFilename(oDestination)
+
+        _logger.Debug("Versioned filename for eml file is {0}", oFinalFileName)
+
+        If oEmailData Is Nothing Then
+            _logger.Warn("Could not get Email Data from firebird-database. File {0} will not be moved!", oSource)
+            Return New EmailData()
+        End If
+
+        '---------------------------
+
+        Try
+            _logger.Info("Moving email from {0} to {1}", oSource, oFinalFileName)
+            File.Move(oSource, oFinalFileName)
+            oEmailData.Attachment = oFinalFileName
+        Catch ex As Exception
+            _logger.Warn("File {0} could not be moved! Original Filename will be used!", oSource)
+            _logger.Error(ex)
+            oEmailData.Attachment = oSource
+        End Try
+
+        _logger.Info("Email [{0}] moved to rejected folder!", pMessageId)
+
+        Return oEmailData
+    End Function
+
+    Private Sub AddRejectedState(oMessageID As String, oTitle As String, oTitle1 As String, oComment As String, Transaction As SqlTransaction)
+        Try
+            'PRCUST_ADD_HISTORY_STATE: @MessageID VARCHAR(250), @TITLE1 VARCHAR(250), @TITLE2 VARCHAR(250)
+            Dim oSQL = $"EXEC PRCUST_ADD_HISTORY_STATE '{oMessageID}','{oTitle}','{oTitle1}','{oComment.Replace("'", "''")}'"
+            _mssql.ExecuteNonQuery(oSQL, Transaction)
+        Catch ex As Exception
+            _logger.Error(ex)
+        End Try
+    End Sub
 End Class