From c187bdbe5e99503446c1b6e9bdf51452cb589d40 Mon Sep 17 00:00:00 2001 From: Jonathan Jenne Date: Mon, 9 Oct 2023 16:05:50 +0200 Subject: [PATCH] Refactor: Processfiles --- Jobs/Exceptions.vb | 8 +- Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb | 697 +++++++---------------------- 2 files changed, 162 insertions(+), 543 deletions(-) diff --git a/Jobs/Exceptions.vb b/Jobs/Exceptions.vb index b853a713..d400e9fa 100644 --- a/Jobs/Exceptions.vb +++ b/Jobs/Exceptions.vb @@ -7,11 +7,13 @@ Public Class Exceptions Inherits ApplicationException Public ReadOnly File As FileInfo + Public ReadOnly MissingProperties As List(Of String) - Public Sub New(File As FileInfo) - MyBase.New($"Missing values in [{File.Name}]") + Public Sub New(pFile As FileInfo, pMissingProperties As List(Of String)) + MyBase.New($"Missing values in [{pFile.Name}]") - Me.File = File + Me.File = pFile + Me.MissingProperties = pMissingProperties End Sub End Class diff --git a/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb b/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb index 403564e2..694004b5 100644 --- a/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb +++ b/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb @@ -45,9 +45,13 @@ Public Class ImportZUGFeRDFiles Private _EmailOutAccountId As Integer Private Class ProcessFileResult - Public ZugferdFileCount As Integer - Public MD5Hash As String = Nothing + Public ZugferdFileFound As Boolean = False + Public ZugferdFileCount As Integer = 0 + Public MD5Checksum As String = Nothing + + Public EmailAttachmentFiles As New List(Of FileInfo) + Public EmbeddedAttachmentFiles As New List(Of PDFEmbeds.EmbeddedFile) End Class Private Class DatabaseConnections @@ -152,147 +156,147 @@ Public Class ImportZUGFeRDFiles Dim oEmbeddedAttachmentFiles As New List(Of PDFEmbeds.EmbeddedFile) Dim oMessageId As String = oFileGroup.Key - Dim oMissingProperties As New List(Of String) Dim oMD5CheckSum As String = String.Empty _logger.Info("Start processing file group {0}", oMessageId) - ' TODO: Use this refactored function - ' ProcessFileGroup(oMessageId, oFileGroupFiles, oConnections, oArgs) + Dim oEmailDataBase = _email.GetEmailDataForMessageId(oMessageId) Try For Each oFile In oFileGroupFiles - ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the - ' different versions of ZUGFeRD and the type is unknown at compile-time. - ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename - ' of the extracted xml file. - ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save - ' the new meta data, ie. the type of schema (zugferd version) - Dim oDocument As ZUGFeRDInterface.ZugferdResult - - ' Start a global group counter for each file - Dim oGlobalGroupCounter = 0 - ' Clear missing properties for the new file - oMissingProperties = New List(Of String) - - ' Only pdf files are allowed from here on - If Not oFile.Name.ToUpper.EndsWith(".PDF") Then - _logger.Debug("Skipping non-pdf file {0}", oFile.Name) - oEmailAttachmentFiles.Add(oFile) - - ' Checking filesize for attachment files - If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then - _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) - Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) - End If - - Continue For + Dim oResult As ProcessFileResult = ProcessFile(oMessageId, oEmailDataBase, oZUGFeRDCount, oFile, oConnections, oArgs) + + If oResult.ZugferdFileFound = False Then + _logger.Debug("Zugferd File found") + oMD5CheckSum = oResult.MD5Checksum + oZUGFeRDCount = oResult.ZugferdFileCount + oEmailAttachmentFiles.AddRange(oResult.EmailAttachmentFiles) + oEmbeddedAttachmentFiles.AddRange(oResult.EmbeddedAttachmentFiles) End If - - _logger.Info("Start processing file {0}", oFile.Name) - - ' Checking filesize for pdf files - If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then - _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) - Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) - End If - - Try - oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName) - - Catch ex As ValidationException - Throw ex - - Catch ex As ZUGFeRDExecption - Select Case ex.ErrorType - Case ZUGFeRDInterface.ErrorType.NoZugferd - _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name) - oEmailAttachmentFiles.Add(oFile) - Continue For - - Case ZUGFeRDInterface.ErrorType.UnsupportedFormat - _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile) - Throw New UnsupportedFerdException(ex.XmlFile) - - Case ZUGFeRDInterface.ErrorType.NoValidZugferd - _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name) - Throw New InvalidFerdException() - - Case Else - _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name) - Throw ex - End Select - End Try - - ' Check if there are more than one ZUGFeRD files - If oZUGFeRDCount = 1 Then - Throw New TooMuchFerdsException() - End If - - ' Since extraction went well, increase the amount of ZUGFeRD files - oZUGFeRDCount += 1 - - ' Extract all attachments with the extensions specified in `AllowedExtensions`. - ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself. - ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`. - Dim oAttachments = oAttachmentExtractor.Extract(oFile.FullName, AllowedExtensions) - If oAttachments Is Nothing Then - _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName) - Else - oEmbeddedAttachmentFiles.AddRange(oAttachments) - End If - - ' Check the Checksum and rejection status - oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, oArgs.IgnoreRejectionStatus) - - ' Check the document against the configured property map and return: - ' - a List of valid properties - ' - a List of missing properties - - Dim oPropertyMap = _zugferd.FilterPropertyMap(oArgs.PropertyMap, oDocument.Specification) - Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId) - - _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count) - - If oCheckResult.MissingProperties.Count > 0 Then - _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count) - oMissingProperties = oCheckResult.MissingProperties - Throw New MissingValueException(oFile) - Else - _logger.Debug("No missing properties found. Continuing.") - - End If - - DeleteExistingPropertyValues(oMessageId, oConnections) - - Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault() - If oFirstProperty IsNot Nothing Then - InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With { - .MessageId = oMessageId, - .Description = "ZUGFeRDSpezifikation", - .GroupCounter = 0, - .IsRequired = False, - .Value = oDocument.Specification, - .TableName = oFirstProperty.TableName, - .TableColumn = "ZUGFERD_SPECIFICATION" - }) - End If - - For Each oProperty In oCheckResult.ValidProperties - InsertPropertyValue(oMessageId, oConnections, oProperty) - Next + ' ------------------------------------------------------------ +#Region "REFACTOR" + + + 'Dim oDocument As ZUGFeRDInterface.ZugferdResult + + '' Only pdf files are allowed from here on + 'If Not oFile.Name.ToUpper.EndsWith(".PDF") Then + ' _logger.Debug("Skipping non-pdf file {0}", oFile.Name) + ' oEmailAttachmentFiles.Add(oFile) + + ' ' Checking filesize for attachment files + ' If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then + ' _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) + ' Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) + ' End If + + ' Continue For + 'End If + + '_logger.Info("Start processing file {0}", oFile.Name) + + '' Checking filesize for pdf files + 'If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then + ' _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) + ' Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes) + 'End If + + 'Try + ' oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName) + + 'Catch ex As ValidationException + ' Throw ex + + 'Catch ex As ZUGFeRDExecption + ' Select Case ex.ErrorType + ' Case ZUGFeRDInterface.ErrorType.NoZugferd + ' _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name) + ' oEmailAttachmentFiles.Add(oFile) + ' Continue For + + ' Case ZUGFeRDInterface.ErrorType.UnsupportedFormat + ' _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile) + ' Throw New UnsupportedFerdException(ex.XmlFile) + + ' Case ZUGFeRDInterface.ErrorType.NoValidZugferd + ' _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name) + ' Throw New InvalidFerdException() + + ' Case Else + ' _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name) + ' Throw ex + ' End Select + 'End Try + + '' Check if there are more than one ZUGFeRD files + 'If oZUGFeRDCount = 1 Then + ' Throw New TooMuchFerdsException() + 'End If + + '' Since extraction went well, increase the amount of ZUGFeRD files + 'oZUGFeRDCount += 1 + + '' Extract all attachments with the extensions specified in `AllowedExtensions`. + '' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself. + '' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`. + 'Dim oAttachments = oAttachmentExtractor.Extract(oFile.FullName, AllowedExtensions) + 'If oAttachments Is Nothing Then + ' _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName) + 'Else + ' oEmbeddedAttachmentFiles.AddRange(oAttachments) + 'End If + + '' Check the Checksum and rejection status + 'oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, oArgs.IgnoreRejectionStatus) + + '' Check the document against the configured property map and return: + '' - a List of valid properties + '' - a List of missing properties + + 'Dim oPropertyMap = _zugferd.FilterPropertyMap(oArgs.PropertyMap, oDocument.Specification) + 'Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId) + + '_logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count) + + 'If oCheckResult.MissingProperties.Count > 0 Then + ' _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count) + ' Throw New MissingValueException(oFile, oCheckResult.MissingProperties) + 'Else + ' _logger.Debug("No missing properties found. Continuing.") + + 'End If + + 'DeleteExistingPropertyValues(oMessageId, oConnections) + + 'Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault() + 'If oFirstProperty IsNot Nothing Then + ' InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With { + ' .MessageId = oMessageId, + ' .Description = "ZUGFeRDSpezifikation", + ' .GroupCounter = 0, + ' .IsRequired = False, + ' .Value = oDocument.Specification, + ' .TableName = oFirstProperty.TableName, + ' .TableColumn = "ZUGFERD_SPECIFICATION" + ' }) + 'End If + + 'For Each oProperty In oCheckResult.ValidProperties + ' InsertPropertyValue(oMessageId, oConnections, oProperty) + 'Next +#End Region Next 'Check if there are no ZUGFeRD files If oZUGFeRDCount = 0 Then ' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated ' This is the default/initial behaviour. - If oArgs.NonZugferdDirectory Is Nothing OrElse oArgs.NonZugferdDirectory = String.Empty Then + If String.IsNullOrEmpty(oArgs.NonZugferdDirectory) Then Throw New NoFerdsException() End If ' Also, if the directory is set but does not exist, still a rejection will be generated. - If Not IO.Directory.Exists(oArgs.NonZugferdDirectory) Then + If Not Directory.Exists(oArgs.NonZugferdDirectory) Then Throw New NoFerdsException() End If @@ -302,12 +306,11 @@ Public Class ImportZUGFeRDFiles End If - 'If no errors occurred... - 'Log the History - If oMD5CheckSum <> String.Empty Then - _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS") - Else + 'If no errors occurred, update the history table for this MessageId + If String.IsNullOrEmpty(oMD5CheckSum) Then _history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)") + Else + _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS") End If oIsSuccess = True @@ -395,13 +398,13 @@ Public Class ImportZUGFeRDFiles _logger.Error(ex) Dim oMessage As String = "" - For Each prop In oMissingProperties + For Each prop In ex.MissingProperties oMessage &= $"- {prop}" Next _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]") - Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties) + Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, ex.MissingProperties) Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId) _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, oArgs.NamePortal) AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oSQLTransaction) @@ -523,404 +526,15 @@ Public Class ImportZUGFeRDFiles End Sub #Region "=== REFACTOR" - Private Function ProcessFileGroup(oMessageId As String, pFiles As List(Of FileInfo), oConnections As DatabaseConnections, pArgs As WorkerArgs) - Dim oMissingProperties = New List(Of String) - Dim oEmailAttachmentFiles = New List(Of FileInfo) - Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile) - Dim oZUGFeRDCount = 0 - Dim oMD5CheckSum As String = Nothing - - ' Set the default Move Directory - Dim oMoveDirectory As String = pArgs.ErrorDirectory - - ' Flag to save if the whole process was a success. - ' Will be set only at the end of the function if no error occurred. - Dim oIsSuccess As Boolean = False - - ' Flag to save if the occurred error (if any) was expected - ' Used to determine if transactions should be committed or not - Dim oExpectedError As Boolean = True - - Try - For Each oFile In pFiles - ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the - ' different versions of ZUGFeRD and the type is unknown at compile-time. - ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename - ' of the extracted xml file. - ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save - ' the new meta data, ie. the type of schema (zugferd version) - Dim oDocument As ZUGFeRDInterface.ZugferdResult - - ' Start a global group counter for each file - Dim oGlobalGroupCounter = 0 - ' Clear missing properties for the new file - oMissingProperties = New List(Of String) - - ' Only pdf files are allowed from here on - If Not oFile.Name.ToUpper.EndsWith(".PDF") Then - _logger.Debug("Skipping non-pdf file {0}", oFile.Name) - oEmailAttachmentFiles.Add(oFile) - - ' Checking filesize for attachment files - If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then - _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) - Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) - End If - - Continue For - End If - - _logger.Info("Start processing file {0}", oFile.Name) - - ' Checking filesize for pdf files - If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then - _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) - Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) - End If - - Try - oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName) - - Catch ex As ValidationException - Throw ex - - Catch ex As ZUGFeRDExecption - Select Case ex.ErrorType - Case ZUGFeRDInterface.ErrorType.NoZugferd - _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name) - oEmailAttachmentFiles.Add(oFile) - Continue For - - Case ZUGFeRDInterface.ErrorType.UnsupportedFormat - _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile) - Throw New UnsupportedFerdException(ex.XmlFile) - - Case ZUGFeRDInterface.ErrorType.NoValidZugferd - _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name) - Throw New InvalidFerdException() - - Case Else - _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name) - Throw ex - End Select - End Try - - ' Check if there are more than one ZUGFeRD files - If oZUGFeRDCount = 1 Then - Throw New TooMuchFerdsException() - End If - - ' Since extraction went well, increase the amount of ZUGFeRD files - oZUGFeRDCount += 1 - - ' Extract all attachments with the extensions specified in `AllowedExtensions`. - ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself. - ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`. - Dim oAttachments = _embeds.Extract(oFile.FullName, AllowedExtensions) - If oAttachments Is Nothing Then - _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName) - Else - oEmbeddedAttachmentFiles.AddRange(oAttachments) - End If - - ' Check the Checksum and rejection status - oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, pArgs.IgnoreRejectionStatus) - - ' Check the document against the configured property map and return: - ' - a List of valid properties - ' - a List of missing properties - - Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, oDocument.Specification) - Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId) - - _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count) - - If oCheckResult.MissingProperties.Count > 0 Then - _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count) - oMissingProperties = oCheckResult.MissingProperties - Throw New MissingValueException(oFile) - Else - _logger.Debug("No missing properties found. Continuing.") - - End If - - DeleteExistingPropertyValues(oMessageId, oConnections) - - Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault() - If oFirstProperty IsNot Nothing Then - InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With { - .MessageId = oMessageId, - .Description = "ZUGFeRDSpezifikation", - .GroupCounter = 0, - .IsRequired = False, - .Value = oDocument.Specification, - .TableName = oFirstProperty.TableName, - .TableColumn = "ZUGFERD_SPECIFICATION" - }) - End If - - For Each oProperty In oCheckResult.ValidProperties - InsertPropertyValue(oMessageId, oConnections, oProperty) - Next - Next - - 'Check if there are no ZUGFeRD files - If oZUGFeRDCount = 0 Then - - ' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated - ' This is the default/initial behaviour. - If pArgs.NonZugferdDirectory Is Nothing OrElse pArgs.NonZugferdDirectory = String.Empty Then - Throw New NoFerdsException() - End If - - ' Also, if the directory is set but does not exist, still a rejection will be generated. - If Not IO.Directory.Exists(pArgs.NonZugferdDirectory) Then - Throw New NoFerdsException() - End If - - ' Only if the directory is set and does exist, it will be used and any file groups which - ' do NOT CONTAIN ANY ZUGFERD DOCUMENTS, are moved to that directory. - Throw New NoFerdsAlternateException() - - End If - - 'If no errors occurred... - 'Log the History - If oMD5CheckSum <> String.Empty Then - _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS") - Else - _history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)") - End If - - oIsSuccess = True - oMoveDirectory = pArgs.SuccessDirectory - - Catch ex As ValidationException - _logger.Error(ex) - - Dim oErrors = ex.ValidationErrors - Dim oMessage = "REJECTED - ZUGFeRD yes but formal validation failed!" - _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, oMessage) - - Dim oErrorList As String = "" - For Each oError In oErrors - oErrorList += $"
  • Element '{oError.ElementName}' mit Wert '{oError.ElementValue}': {oError.ErrorMessage}
    • " - Next - - Dim oBody = String.Format(EmailStrings.EMAIL_VALIDATION_ERROR, oErrorList) - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "ValidationException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "ValidationException", "Die Rechnungsvalidierung ist fehlgeschlagen!", "", oConnections.SQLServerTransaction) - - Catch ex As MD5HashException - _logger.Error(ex) - - ' When MD5HashException is thrown, we don't have a MD5Hash yet. - ' That 's why we set it to String.Empty here. - Dim oMessage = "REJECTED - Already processed (MD5Hash)" - _history.Update_HistoryEntry(oMessageId, String.Empty, oMessage) - - Dim oBody = EmailStrings.EMAIL_MD5_ERROR - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MD5HashException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "", oConnections.SQLServerTransaction) - - Catch ex As UnsupportedFerdException - _logger.Error(ex) - - ' When UnsupportedFerdException is thrown, we don't have a MD5Hash yet. - ' That 's why we set it to String.Empty here. - _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format") - - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, oEmailData.Subject, ex.XmlFile) - - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnsupportedFerdException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "UnsupportedFerdException", "Nicht unterstütztes Datenformat", "", oConnections.SQLServerTransaction) - - Catch ex As InvalidFerdException - _logger.Error(ex) - - ' When InvalidFerdException is thrown, we don't have a MD5Hash yet. - ' That 's why we set it to String.Empty here. - _history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format") - - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - Dim oBody = String.Format(EmailStrings.EMAIL_INVALID_DOCUMENT, oEmailData.Subject) - - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "InvalidFerdException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "InvalidFerdException", "Inkorrekte Formate", "", oConnections.SQLServerTransaction) - - Catch ex As TooMuchFerdsException - _logger.Error(ex) - - _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email") - - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - Dim oBody = String.Format(EmailStrings.EMAIL_TOO_MUCH_FERDS, oEmailData.Subject) - - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "TooMuchFerdsException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "", oConnections.SQLServerTransaction) - - Catch ex As NoFerdsException - _logger.Error(ex) - - _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email") - - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - Dim oBody = String.Format(EmailStrings.EMAIL_NO_FERDS, oEmailData.Subject) - - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "NoFerdsException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "", oConnections.SQLServerTransaction) - - Catch ex As MissingValueException - _logger.Error(ex) - - Dim oMessage As String = "" - For Each prop In oMissingProperties - oMessage &= $"- {prop}" - Next - - _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]") - - Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties) - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oConnections.SQLServerTransaction) - - Catch ex As FileSizeLimitReachedException - _logger.Error(ex) - - _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - File size limit reached") - - Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId) - - Dim oKey = FileSizeLimitReachedException.KEY_FILENAME - Dim oFileExceedingThreshold As String = IIf(ex.Data.Contains(oKey), ex.Data.Item(oKey), "") - Dim oFileWithoutMessageId = oFileExceedingThreshold. - Replace(oMessageId, ""). - Replace("~", "") - - Dim oBody = String.Format(EmailStrings.EMAIL_FILE_SIZE_REACHED, pArgs.MaxAttachmentSizeInMegaBytes, oFileWithoutMessageId) - - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileSizeLimitReachedException", _EmailOutAccountId, pArgs.NamePortal) - AddRejectedState(oMessageId, "FileSizeLimitReachedException", "Erlaubte Dateigröße überschritten", "", oConnections.SQLServerTransaction) - - - Catch ex As NoFerdsAlternateException - ' TODO: Maybe dont even log this 'error', since it's not really an error and it might happen *A LOT* - _logger.Error(ex) - oMoveDirectory = pArgs.NonZugferdDirectory - - Catch ex As OutOfMemoryException - _logger.Warn("OutOfMemory Error occurred: {0}", ex.Message) - _logger.Error(ex) - - ' Send Email to Digital Data - Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex) - Dim oEmailData As New EmailData With { - .From = pArgs.ExceptionEmailAddress, - .Subject = $"OutOfMemoryException im ZUGFeRD-Parser @ {oMessageId}" - } - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "OutOfMemoryException", _EmailOutAccountId, pArgs.NamePortal) - - ' Rollback Transaction - oConnections.SQLServerTransaction.Rollback() - - oMoveDirectory = DIRECTORY_DONT_MOVE - - oExpectedError = False - - Catch ex As Exception - _logger.Warn("Unknown Error occurred: {0}", ex.Message) - _logger.Error(ex) - - ' Send Email to Digital Data - Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex) - Dim oEmailData As New EmailData With { - .From = pArgs.ExceptionEmailAddress, - .Subject = $"UnhandledException im ZUGFeRD-Parser @ {oMessageId}" - } - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnhandledException", _EmailOutAccountId, pArgs.NamePortal) - - ' Rollback Transaction - oConnections.SQLServerTransaction.Rollback() - - oMoveDirectory = DIRECTORY_DONT_MOVE - - oExpectedError = False - - Finally - Try - ' If an application error occurred, dont move files so they will be processed again later - If oMoveDirectory = DIRECTORY_DONT_MOVE Then - _logger.Info("Application Error occurred. Files for message Id {0} will not be moved.", oMessageId) - Else - ' Move all files of the current group - _file.MoveFiles(pArgs, oMessageId, pFiles, oEmailAttachmentFiles, oEmbeddedAttachmentFiles, oMoveDirectory, oIsSuccess) - End If - _logger.Info("Finished processing file group {0}", oMessageId) - Catch ex As Exception - ' Send Email to Digital Data - Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex) - Dim oEmailData As New EmailData With { - .From = pArgs.ExceptionEmailAddress, - .Subject = $"FileMoveException im ZUGFeRD-Parser @ {oMessageId}" - } - _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileMoveException", _EmailOutAccountId, pArgs.NamePortal) - - _logger.Warn("Could not move files!") - _logger.Error(ex) - Throw ex - End Try - - Try - ' If everything went OK or an expected error occurred, - ' finally commit all changes To the Database - ' ================================================================== - If oIsSuccess Or oExpectedError Then - ' Commit Transaction - oConnections.SQLServerTransaction.Commit() - End If - - _logger.Info("FileGroup for MessageId [{0}] successfully processed!", oMessageId) - Catch ex As Exception - _logger.Error(ex) - _logger.Warn("Database Transactions were not committed successfully.") - End Try - - Try - oConnections.SQLServerConnection.Close() - Catch ex As Exception - _logger.Error(ex) - _logger.Warn("Database Connections were not closed successfully.") - End Try - End Try - End Function - - Private Function ProcessFile(pMessageId As String, oFile As FileInfo, oConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult - ' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the - ' different versions of ZUGFeRD and the type is unknown at compile-time. - ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename - ' of the extracted xml file. - ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save - ' the new meta data, ie. the type of schema (zugferd version) + Private Function ProcessFile(pMessageId As String, pEmailData As EmailData, pZugferdFiles As Integer, oFile As FileInfo, oConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult Dim oDocument As ZUGFeRDInterface.ZugferdResult - - Dim oMissingProperties = New List(Of String) - Dim oEmailAttachmentFiles = New List(Of FileInfo) - Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile) - Dim oZugferdFiles As Integer = 0 - - ' Start a global group counter for each file - Dim oGlobalGroupCounter = 0 - ' Clear missing properties for the new file - oMissingProperties = New List(Of String) + Dim oResult As New ProcessFileResult() + Dim oMissingProperties As New List(Of String) ' Only pdf files are allowed from here on If Not oFile.Name.ToUpper.EndsWith(".PDF") Then _logger.Debug("Skipping non-pdf file {0}", oFile.Name) - oEmailAttachmentFiles.Add(oFile) + oResult.EmailAttachmentFiles.Add(oFile) ' Checking filesize for attachment files If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then @@ -928,8 +542,7 @@ Public Class ImportZUGFeRDFiles Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) End If - 'TODO: how to handle? - 'Continue For + Return oResult End If _logger.Info("Start processing file {0}", oFile.Name) @@ -950,10 +563,9 @@ Public Class ImportZUGFeRDFiles Select Case ex.ErrorType Case ZUGFeRDInterface.ErrorType.NoZugferd _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name) - oEmailAttachmentFiles.Add(oFile) - 'TODO: How to handle? - 'Continue For + oResult.EmailAttachmentFiles.Add(oFile) + Return oResult Case ZUGFeRDInterface.ErrorType.UnsupportedFormat _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile) @@ -970,12 +582,12 @@ Public Class ImportZUGFeRDFiles End Try ' Check if there are more than one ZUGFeRD files - If oZugferdFiles = 1 Then + If pZugferdFiles = 1 Then Throw New TooMuchFerdsException() End If ' Since extraction went well, increase the amount of ZUGFeRD files - oZugferdFiles += 1 + pZugferdFiles += 1 ' Extract all attachments with the extensions specified in `AllowedExtensions`. ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself. @@ -984,11 +596,11 @@ Public Class ImportZUGFeRDFiles If oAttachments Is Nothing Then _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName) Else - oEmbeddedAttachmentFiles.AddRange(oAttachments) + oResult.EmbeddedAttachmentFiles.AddRange(oAttachments) End If ' Check the Checksum and rejection status - Dim oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, pMessageId, pArgs.IgnoreRejectionStatus) + Dim oMD5Checksum = GenerateAndCheck_MD5Sum(oFile, pMessageId, pArgs.IgnoreRejectionStatus) ' Check the document against the configured property map and return: ' - a List of valid properties @@ -1002,7 +614,7 @@ Public Class ImportZUGFeRDFiles If oCheckResult.MissingProperties.Count > 0 Then _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count) oMissingProperties = oCheckResult.MissingProperties - Throw New MissingValueException(oFile) + Throw New MissingValueException(oFile, oCheckResult.MissingProperties) Else _logger.Debug("No missing properties found. Continuing.") @@ -1027,9 +639,14 @@ Public Class ImportZUGFeRDFiles InsertPropertyValue(pMessageId, oConnections, oProperty) Next - Return New ProcessFileResult With { - .ZugferdFileCount = oZugferdFiles - } + _logger.Debug("File processed.") + + oResult.ZugferdFileFound = True + oResult.MD5Checksum = oMD5Checksum + oResult.ZugferdFileCount = pZugferdFiles + + Return oResult + End Function #End Region @@ -1057,9 +674,9 @@ Public Class ImportZUGFeRDFiles Dim oCommand = $"INSERT INTO {pProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER, SPEC_NAME, IS_REQUIRED) VALUES ('{pMessageId}', '{pProperty.Description}', '{pProperty.Value.Replace("'", "''")}', {oGroupCounterValue},'{pProperty.TableColumn}','{pProperty.IsRequired}')" - _logger.Debug("Mapping Property [{0}] with value [{1}], Will be inserted into table [{2}]", pProperty.TableColumn, pProperty.Value.Replace("'", "''"), pProperty.TableName) - ' Insert into SQL Server + _logger.Debug("Mapping Property [{0}] with value [{1}], Will be inserted into table [{2}]", pProperty.TableColumn, pProperty.Value.Replace("'", "''"), pProperty.TableName) + ' Insert into SQL Server Dim oResult = _mssql.ExecuteNonQueryWithConnectionObject(oCommand, pConnections.SQLServerConnection, MSSQLServer.TransactionMode.ExternalTransaction, pConnections.SQLServerTransaction) If oResult = False Then _logger.Warn($"SQL Command [{oCommand}] was not successful. Check the log.")