AppStd/Modules
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor: Processfiles

Browse Source
This commit is contained in:
Jonathan Jenne 2023-10-09 16:05:50 +02:00
parent 3c149a32e9
commit c187bdbe5e
2 changed files with 141 additions and 522 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Jobs/Exceptions.vb
View File

@ -7,11 +7,13 @@ Public Class Exceptions
Inherits ApplicationException Inherits ApplicationException
Public ReadOnly File As FileInfo Public ReadOnly File As FileInfo
Public ReadOnly MissingProperties As List(Of String)
Public Sub New(File As FileInfo) Public Sub New(pFile As FileInfo, pMissingProperties As List(Of String))
MyBase.New($"Missing values in [{File.Name}]") MyBase.New($"Missing values in [{pFile.Name}]")
Me.File = File Me.File = pFile
Me.MissingProperties = pMissingProperties
End Sub End Sub
End Class End Class

655
Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb
View File

@ -45,9 +45,13 @@ Public Class ImportZUGFeRDFiles
Private _EmailOutAccountId As Integer Private _EmailOutAccountId As Integer
Private Class ProcessFileResult Private Class ProcessFileResult
Public ZugferdFileCount As Integer Public ZugferdFileFound As Boolean = False
Public MD5Hash As String = Nothing
Public ZugferdFileCount As Integer = 0
Public MD5Checksum As String = Nothing
Public EmailAttachmentFiles As New List(Of FileInfo)
Public EmbeddedAttachmentFiles As New List(Of PDFEmbeds.EmbeddedFile)
End Class End Class
Private Class DatabaseConnections Private Class DatabaseConnections
@ -152,147 +156,147 @@ Public Class ImportZUGFeRDFiles
Dim oEmbeddedAttachmentFiles As New List(Of PDFEmbeds.EmbeddedFile) Dim oEmbeddedAttachmentFiles As New List(Of PDFEmbeds.EmbeddedFile)
Dim oMessageId As String = oFileGroup.Key Dim oMessageId As String = oFileGroup.Key
Dim oMissingProperties As New List(Of String)
Dim oMD5CheckSum As String = String.Empty Dim oMD5CheckSum As String = String.Empty
_logger.Info("Start processing file group {0}", oMessageId) _logger.Info("Start processing file group {0}", oMessageId)
' TODO: Use this refactored function Dim oEmailDataBase = _email.GetEmailDataForMessageId(oMessageId)
' ProcessFileGroup(oMessageId, oFileGroupFiles, oConnections, oArgs)
Try Try
For Each oFile In oFileGroupFiles For Each oFile In oFileGroupFiles
' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the Dim oResult As ProcessFileResult = ProcessFile(oMessageId, oEmailDataBase, oZUGFeRDCount, oFile, oConnections, oArgs)
' different versions of ZUGFeRD and the type is unknown at compile-time.
' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
' of the extracted xml file.
' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
' the new meta data, ie. the type of schema (zugferd version)
Dim oDocument As ZUGFeRDInterface.ZugferdResult
' Start a global group counter for each file If oResult.ZugferdFileFound = False Then
Dim oGlobalGroupCounter = 0 _logger.Debug("Zugferd File found")
' Clear missing properties for the new file oMD5CheckSum = oResult.MD5Checksum
oMissingProperties = New List(Of String) oZUGFeRDCount = oResult.ZugferdFileCount
oEmailAttachmentFiles.AddRange(oResult.EmailAttachmentFiles)
' Only pdf files are allowed from here on oEmbeddedAttachmentFiles.AddRange(oResult.EmbeddedAttachmentFiles)
If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
_logger.Debug("Skipping non-pdf file {0}", oFile.Name)
oEmailAttachmentFiles.Add(oFile)
' Checking filesize for attachment files
If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then
_logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
End If
Continue For
End If End If
' ------------------------------------------------------------
#Region "REFACTOR"
_logger.Info("Start processing file {0}", oFile.Name)
' Checking filesize for pdf files 'Dim oDocument As ZUGFeRDInterface.ZugferdResult
If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then
_logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
End If
Try '' Only pdf files are allowed from here on
oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName) 'If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
' _logger.Debug("Skipping non-pdf file {0}", oFile.Name)
' oEmailAttachmentFiles.Add(oFile)
Catch ex As ValidationException ' ' Checking filesize for attachment files
Throw ex ' If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then
' _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
' Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
' End If
Catch ex As ZUGFeRDExecption ' Continue For
Select Case ex.ErrorType 'End If
Case ZUGFeRDInterface.ErrorType.NoZugferd
_logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
oEmailAttachmentFiles.Add(oFile)
Continue For
Case ZUGFeRDInterface.ErrorType.UnsupportedFormat '_logger.Info("Start processing file {0}", oFile.Name)
_logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
Throw New UnsupportedFerdException(ex.XmlFile)
Case ZUGFeRDInterface.ErrorType.NoValidZugferd '' Checking filesize for pdf files
_logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name) 'If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, oArgs.MaxAttachmentSizeInMegaBytes) = False Then
Throw New InvalidFerdException() ' _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
' Throw New FileSizeLimitReachedException(oFile.Name, oArgs.MaxAttachmentSizeInMegaBytes)
'End If
Case Else 'Try
_logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name) ' oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName)
Throw ex
End Select
End Try
' Check if there are more than one ZUGFeRD files 'Catch ex As ValidationException
If oZUGFeRDCount = 1 Then ' Throw ex
Throw New TooMuchFerdsException()
End If
' Since extraction went well, increase the amount of ZUGFeRD files 'Catch ex As ZUGFeRDExecption
oZUGFeRDCount += 1 ' Select Case ex.ErrorType
' Case ZUGFeRDInterface.ErrorType.NoZugferd
' _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
' oEmailAttachmentFiles.Add(oFile)
' Continue For
' Extract all attachments with the extensions specified in `AllowedExtensions`. ' Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself. ' _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`. ' Throw New UnsupportedFerdException(ex.XmlFile)
Dim oAttachments = oAttachmentExtractor.Extract(oFile.FullName, AllowedExtensions)
If oAttachments Is Nothing Then
_logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
Else
oEmbeddedAttachmentFiles.AddRange(oAttachments)
End If
' Check the Checksum and rejection status ' Case ZUGFeRDInterface.ErrorType.NoValidZugferd
oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, oArgs.IgnoreRejectionStatus) ' _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name)
' Throw New InvalidFerdException()
' Check the document against the configured property map and return: ' Case Else
' - a List of valid properties ' _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name)
' - a List of missing properties ' Throw ex
' End Select
'End Try
Dim oPropertyMap = _zugferd.FilterPropertyMap(oArgs.PropertyMap, oDocument.Specification) '' Check if there are more than one ZUGFeRD files
Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId) 'If oZUGFeRDCount = 1 Then
' Throw New TooMuchFerdsException()
'End If
_logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count) '' Since extraction went well, increase the amount of ZUGFeRD files
'oZUGFeRDCount += 1
If oCheckResult.MissingProperties.Count > 0 Then '' Extract all attachments with the extensions specified in `AllowedExtensions`.
_logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count) '' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
oMissingProperties = oCheckResult.MissingProperties '' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`.
Throw New MissingValueException(oFile) 'Dim oAttachments = oAttachmentExtractor.Extract(oFile.FullName, AllowedExtensions)
Else 'If oAttachments Is Nothing Then
_logger.Debug("No missing properties found. Continuing.") ' _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
'Else
' oEmbeddedAttachmentFiles.AddRange(oAttachments)
'End If
End If '' Check the Checksum and rejection status
'oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, oArgs.IgnoreRejectionStatus)
DeleteExistingPropertyValues(oMessageId, oConnections) '' Check the document against the configured property map and return:
'' - a List of valid properties
'' - a List of missing properties
Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault() 'Dim oPropertyMap = _zugferd.FilterPropertyMap(oArgs.PropertyMap, oDocument.Specification)
If oFirstProperty IsNot Nothing Then 'Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId)
InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With {
.MessageId = oMessageId,
.Description = "ZUGFeRDSpezifikation",
.GroupCounter = 0,
.IsRequired = False,
.Value = oDocument.Specification,
.TableName = oFirstProperty.TableName,
.TableColumn = "ZUGFERD_SPECIFICATION"
})
End If
For Each oProperty In oCheckResult.ValidProperties '_logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)
InsertPropertyValue(oMessageId, oConnections, oProperty)
Next 'If oCheckResult.MissingProperties.Count > 0 Then
' _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
' Throw New MissingValueException(oFile, oCheckResult.MissingProperties)
'Else
' _logger.Debug("No missing properties found. Continuing.")
'End If
'DeleteExistingPropertyValues(oMessageId, oConnections)
'Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault()
'If oFirstProperty IsNot Nothing Then
' InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With {
' .MessageId = oMessageId,
' .Description = "ZUGFeRDSpezifikation",
' .GroupCounter = 0,
' .IsRequired = False,
' .Value = oDocument.Specification,
' .TableName = oFirstProperty.TableName,
' .TableColumn = "ZUGFERD_SPECIFICATION"
' })
'End If
'For Each oProperty In oCheckResult.ValidProperties
' InsertPropertyValue(oMessageId, oConnections, oProperty)
'Next
#End Region
Next Next
'Check if there are no ZUGFeRD files 'Check if there are no ZUGFeRD files
If oZUGFeRDCount = 0 Then If oZUGFeRDCount = 0 Then
' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated ' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated
' This is the default/initial behaviour. ' This is the default/initial behaviour.
If oArgs.NonZugferdDirectory Is Nothing OrElse oArgs.NonZugferdDirectory = String.Empty Then If String.IsNullOrEmpty(oArgs.NonZugferdDirectory) Then
Throw New NoFerdsException() Throw New NoFerdsException()
End If End If
' Also, if the directory is set but does not exist, still a rejection will be generated. ' Also, if the directory is set but does not exist, still a rejection will be generated.
If Not IO.Directory.Exists(oArgs.NonZugferdDirectory) Then If Not Directory.Exists(oArgs.NonZugferdDirectory) Then
Throw New NoFerdsException() Throw New NoFerdsException()
End If End If
@ -302,12 +306,11 @@ Public Class ImportZUGFeRDFiles
End If End If
'If no errors occurred... 'If no errors occurred, update the history table for this MessageId
'Log the History If String.IsNullOrEmpty(oMD5CheckSum) Then
If oMD5CheckSum <> String.Empty Then
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS")
Else
_history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)") _history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)")
Else
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS")
End If End If
oIsSuccess = True oIsSuccess = True
@ -395,13 +398,13 @@ Public Class ImportZUGFeRDFiles
_logger.Error(ex) _logger.Error(ex)
Dim oMessage As String = "" Dim oMessage As String = ""
For Each prop In oMissingProperties For Each prop In ex.MissingProperties
oMessage &= $"- {prop}" oMessage &= $"- {prop}"
Next Next
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]") _history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]")
Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties) Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, ex.MissingProperties)
Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId) Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, oArgs.NamePortal) _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, oArgs.NamePortal)
AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oSQLTransaction) AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oSQLTransaction)
@ -523,404 +526,15 @@ Public Class ImportZUGFeRDFiles
End Sub End Sub
#Region "=== REFACTOR" #Region "=== REFACTOR"
Private Function ProcessFileGroup(oMessageId As String, pFiles As List(Of FileInfo), oConnections As DatabaseConnections, pArgs As WorkerArgs) Private Function ProcessFile(pMessageId As String, pEmailData As EmailData, pZugferdFiles As Integer, oFile As FileInfo, oConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult
Dim oMissingProperties = New List(Of String)
Dim oEmailAttachmentFiles = New List(Of FileInfo)
Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile)
Dim oZUGFeRDCount = 0
Dim oMD5CheckSum As String = Nothing
' Set the default Move Directory
Dim oMoveDirectory As String = pArgs.ErrorDirectory
' Flag to save if the whole process was a success.
' Will be set only at the end of the function if no error occurred.
Dim oIsSuccess As Boolean = False
' Flag to save if the occurred error (if any) was expected
' Used to determine if transactions should be committed or not
Dim oExpectedError As Boolean = True
Try
For Each oFile In pFiles
' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the
' different versions of ZUGFeRD and the type is unknown at compile-time.
' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
' of the extracted xml file.
' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
' the new meta data, ie. the type of schema (zugferd version)
Dim oDocument As ZUGFeRDInterface.ZugferdResult
' Start a global group counter for each file
Dim oGlobalGroupCounter = 0
' Clear missing properties for the new file
oMissingProperties = New List(Of String)
' Only pdf files are allowed from here on
If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
_logger.Debug("Skipping non-pdf file {0}", oFile.Name)
oEmailAttachmentFiles.Add(oFile)
' Checking filesize for attachment files
If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
_logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
End If
Continue For
End If
_logger.Info("Start processing file {0}", oFile.Name)
' Checking filesize for pdf files
If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
_logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
End If
Try
oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName)
Catch ex As ValidationException
Throw ex
Catch ex As ZUGFeRDExecption
Select Case ex.ErrorType
Case ZUGFeRDInterface.ErrorType.NoZugferd
_logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
oEmailAttachmentFiles.Add(oFile)
Continue For
Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
_logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
Throw New UnsupportedFerdException(ex.XmlFile)
Case ZUGFeRDInterface.ErrorType.NoValidZugferd
_logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name)
Throw New InvalidFerdException()
Case Else
_logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name)
Throw ex
End Select
End Try
' Check if there are more than one ZUGFeRD files
If oZUGFeRDCount = 1 Then
Throw New TooMuchFerdsException()
End If
' Since extraction went well, increase the amount of ZUGFeRD files
oZUGFeRDCount += 1
' Extract all attachments with the extensions specified in `AllowedExtensions`.
' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`.
Dim oAttachments = _embeds.Extract(oFile.FullName, AllowedExtensions)
If oAttachments Is Nothing Then
_logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
Else
oEmbeddedAttachmentFiles.AddRange(oAttachments)
End If
' Check the Checksum and rejection status
oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, oMessageId, pArgs.IgnoreRejectionStatus)
' Check the document against the configured property map and return:
' - a List of valid properties
' - a List of missing properties
Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, oDocument.Specification)
Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId)
_logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)
If oCheckResult.MissingProperties.Count > 0 Then
_logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
oMissingProperties = oCheckResult.MissingProperties
Throw New MissingValueException(oFile)
Else
_logger.Debug("No missing properties found. Continuing.")
End If
DeleteExistingPropertyValues(oMessageId, oConnections)
Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault()
If oFirstProperty IsNot Nothing Then
InsertPropertyValue(oMessageId, oConnections, New PropertyValues.ValidProperty() With {
.MessageId = oMessageId,
.Description = "ZUGFeRDSpezifikation",
.GroupCounter = 0,
.IsRequired = False,
.Value = oDocument.Specification,
.TableName = oFirstProperty.TableName,
.TableColumn = "ZUGFERD_SPECIFICATION"
})
End If
For Each oProperty In oCheckResult.ValidProperties
InsertPropertyValue(oMessageId, oConnections, oProperty)
Next
Next
'Check if there are no ZUGFeRD files
If oZUGFeRDCount = 0 Then
' If NonZugferdDirectory is not set, a NoFerdsException will be thrown and a rejection will be generated
' This is the default/initial behaviour.
If pArgs.NonZugferdDirectory Is Nothing OrElse pArgs.NonZugferdDirectory = String.Empty Then
Throw New NoFerdsException()
End If
' Also, if the directory is set but does not exist, still a rejection will be generated.
If Not IO.Directory.Exists(pArgs.NonZugferdDirectory) Then
Throw New NoFerdsException()
End If
' Only if the directory is set and does exist, it will be used and any file groups which
' do NOT CONTAIN ANY ZUGFERD DOCUMENTS, are moved to that directory.
Throw New NoFerdsAlternateException()
End If
'If no errors occurred...
'Log the History
If oMD5CheckSum <> String.Empty Then
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "SUCCESS")
Else
_history.Update_HistoryEntry(oMessageId, String.Empty, "SUCCESS (with empty MD5Hash)")
End If
oIsSuccess = True
oMoveDirectory = pArgs.SuccessDirectory
Catch ex As ValidationException
_logger.Error(ex)
Dim oErrors = ex.ValidationErrors
Dim oMessage = "REJECTED - ZUGFeRD yes but formal validation failed!"
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, oMessage)
Dim oErrorList As String = ""
For Each oError In oErrors
oErrorList += $"<li>Element '{oError.ElementName}' mit Wert '{oError.ElementValue}': {oError.ErrorMessage}</li>"
Next
Dim oBody = String.Format(EmailStrings.EMAIL_VALIDATION_ERROR, oErrorList)
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "ValidationException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "ValidationException", "Die Rechnungsvalidierung ist fehlgeschlagen!", "", oConnections.SQLServerTransaction)
Catch ex As MD5HashException
_logger.Error(ex)
' When MD5HashException is thrown, we don't have a MD5Hash yet.
' That 's why we set it to String.Empty here.
Dim oMessage = "REJECTED - Already processed (MD5Hash)"
_history.Update_HistoryEntry(oMessageId, String.Empty, oMessage)
Dim oBody = EmailStrings.EMAIL_MD5_ERROR
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MD5HashException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "", oConnections.SQLServerTransaction)
Catch ex As UnsupportedFerdException
_logger.Error(ex)
' When UnsupportedFerdException is thrown, we don't have a MD5Hash yet.
' That 's why we set it to String.Empty here.
_history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format")
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, oEmailData.Subject, ex.XmlFile)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnsupportedFerdException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "UnsupportedFerdException", "Nicht unterstütztes Datenformat", "", oConnections.SQLServerTransaction)
Catch ex As InvalidFerdException
_logger.Error(ex)
' When InvalidFerdException is thrown, we don't have a MD5Hash yet.
' That 's why we set it to String.Empty here.
_history.Update_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format")
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
Dim oBody = String.Format(EmailStrings.EMAIL_INVALID_DOCUMENT, oEmailData.Subject)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "InvalidFerdException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "InvalidFerdException", "Inkorrekte Formate", "", oConnections.SQLServerTransaction)
Catch ex As TooMuchFerdsException
_logger.Error(ex)
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email")
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
Dim oBody = String.Format(EmailStrings.EMAIL_TOO_MUCH_FERDS, oEmailData.Subject)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "TooMuchFerdsException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "", oConnections.SQLServerTransaction)
Catch ex As NoFerdsException
_logger.Error(ex)
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email")
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
Dim oBody = String.Format(EmailStrings.EMAIL_NO_FERDS, oEmailData.Subject)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "NoFerdsException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "", oConnections.SQLServerTransaction)
Catch ex As MissingValueException
_logger.Error(ex)
Dim oMessage As String = ""
For Each prop In oMissingProperties
oMessage &= $"- {prop}"
Next
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, $"REJECTED - Missing Required Properties: [{oMessage}]")
Dim oBody = _email.CreateBodyForMissingProperties(ex.File.Name, oMissingProperties)
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage, oConnections.SQLServerTransaction)
Catch ex As FileSizeLimitReachedException
_logger.Error(ex)
_history.Update_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - File size limit reached")
Dim oEmailData = MoveAndRenameEmailToRejected(pArgs, oMessageId)
Dim oKey = FileSizeLimitReachedException.KEY_FILENAME
Dim oFileExceedingThreshold As String = IIf(ex.Data.Contains(oKey), ex.Data.Item(oKey), "")
Dim oFileWithoutMessageId = oFileExceedingThreshold.
Replace(oMessageId, "").
Replace("~", "")
Dim oBody = String.Format(EmailStrings.EMAIL_FILE_SIZE_REACHED, pArgs.MaxAttachmentSizeInMegaBytes, oFileWithoutMessageId)
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileSizeLimitReachedException", _EmailOutAccountId, pArgs.NamePortal)
AddRejectedState(oMessageId, "FileSizeLimitReachedException", "Erlaubte Dateigröße überschritten", "", oConnections.SQLServerTransaction)
Catch ex As NoFerdsAlternateException
' TODO: Maybe dont even log this 'error', since it's not really an error and it might happen *A LOT*
_logger.Error(ex)
oMoveDirectory = pArgs.NonZugferdDirectory
Catch ex As OutOfMemoryException
_logger.Warn("OutOfMemory Error occurred: {0}", ex.Message)
_logger.Error(ex)
' Send Email to Digital Data
Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
Dim oEmailData As New EmailData With {
.From = pArgs.ExceptionEmailAddress,
.Subject = $"OutOfMemoryException im ZUGFeRD-Parser @ {oMessageId}"
}
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "OutOfMemoryException", _EmailOutAccountId, pArgs.NamePortal)
' Rollback Transaction
oConnections.SQLServerTransaction.Rollback()
oMoveDirectory = DIRECTORY_DONT_MOVE
oExpectedError = False
Catch ex As Exception
_logger.Warn("Unknown Error occurred: {0}", ex.Message)
_logger.Error(ex)
' Send Email to Digital Data
Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
Dim oEmailData As New EmailData With {
.From = pArgs.ExceptionEmailAddress,
.Subject = $"UnhandledException im ZUGFeRD-Parser @ {oMessageId}"
}
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnhandledException", _EmailOutAccountId, pArgs.NamePortal)
' Rollback Transaction
oConnections.SQLServerTransaction.Rollback()
oMoveDirectory = DIRECTORY_DONT_MOVE
oExpectedError = False
Finally
Try
' If an application error occurred, dont move files so they will be processed again later
If oMoveDirectory = DIRECTORY_DONT_MOVE Then
_logger.Info("Application Error occurred. Files for message Id {0} will not be moved.", oMessageId)
Else
' Move all files of the current group
_file.MoveFiles(pArgs, oMessageId, pFiles, oEmailAttachmentFiles, oEmbeddedAttachmentFiles, oMoveDirectory, oIsSuccess)
End If
_logger.Info("Finished processing file group {0}", oMessageId)
Catch ex As Exception
' Send Email to Digital Data
Dim oBody = _email.CreateBodyForUnhandledException(oMessageId, ex)
Dim oEmailData As New EmailData With {
.From = pArgs.ExceptionEmailAddress,
.Subject = $"FileMoveException im ZUGFeRD-Parser @ {oMessageId}"
}
_email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "FileMoveException", _EmailOutAccountId, pArgs.NamePortal)
_logger.Warn("Could not move files!")
_logger.Error(ex)
Throw ex
End Try
Try
' If everything went OK or an expected error occurred,
' finally commit all changes To the Database
' ==================================================================
If oIsSuccess Or oExpectedError Then
' Commit Transaction
oConnections.SQLServerTransaction.Commit()
End If
_logger.Info("FileGroup for MessageId [{0}] successfully processed!", oMessageId)
Catch ex As Exception
_logger.Error(ex)
_logger.Warn("Database Transactions were not committed successfully.")
End Try
Try
oConnections.SQLServerConnection.Close()
Catch ex As Exception
_logger.Error(ex)
_logger.Warn("Database Connections were not closed successfully.")
End Try
End Try
End Function
Private Function ProcessFile(pMessageId As String, oFile As FileInfo, oConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult
' 09.12.2021: oDocument is now an Object, because have different classes corresponding to the
' different versions of ZUGFeRD and the type is unknown at compile-time.
' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
' of the extracted xml file.
' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
' the new meta data, ie. the type of schema (zugferd version)
Dim oDocument As ZUGFeRDInterface.ZugferdResult Dim oDocument As ZUGFeRDInterface.ZugferdResult
Dim oResult As New ProcessFileResult()
Dim oMissingProperties = New List(Of String) Dim oMissingProperties As New List(Of String)
Dim oEmailAttachmentFiles = New List(Of FileInfo)
Dim oEmbeddedAttachmentFiles = New List(Of PDFEmbeds.EmbeddedFile)
Dim oZugferdFiles As Integer = 0
' Start a global group counter for each file
Dim oGlobalGroupCounter = 0
' Clear missing properties for the new file
oMissingProperties = New List(Of String)
' Only pdf files are allowed from here on ' Only pdf files are allowed from here on
If Not oFile.Name.ToUpper.EndsWith(".PDF") Then If Not oFile.Name.ToUpper.EndsWith(".PDF") Then
_logger.Debug("Skipping non-pdf file {0}", oFile.Name) _logger.Debug("Skipping non-pdf file {0}", oFile.Name)
oEmailAttachmentFiles.Add(oFile) oResult.EmailAttachmentFiles.Add(oFile)
' Checking filesize for attachment files ' Checking filesize for attachment files
If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then
@ -928,8 +542,7 @@ Public Class ImportZUGFeRDFiles
Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes)
End If End If
'TODO: how to handle? Return oResult
'Continue For
End If End If
_logger.Info("Start processing file {0}", oFile.Name) _logger.Info("Start processing file {0}", oFile.Name)
@ -950,10 +563,9 @@ Public Class ImportZUGFeRDFiles
Select Case ex.ErrorType Select Case ex.ErrorType
Case ZUGFeRDInterface.ErrorType.NoZugferd Case ZUGFeRDInterface.ErrorType.NoZugferd
_logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name) _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name)
oEmailAttachmentFiles.Add(oFile)
'TODO: How to handle? oResult.EmailAttachmentFiles.Add(oFile)
'Continue For Return oResult
Case ZUGFeRDInterface.ErrorType.UnsupportedFormat Case ZUGFeRDInterface.ErrorType.UnsupportedFormat
_logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile) _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile)
@ -970,12 +582,12 @@ Public Class ImportZUGFeRDFiles
End Try End Try
' Check if there are more than one ZUGFeRD files ' Check if there are more than one ZUGFeRD files
If oZugferdFiles = 1 Then If pZugferdFiles = 1 Then
Throw New TooMuchFerdsException() Throw New TooMuchFerdsException()
End If End If
' Since extraction went well, increase the amount of ZUGFeRD files ' Since extraction went well, increase the amount of ZUGFeRD files
oZugferdFiles += 1 pZugferdFiles += 1
' Extract all attachments with the extensions specified in `AllowedExtensions`. ' Extract all attachments with the extensions specified in `AllowedExtensions`.
' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself. ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself.
@ -984,11 +596,11 @@ Public Class ImportZUGFeRDFiles
If oAttachments Is Nothing Then If oAttachments Is Nothing Then
_logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName) _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName)
Else Else
oEmbeddedAttachmentFiles.AddRange(oAttachments) oResult.EmbeddedAttachmentFiles.AddRange(oAttachments)
End If End If
' Check the Checksum and rejection status ' Check the Checksum and rejection status
Dim oMD5CheckSum = GenerateAndCheck_MD5Sum(oFile, pMessageId, pArgs.IgnoreRejectionStatus) Dim oMD5Checksum = GenerateAndCheck_MD5Sum(oFile, pMessageId, pArgs.IgnoreRejectionStatus)
' Check the document against the configured property map and return: ' Check the document against the configured property map and return:
' - a List of valid properties ' - a List of valid properties
@ -1002,7 +614,7 @@ Public Class ImportZUGFeRDFiles
If oCheckResult.MissingProperties.Count > 0 Then If oCheckResult.MissingProperties.Count > 0 Then
_logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count) _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count)
oMissingProperties = oCheckResult.MissingProperties oMissingProperties = oCheckResult.MissingProperties
Throw New MissingValueException(oFile) Throw New MissingValueException(oFile, oCheckResult.MissingProperties)
Else Else
_logger.Debug("No missing properties found. Continuing.") _logger.Debug("No missing properties found. Continuing.")
@ -1027,9 +639,14 @@ Public Class ImportZUGFeRDFiles
InsertPropertyValue(pMessageId, oConnections, oProperty) InsertPropertyValue(pMessageId, oConnections, oProperty)
Next Next
Return New ProcessFileResult With { _logger.Debug("File processed.")
.ZugferdFileCount = oZugferdFiles
} oResult.ZugferdFileFound = True
oResult.MD5Checksum = oMD5Checksum
oResult.ZugferdFileCount = pZugferdFiles
Return oResult
End Function End Function
#End Region #End Region
@ -1057,9 +674,9 @@ Public Class ImportZUGFeRDFiles
Dim oCommand = $"INSERT INTO {pProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER, SPEC_NAME, IS_REQUIRED) VALUES Dim oCommand = $"INSERT INTO {pProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER, SPEC_NAME, IS_REQUIRED) VALUES
('{pMessageId}', '{pProperty.Description}', '{pProperty.Value.Replace("'", "''")}', {oGroupCounterValue},'{pProperty.TableColumn}','{pProperty.IsRequired}')" ('{pMessageId}', '{pProperty.Description}', '{pProperty.Value.Replace("'", "''")}', {oGroupCounterValue},'{pProperty.TableColumn}','{pProperty.IsRequired}')"
_logger.Debug("Mapping Property [{0}] with value [{1}], Will be inserted into table [{2}]", pProperty.TableColumn, pProperty.Value.Replace("'", "''"), pProperty.TableName) _logger.Debug("Mapping Property [{0}] with value [{1}], Will be inserted into table [{2}]", pProperty.TableColumn, pProperty.Value.Replace("'", "''"), pProperty.TableName)
' Insert into SQL Server
' Insert into SQL Server
Dim oResult = _mssql.ExecuteNonQueryWithConnectionObject(oCommand, pConnections.SQLServerConnection, MSSQLServer.TransactionMode.ExternalTransaction, pConnections.SQLServerTransaction) Dim oResult = _mssql.ExecuteNonQueryWithConnectionObject(oCommand, pConnections.SQLServerConnection, MSSQLServer.TransactionMode.ExternalTransaction, pConnections.SQLServerTransaction)
If oResult = False Then If oResult = False Then
_logger.Warn($"SQL Command [{oCommand}] was not successful. Check the log.") _logger.Warn($"SQL Command [{oCommand}] was not successful. Check the log.")