From 3e2606a582dace99cd3214146a66e4dd8aee1605 Mon Sep 17 00:00:00 2001 From: pitzm Date: Tue, 28 Jan 2025 14:21:01 +0100 Subject: [PATCH] Modules.Interfaces & Modules.Jobs: Verarbeitung von XML-Belegen im ZUGFeRD Service implementiert --- Interfaces/ZUGFeRDInterface.vb | 40 ++++++++ Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb | 157 ++++++++++++++++++++++------- 2 files changed, 158 insertions(+), 39 deletions(-) diff --git a/Interfaces/ZUGFeRDInterface.vb b/Interfaces/ZUGFeRDInterface.vb index 26242467..7da0e288 100644 --- a/Interfaces/ZUGFeRDInterface.vb +++ b/Interfaces/ZUGFeRDInterface.vb @@ -1,7 +1,9 @@ Imports System.IO +Imports System.Reflection Imports System.Xml Imports System.Xml.Serialization Imports DigitalData.Modules.Interfaces.Exceptions +Imports DigitalData.Modules.Interfaces.PDFEmbeds Imports DigitalData.Modules.Interfaces.Peppol Imports DigitalData.Modules.Interfaces.ZUGFeRD Imports DigitalData.Modules.Logging @@ -138,6 +140,44 @@ Public Class ZUGFeRDInterface End If End Function + Public Function GetSerializedXMLContentFromFile(oFileInfo As FileInfo) As ZugferdResult + Dim oResult = New ZugferdResult() + + Try + Dim oFileSize As Integer = oFileInfo.Length + Dim oFileData As Byte() = File.ReadAllBytes(oFileInfo.FullName) + + Dim oXmlFile As EmbeddedFile = New EmbeddedFile() With { + .FileName = oFileInfo.Name, + .FileContents = oFileData + } + + Using oStream As New MemoryStream(oXmlFile.FileContents) + oResult = New ZugferdResult With { + .DataFileName = oXmlFile.FileName, + .XElementObject = XElement.Load(oStream) + } + End Using + + Catch ex As ZUGFeRDExecption + ' Don't log ZUGFeRD Exceptions here, they should be handled by the calling code. + ' It also produces misleading error messages when checking if an attachment is a zugferd file. + Throw ex + + Catch ex As Exception + _logger.Error(ex) + Throw New ZUGFeRDExecption(ErrorType.NoValidZugferd, "Datei ist eine ungültige XML Datei.") + End Try + + If oResult.ValidationErrors.Any() Then + Throw New ValidationException() With { + .ValidationErrors = oResult.ValidationErrors + } + End If + + Return SerializeZUGFeRDDocument(oResult) + End Function + ''' ''' Validates a ZUGFeRD File and extracts the XML Document from it ''' diff --git a/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb b/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb index 9c74d35f..3a8f0d35 100644 --- a/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb +++ b/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb @@ -166,10 +166,18 @@ Public Class ImportZUGFeRDFiles End If Dim oEmailDataBase = _email.GetEmailDataForMessageId(oMessageId) + Dim oFileCounter As Integer = 0 Try For Each oFile In oFileGroupFiles - Dim oResult As ProcessFileResult = ProcessFile(oMessageId, oEmailDataBase, oZUGFeRDCount, oFile, oConnections, oArgs) + oFileCounter += 1 + Dim oResult As ProcessFileResult + + If oFileCounter = 1 AndAlso oFile.Name.ToUpper.EndsWith(".XML") Then + oResult = ProcessXMLFile(oMessageId, oZUGFeRDCount, oFile, oConnections, oArgs) + Else + oResult = ProcessFile(oMessageId, oZUGFeRDCount, oFile, oConnections, oArgs) + End If If oResult.ZugferdFileFound = True Then _logger.Debug("Zugferd File found") @@ -455,34 +463,96 @@ Public Class ImportZUGFeRDFiles Return oRejectionCodeString End Function - Private Function ProcessFile(pMessageId As String, pEmailData As EmailData, pZugferdFiles As Integer, oFile As FileInfo, oConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult + Private Function ProcessXMLFile(pMessageId As String, pZugferdFileCounter As Integer, pFile As FileInfo, pConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult + Dim oDocument As ZUGFeRDInterface.ZugferdResult + Dim oResult As New ProcessFileResult() + + If pFile.Extension.Equals(".xml", StringComparison.OrdinalIgnoreCase) = False Then + ' Diese Methode ist nur für den xml-Beleg gedacht + Return oResult + End If + + _logger.Info("Start xml processing file {0}", pFile.Name) + + ' Checking filesize + If _filesystem.TestFileSizeIsLessThanMaxFileSize(pFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then + _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", pFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) + Throw New FileSizeLimitReachedException(pFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) + End If + + Try + oDocument = _zugferd.GetSerializedXMLContentFromFile(pFile) + + Catch ex As ValidationException + Throw ex + + Catch ex As ZUGFeRDExecption + Select Case ex.ErrorType + Case ZUGFeRDInterface.ErrorType.NoZugferd + _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", pFile.Name) + + oResult.EmailAttachmentFiles.Add(pFile) + Return oResult + + Case ZUGFeRDInterface.ErrorType.UnsupportedFormat + _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", pFile.Name, ex.XmlFile) + Throw New UnsupportedFerdException(ex.XmlFile) + + Case ZUGFeRDInterface.ErrorType.NoValidZugferd + _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", pFile.Name) + Throw New InvalidFerdException() + + Case Else + _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", pFile.Name) + Throw ex + End Select + + End Try + + Try + Dim sqlResult As Boolean = StoreXMLItemsInDatabase(pMessageId, oDocument, pFile, pConnections, pArgs) + Catch ex As Exception + Throw ex + End Try + + _logger.Debug("File processed.") + + Dim oMD5Checksum = _hash.GenerateAndCheck_MD5Sum(pFile, pMessageId, pArgs.IgnoreRejectionStatus) + oResult.ZugferdFileFound = True + oResult.MD5Checksum = oMD5Checksum + oResult.ZugferdFileCount = 1 ' Es kann hier nur genau einen Treffer geben! + + Return oResult + End Function + + Private Function ProcessFile(pMessageId As String, pZugferdFileCounter As Integer, pFile As FileInfo, pConnections As DatabaseConnections, pArgs As WorkerArgs) As ProcessFileResult Dim oDocument As ZUGFeRDInterface.ZugferdResult Dim oResult As New ProcessFileResult() ' Only pdf files are allowed from here on - If Not oFile.Name.ToUpper.EndsWith(".PDF") Then - _logger.Debug("Skipping non-pdf file {0}", oFile.Name) - oResult.EmailAttachmentFiles.Add(oFile) + If Not pFile.Name.ToUpper.EndsWith(".PDF") Then + _logger.Debug("Skipping non-pdf file {0}", pFile.Name) + oResult.EmailAttachmentFiles.Add(pFile) ' Checking filesize for attachment files - If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then - _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) - Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) + If _filesystem.TestFileSizeIsLessThanMaxFileSize(pFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then + _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", pFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) + Throw New FileSizeLimitReachedException(pFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) End If Return oResult End If - _logger.Info("Start processing file {0}", oFile.Name) + _logger.Info("Start processing file {0}", pFile.Name) ' Checking filesize for pdf files - If _filesystem.TestFileSizeIsLessThanMaxFileSize(oFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then - _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) - Throw New FileSizeLimitReachedException(oFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) + If _filesystem.TestFileSizeIsLessThanMaxFileSize(pFile.FullName, pArgs.MaxAttachmentSizeInMegaBytes) = False Then + _logger.Warn("Filesize for File [{0}] exceeded limit of {1} MB", pFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) + Throw New FileSizeLimitReachedException(pFile.Name, pArgs.MaxAttachmentSizeInMegaBytes) End If Try - oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(oFile.FullName) + oDocument = _zugferd.ExtractZUGFeRDFileWithGDPicture(pFile.FullName) Catch ex As ValidationException Throw ex @@ -490,70 +560,85 @@ Public Class ImportZUGFeRDFiles Catch ex As ZUGFeRDExecption Select Case ex.ErrorType Case ZUGFeRDInterface.ErrorType.NoZugferd - _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", oFile.Name) + _logger.Info("File [{0}] is not a valid ZUGFeRD document. Skipping.", pFile.Name) - oResult.EmailAttachmentFiles.Add(oFile) + oResult.EmailAttachmentFiles.Add(pFile) Return oResult Case ZUGFeRDInterface.ErrorType.UnsupportedFormat - _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", oFile.Name, ex.XmlFile) + _logger.Info("File [{0}/{1}] is an unsupported ZUFeRD document format!", pFile.Name, ex.XmlFile) Throw New UnsupportedFerdException(ex.XmlFile) Case ZUGFeRDInterface.ErrorType.NoValidZugferd - _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", oFile.Name) + _logger.Info("File [{0}] is an Incorrectly formatted ZUGFeRD document!", pFile.Name) Throw New InvalidFerdException() Case Else - _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", oFile.Name) + _logger.Warn("Unexpected Error occurred while extracting ZUGFeRD Information from file {0}", pFile.Name) Throw ex End Select End Try ' Check if there are more than one ZUGFeRD files - If pZugferdFiles = 1 Then + If pZugferdFileCounter = 1 Then Throw New TooMuchFerdsException() End If ' Since extraction went well, increase the amount of ZUGFeRD files - pZugferdFiles += 1 + pZugferdFileCounter += 1 _logger.Info("Zugferd file found. Increasing counter.") ' Extract all attachments with the extensions specified in `AllowedExtensions`. ' If you need to extract and use embedded xml files, you need to filter out the zugferd-invoice.xml yourself. ' Right now the zugferd-invoice.xml is filtered out because `AllowedExtensions` does not contain `xml`. - Dim oAttachments = _embeds.Extract(oFile.FullName, AllowedExtensions) + Dim oAttachments = _embeds.Extract(pFile.FullName, AllowedExtensions) If oAttachments Is Nothing Then - _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName) + _logger.Warn("Attachments for file [{0}] could not be extracted", pFile.FullName) Else oResult.EmbeddedAttachmentFiles.AddRange(oAttachments) End If + Try + Dim sqlResult As Boolean = StoreXMLItemsInDatabase(pMessageId, oDocument, pFile, pConnections, pArgs) + Catch ex As Exception + Throw ex + End Try + + _logger.Debug("File processed.") + ' Check the Checksum and rejection status - Dim oMD5Checksum = _hash.GenerateAndCheck_MD5Sum(oFile, pMessageId, pArgs.IgnoreRejectionStatus) + Dim oMD5Checksum = _hash.GenerateAndCheck_MD5Sum(pFile, pMessageId, pArgs.IgnoreRejectionStatus) + oResult.ZugferdFileFound = True + oResult.MD5Checksum = oMD5Checksum + oResult.ZugferdFileCount = pZugferdFileCounter + + Return oResult + End Function + + Private Function StoreXMLItemsInDatabase(pMessageId As String, pDocument As ZUGFeRDInterface.ZugferdResult, pFile As FileInfo, pConnections As DatabaseConnections, pArgs As WorkerArgs) As Boolean ' Check the document against the configured property map and return: ' - a List of valid properties ' - a List of missing properties - Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, oDocument.Specification) - Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, pMessageId) + Dim oPropertyMap = _zugferd.FilterPropertyMap(pArgs.PropertyMap, pDocument.Specification) + Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(pDocument.SchemaObject, oPropertyMap, pMessageId) _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count) If oCheckResult.MissingProperties.Count > 0 Then _logger.Warn("[{0}] missing properties found. Exiting.", oCheckResult.MissingProperties.Count) - Throw New MissingValueException(oFile, oCheckResult.MissingProperties) + Throw New MissingValueException(pFile, oCheckResult.MissingProperties) Else _logger.Debug("No missing properties found. Continuing.") - End If - If DeleteExistingPropertyValues(pMessageId, oConnections) = False Then + If DeleteExistingPropertyValues(pMessageId, pConnections) = False Then Throw New Exception("Could not cleanup data. Exiting.") End If ' DataTable vorbereiten - Dim oDataTable As DataTable = FillDataTable(pMessageId, oCheckResult, oDocument.Specification, oDocument.UsedXMLSchema) + Dim oDataTable As DataTable = FillDataTable(pMessageId, oCheckResult, pDocument.Specification, pDocument.UsedXMLSchema) ' ColumnList initialisieren Dim oColumnNames As List(Of String) = New List(Of String) From { @@ -565,7 +650,7 @@ Public Class ImportZUGFeRDFiles "IS_REQUIRED" } - Dim oBulkResult = BulkInsert(oConnections, oDataTable, "TBEDMI_ITEM_VALUE", oColumnNames) + Dim oBulkResult = BulkInsert(pConnections, oDataTable, "TBEDMI_ITEM_VALUE", oColumnNames) If oBulkResult = False Then _logger.Error("Bulk Insert for MessageId [{0}] failed!", pMessageId) @@ -573,15 +658,7 @@ Public Class ImportZUGFeRDFiles End If _logger.Info("Bulk Insert finished. [{0}] rows inserted for MessageId [{1}].", oDataTable.Rows.Count, pMessageId) - - _logger.Debug("File processed.") - - oResult.ZugferdFileFound = True - oResult.MD5Checksum = oMD5Checksum - oResult.ZugferdFileCount = pZugferdFiles - - Return oResult - + Return True End Function Private Function FillDataTable(pMessageId As String, pCheckResult As PropertyValues.CheckPropertyValuesResult, pSpecification As String, pUsedXMLSchema As String) As DataTable @@ -656,6 +733,8 @@ Public Class ImportZUGFeRDFiles Catch ex As Exception _logger.Warn("Step [{0}] with SQL [{1}] was not successful.", oStep, oDelSQL) End Try + + Return False End Function Private Function BulkInsert(pConnections As DatabaseConnections, pTable As DataTable, pDestinationTable As String, pColumns As List(Of String)) As Boolean