From 1e02757b2247d7444fa25ffaf987b9a9b49abfe1 Mon Sep 17 00:00:00 2001 From: Jonathan Jenne Date: Tue, 10 Mar 2020 12:29:40 +0100 Subject: [PATCH] Add MessageId to Embedded Attachments --- .../EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb | 82 +++++++++++-------- Modules.Jobs/EDMI/ZUGFeRD/PDFAttachments.vb | 39 ++++++--- 2 files changed, 75 insertions(+), 46 deletions(-) diff --git a/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb b/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb index 20d7cca4..d76b8f1e 100644 --- a/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb +++ b/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb @@ -377,14 +377,15 @@ Public Class ImportZUGFeRDFiles Dim oMoveDirectory As String = oArgs.SuccessDirectory ' Create file lists Dim oFileGroupFiles As List(Of FileInfo) = oFileGroup.Value - Dim oFileAttachmentFiles As New List(Of FileInfo) + Dim oEmailAttachmentFiles As New List(Of FileInfo) + Dim oEmbeddedAttachmentFiles As New List(Of PDFAttachments.AttachmentResult) - Dim oFileGroupId As String = oFileGroup.Key + Dim oMessageId As String = oFileGroup.Key Dim oMissingProperties As New List(Of String) Dim oMD5CheckSum As String = String.Empty - _logger.NewBlock($"Message Id {oFileGroupId}") - _logger.Info("Start processing file group {0}", oFileGroupId) + _logger.NewBlock($"Message Id {oMessageId}") + _logger.Info("Start processing file group {0}", oMessageId) Try For Each oFile In oFileGroupFiles @@ -398,7 +399,7 @@ Public Class ImportZUGFeRDFiles ' Only pdf files are allowed from here on If Not oFile.Name.EndsWith(".pdf") Then _logger.Debug("Skipping non-pdf file {0}", oFile.Name) - oFileAttachmentFiles.Add(oFile) + oEmailAttachmentFiles.Add(oFile) Continue For End If @@ -410,7 +411,7 @@ Public Class ImportZUGFeRDFiles Select Case ex.ErrorType Case ZUGFeRDInterface.ErrorType.NoZugferd _logger.Warn("File is not a valid ZUGFeRD document! Skipping.") - oFileAttachmentFiles.Add(oFile) + oEmailAttachmentFiles.Add(oFile) Continue For Case ZUGFeRDInterface.ErrorType.NoValidZugferd @@ -428,10 +429,7 @@ Public Class ImportZUGFeRDFiles If oAttachments Is Nothing Then _logger.Warn("Attachments for file [{0}] could not be extracted", oFile.FullName) Else - 'oFileAttachmentFiles.AddRange(oFileGroupFiles) - 'oFileAttachmentFiles.AddRange(oAttachments) - - oFileAttachmentFiles.AddRange(oAttachments) + oEmbeddedAttachmentFiles.AddRange(oAttachments) End If oMD5CheckSum = CreateMD5(oFile.FullName) @@ -550,7 +548,7 @@ Public Class ImportZUGFeRDFiles _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue) - Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oFileGroupId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})" + Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})" _logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2} with RowCounter {3}", oPropertyDescription, oPropertyValue, oTableName, oRowCounter) ' Insert into SQL Server @@ -619,7 +617,7 @@ Public Class ImportZUGFeRDFiles End If Dim oTableName = Item.Value.TableName - Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oFileGroupId}', '{oPropertyDescription}', '{oPropertyValue}')" + Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}')" _logger.Debug("Mapping Property [{0}] to value [{1}] . Will be inserted into table {2}", oPropertyDescription, oPropertyValue, oTableName) ' Insert into SQL Server @@ -647,12 +645,12 @@ Public Class ImportZUGFeRDFiles 'If no errors occurred... 'Log the History If oMD5CheckSum <> String.Empty Then - Dim oInsertCommand = $"INSERT INTO TBEDM_ZUGFERD_HISTORY_IN (MESSAGE_ID, MD5HASH) VALUES ('{oFileGroupId}', '{oMD5CheckSum}')" + Dim oInsertCommand = $"INSERT INTO TBEDM_ZUGFERD_HISTORY_IN (MESSAGE_ID, MD5HASH) VALUES ('{oMessageId}', '{oMD5CheckSum}')" _firebird.ExecuteNonQueryWithConnection(oInsertCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction) 'commit the transaction oTransaction.Commit() Try - Dim oSQL = $"SELECT MAX(GUID) FROM TBEDM_ZUGFERD_HISTORY_IN WHERE MESSAGE_ID = '{oFileGroupId}'" + Dim oSQL = $"SELECT MAX(GUID) FROM TBEDM_ZUGFERD_HISTORY_IN WHERE MESSAGE_ID = '{oMessageId}'" HISTORY_ID = _firebird.GetScalarValue(oSQL) Catch ex As Exception HISTORY_ID = 0 @@ -666,9 +664,9 @@ Public Class ImportZUGFeRDFiles _firebird.ExecuteNonQuery(oSQL) Dim oBody = EMAIL_MD5_ERROR - Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oFileGroupId) - AddToEmailQueueMSSQL(oFileGroupId, oBody, oEmailData, "MD5HashException") - AddRejectedState(oFileGroupId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "") + Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId) + AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MD5HashException") + AddRejectedState(oMessageId, "MD5HashException", "Die gesendete Rechnung wurde bereits verarbeitet!", "") Catch ex As InvalidFerdException _logger.Error(ex) @@ -676,9 +674,9 @@ Public Class ImportZUGFeRDFiles Dim oSQL = $"UPDATE TBEDM_ZUGFERD_HISTORY_IN SET COMMENT = 'REJECTED - ZUGFeRD yes but incorrect format' WHERE GUID = '{HISTORY_ID}'" _firebird.ExecuteNonQuery(oSQL) Dim oBody = EMAIL_INVALID_DOCUMENT - Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oFileGroupId) - AddToEmailQueueMSSQL(oFileGroupId, oBody, oEmailData, "InvalidFerdException") - AddRejectedState(oFileGroupId, "InvalidFerdException", "Inkorrekte Formate", "") + Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId) + AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "InvalidFerdException") + AddRejectedState(oMessageId, "InvalidFerdException", "Inkorrekte Formate", "") Catch ex As TooMuchFerdsException _logger.Error(ex) @@ -686,9 +684,9 @@ Public Class ImportZUGFeRDFiles Dim oSQL = $"UPDATE TBEDM_ZUGFERD_HISTORY_IN SET COMMENT = 'REJECTED - More than one ZUGFeRD-document in email' WHERE GUID = '{HISTORY_ID}'" _firebird.ExecuteNonQuery(oSQL) Dim oBody = EMAIL_TOO_MUCH_FERDS - Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oFileGroupId) - AddToEmailQueueMSSQL(oFileGroupId, oBody, oEmailData, "TooMuchFerdsException") - AddRejectedState(oFileGroupId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "") + Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId) + AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "TooMuchFerdsException") + AddRejectedState(oMessageId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "") Catch ex As NoFerdsException _logger.Error(ex) @@ -696,9 +694,9 @@ Public Class ImportZUGFeRDFiles Dim oSQL = $"UPDATE TBEDM_ZUGFERD_HISTORY_IN SET COMMENT = 'REJECTED - no ZUGFeRD-Document in email' WHERE GUID = '{HISTORY_ID}'" _firebird.ExecuteNonQuery(oSQL) Dim oBody = EMAIL_NO_FERDS - Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oFileGroupId) - AddToEmailQueueMSSQL(oFileGroupId, oBody, oEmailData, "NoFerdsException") - AddRejectedState(oFileGroupId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "") + Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId) + AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "NoFerdsException") + AddRejectedState(oMessageId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "") Catch ex As MissingValueException _logger.Error(ex) @@ -711,9 +709,9 @@ Public Class ImportZUGFeRDFiles _firebird.ExecuteNonQuery(oSQL) Dim oBody = CreateBodyForMissingProperties(ex.File.Name, oMissingProperties) - Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oFileGroupId) - AddToEmailQueueMSSQL(oFileGroupId, oBody, oEmailData, "MissingValueException") - AddRejectedState(oFileGroupId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage) + Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId) + AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "MissingValueException") + AddRejectedState(oMessageId, "MissingValueException", "Es fehlten ZugferdSpezifikationen", oMessage) Catch ex As Exception _logger.Warn("Unknown Error occurred: {0}", ex.Message) @@ -721,14 +719,14 @@ Public Class ImportZUGFeRDFiles Dim oSQL = $"UPDATE TBEDM_ZUGFERD_HISTORY_IN SET COMMENT = 'REJECTED - Unknown error occured' WHERE GUID = '{HISTORY_ID}'" _firebird.ExecuteNonQuery(oSQL) oMoveDirectory = oArgs.ErrorDirectory - AddRejectedState(oFileGroupId, "UnexpectedException", "", ex.Message) + AddRejectedState(oMessageId, "UnexpectedException", "", ex.Message) Finally oConnection.Close() ' Move all files of the current group Try - MoveFiles(oArgs, oFileGroupFiles, oFileAttachmentFiles, oMoveDirectory) - _logger.Info("Finished processing file group {0}", oFileGroupId) + MoveFiles(oArgs, oMessageId, oFileGroupFiles, oEmailAttachmentFiles, oEmbeddedAttachmentFiles, oMoveDirectory) + _logger.Info("Finished processing file group {0}", oMessageId) Catch ex As Exception _logger.Warn("Could not move files!") _logger.Error(ex) @@ -748,10 +746,11 @@ Public Class ImportZUGFeRDFiles End Try End Sub - Private Sub MoveFiles(Args As WorkerArgs, Files As List(Of FileInfo), AttachmentFiles As List(Of FileInfo), MoveDirectory As String) + Private Sub MoveFiles(Args As WorkerArgs, MessageId As String, Files As List(Of FileInfo), AttachmentFiles As List(Of FileInfo), EmbeddedAttachments As List(Of PDFAttachments.AttachmentResult), MoveDirectory As String) Dim oFinalMoveDirectory As String = MoveDirectory Dim oAttachmentDirectory As String = Path.Combine(MoveDirectory, Args.AttachmentsSubDirectory) + ' Create directories if they don't exist If Not Directory.Exists(oFinalMoveDirectory) Then Try Directory.CreateDirectory(oFinalMoveDirectory) @@ -768,6 +767,7 @@ Public Class ImportZUGFeRDFiles End Try End If + ' Move PDF/A Files For Each oFile In Files Try Dim oFileName = _filesystem.GetVersionedFilename(Path.Combine(oFinalMoveDirectory, oFile.Name)) @@ -782,6 +782,7 @@ Public Class ImportZUGFeRDFiles End Try Next + ' Move non-PDF/A Email Attachments/Files For Each oFile In AttachmentFiles Try Dim oFileName = _filesystem.GetVersionedFilename(Path.Combine(oAttachmentDirectory, oFile.Name)) @@ -795,6 +796,21 @@ Public Class ImportZUGFeRDFiles _logger.Error(ex) End Try Next + + ' Write Embedded Files to disk + For Each oResult In EmbeddedAttachments + Try + Dim oFileName As String = $"{MessageId}~{oResult.FileName}" + Dim oFilePath As String = Path.Combine(oAttachmentDirectory, oFileName) + + Using oWriter As New FileStream(oFilePath, FileMode.Create) + oWriter.Write(oResult.FileContents, 0, oResult.FileContents.Length) + End Using + Catch ex As Exception + _logger.Warn("Could not save embedded attachment {0}", oResult.FileName) + _logger.Error(ex) + End Try + Next End Sub diff --git a/Modules.Jobs/EDMI/ZUGFeRD/PDFAttachments.vb b/Modules.Jobs/EDMI/ZUGFeRD/PDFAttachments.vb index 86efe522..e22d48ba 100644 --- a/Modules.Jobs/EDMI/ZUGFeRD/PDFAttachments.vb +++ b/Modules.Jobs/EDMI/ZUGFeRD/PDFAttachments.vb @@ -5,17 +5,20 @@ Imports GdPicture14 Public Class PDFAttachments Private Logger As Logger - Private Filesystem As Filesystem.File Private Const ZUGFERD_XML_FILENAME = "ZUGFeRD-invoice.xml" + Public Class AttachmentResult + Public FileName As String + Public FileContents As Byte() + End Class + Public Sub New(LogConfig As LogConfig, GdPictureKey As String) Logger = LogConfig.GetLogger - Filesystem = New Filesystem.File(LogConfig) End Sub - Public Function Extract(FileName As String, AllowedExtensions As List(Of String)) As List(Of FileInfo) - Dim oResults As New List(Of FileInfo) + Public Function Extract(FileName As String, AllowedExtensions As List(Of String)) As List(Of AttachmentResult) + Dim oResults As New List(Of AttachmentResult) Dim oExtensions = AllowedExtensions.ConvertAll(Of String)(New Converter(Of String, String)(Function(ext) ext.ToUpper)) Try @@ -34,21 +37,31 @@ Public Class PDFAttachments Dim FileSize As Integer = oGDPicturePDF.GetEmbeddedFileSize(index) If oGDPicturePDF.GetStat() = GdPictureStatus.OK Then - Dim FileData As Byte() = New Byte(FileSize) {} - Dim status As GdPictureStatus = oGDPicturePDF.ExtractEmbeddedFile(index, FileData) + Dim oFileData As Byte() = New Byte(FileSize) {} + Dim status As GdPictureStatus = oGDPicturePDF.ExtractEmbeddedFile(index, oFileData) If status = GdPictureStatus.OK Then - Dim oVersionedName = Filesystem.GetVersionedFilename(oFileName) - Dim oTempName As String = Path.Combine(Path.GetTempPath(), oVersionedName) - Using oFileStream As New FileStream(oTempName, FileMode.OpenOrCreate) - oFileStream.Write(FileData, 0, FileData.Length) - End Using - - oResults.Add(New FileInfo(oTempName)) + oResults.Add(New AttachmentResult() With { + .FileContents = oFileData, + .FileName = oFileName + }) Else Logger.Error("The embedded file [{0}] has failed to extract. Status: {1}", oFileName, oGDPicturePDF.GetStat().ToString()) Continue For End If + + 'If status = GdPictureStatus.OK Then + ' Dim oVersionedName = Filesystem.GetVersionedFilename(oFileName) + ' Dim oTempName As String = Path.Combine(Path.GetTempPath(), oVersionedName) + ' Using oFileStream As New FileStream(oTempName, FileMode.OpenOrCreate) + ' oFileStream.Write(oFileData, 0, oFileData.Length) + ' End Using + + ' oResults.Add(New FileInfo(oTempName)) + 'Else + ' Logger.Error("The embedded file [{0}] has failed to extract. Status: {1}", oFileName, oGDPicturePDF.GetStat().ToString()) + ' Continue For + 'End If Else Logger.Error("An error occurred getting the file size for [{0}]. Status: {1}", oFileName, oGDPicturePDF.GetStat().ToString()) Continue For