AppStd/Monorepo
8
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Tweak logging, ZUGFeRD Import

Browse Source
This commit is contained in:
Jonathan Jenne 2019-03-12 15:48:47 +01:00
parent af00fab575
commit 463f4ce22d
3 changed files with 125 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

220
Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb
View File

@ -4,7 +4,6 @@ Imports System.Linq
Imports DigitalData.Modules.Database Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Interfaces Imports DigitalData.Modules.Interfaces
Imports DigitalData.Modules.Logging Imports DigitalData.Modules.Logging
Imports DigitalData.Modules.Filesystem
Imports System.Text.RegularExpressions Imports System.Text.RegularExpressions
Imports DigitalData.Modules.Jobs.Exceptions Imports DigitalData.Modules.Jobs.Exceptions
Imports FirebirdSql.Data.FirebirdClient Imports FirebirdSql.Data.FirebirdClient
@ -111,7 +110,7 @@ Public Class ImportZUGFeRDFiles
_firebird.ExecuteNonQuery(oSQLInsert) _firebird.ExecuteNonQuery(oSQLInsert)
_logger.Info("Email Queue updated for MessageId {0}.", FileGuid, oEmailTo) _logger.Debug("Email Queue updated for MessageId {0}.", FileGuid, oEmailTo)
Catch ex As Exception Catch ex As Exception
_logger.Error(ex) _logger.Error(ex)
End Try End Try
@ -160,139 +159,148 @@ Public Class ImportZUGFeRDFiles
_logger.Info("Starting Job {0}", Me.GetType.Name) _logger.Info("Starting Job {0}", Me.GetType.Name)
For Each oPath As String In args.WatchDirectories Try
Dim oDirInfo As New DirectoryInfo(oPath) For Each oPath As String In args.WatchDirectories
Dim oDirInfo As New DirectoryInfo(oPath)
_logger.Info($"Start processing directory {oDirInfo.FullName}") _logger.Info($"Start processing directory {oDirInfo.FullName}")
If oDirInfo.Exists Then If oDirInfo.Exists Then
' Filter out *.lock files ' Filter out *.lock files
Dim oFiles As List(Of FileInfo) = oDirInfo. Dim oFiles As List(Of FileInfo) = oDirInfo.
GetFiles(). GetFiles().
Where(Function(f) Not f.Name.EndsWith(".lock")). Where(Function(f) Not f.Name.EndsWith(".lock")).
ToList() ToList()
Dim oFileCount = oFiles.Count Dim oFileCount = oFiles.Count
Dim oCurrentFileCount = 0 Dim oCurrentFileCount = 0
_logger.Info("Found {0} files", oFileCount) _logger.Info("Found {0} files", oFileCount)
' Group files by messageId ' Group files by messageId
Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = GroupFiles(oFiles) Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = GroupFiles(oFiles)
_logger.Info("Found {0} file groups", oGrouped.Count) _logger.Info("Found {0} file groups", oGrouped.Count)
' Process each file group together ' Process each file group together
For Each oFileGroup In oGrouped For Each oFileGroup In oGrouped
' Start a new transaction for each file group. ' Start a new transaction for each file group.
' This way we can rollback database changes for the whole filegroup in case something goes wrong. ' This way we can rollback database changes for the whole filegroup in case something goes wrong.
Dim oConnection As FbConnection = _firebird.GetConnection() Dim oConnection As FbConnection = _firebird.GetConnection()
Dim oTransaction As FbTransaction = oConnection.BeginTransaction() Dim oTransaction As FbTransaction = oConnection.BeginTransaction()
' Count the amount of ZUGFeRD files ' Count the amount of ZUGFeRD files
Dim oZUGFeRDCount As Integer = 0 Dim oZUGFeRDCount As Integer = 0
' Set the default Move Directory ' Set the default Move Directory
Dim oMoveDirectory As String = args.SuccessDirectory Dim oMoveDirectory As String = args.SuccessDirectory
Dim oFileGroupFiles As List(Of FileInfo) = oFileGroup.Value Dim oFileGroupFiles As List(Of FileInfo) = oFileGroup.Value
Dim oFileGroupId As String = oFileGroup.Key Dim oFileGroupId As String = oFileGroup.Key
Dim oMissingProperties As New List(Of String) Dim oMissingProperties As New List(Of String)
_logger.Info("Start processing file group {0}", oFileGroupId) _logger.NewBlock($"Message Id {oFileGroupId}")
_logger.Info("Start processing file group {0}", oFileGroupId)
Try Try
For Each oFile In oFileGroupFiles For Each oFile In oFileGroupFiles
Dim oDocument As CrossIndustryDocumentType Dim oDocument As CrossIndustryDocumentType
' Clear missing properties for the new file ' Clear missing properties for the new file
oMissingProperties = New List(Of String) oMissingProperties = New List(Of String)
oCurrentFileCount += 1 oCurrentFileCount += 1
_logger.Info($"({oCurrentFileCount}/{oFileCount}) Start processing file {oFile.Name}") _logger.Info("Start processing file {0}", oFile.Name)
Try Try
oDocument = _zugferd.ExtractZUGFeRDFile(oFile.FullName) oDocument = _zugferd.ExtractZUGFeRDFile(oFile.FullName)
Catch ex As Exception Catch ex As Exception
_logger.Warn($"({oCurrentFileCount}/{oFileCount}) File is not a valid ZUGFeRD document! Skipping.") _logger.Warn("File is not a valid ZUGFeRD document! Skipping.")
Continue For Continue For
End Try End Try
' Check if there are more than one ZUGFeRD files ' Check if there are more than one ZUGFeRD files
If oZUGFeRDCount = 1 Then If oZUGFeRDCount = 1 Then
Throw New TooMuchFerdsException() Throw New TooMuchFerdsException()
End If
' Since extraction went well, increase the amount of ZUGFeRD files
oZUGFeRDCount += 1
For Each Item As KeyValuePair(Of String, XmlItemProperty) In args.PropertyMap
Dim propertyValue As String = PropertyValues.GetPropValue(oDocument, Item.Key)
Dim propertyDescripton As String = Item.Value.Description
If String.IsNullOrEmpty(propertyValue) Then
If Item.Value.IsRequired Then
_logger.Warn("Property {0} is empty but marked as required! Skipping.", propertyDescripton)
oMissingProperties.Add(propertyDescripton)
Continue For
Else
_logger.Debug("Property {0} is empty or not found. Skipping.", propertyDescripton)
Continue For
End If
End If End If
Dim oTableName = Item.Value.TableName ' Since extraction went well, increase the amount of ZUGFeRD files
Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oFileGroupId}', '{propertyDescripton}', '{propertyValue}')" oZUGFeRDCount += 1
_logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2}", propertyDescripton, propertyValue, oTableName) For Each Item As KeyValuePair(Of String, XmlItemProperty) In args.PropertyMap
Dim propertyValue As String = PropertyValues.GetPropValue(oDocument, Item.Key)
Dim propertyDescripton As String = Item.Value.Description
_firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction) If String.IsNullOrEmpty(propertyValue) Then
If Item.Value.IsRequired Then
_logger.Warn("Property {0} is empty but marked as required! Skipping.", propertyDescripton)
oMissingProperties.Add(propertyDescripton)
Continue For
Else
_logger.Debug("Property {0} is empty or not found. Skipping.", propertyDescripton)
Continue For
End If
End If
Dim oTableName = Item.Value.TableName
Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oFileGroupId}', '{propertyDescripton}', '{propertyValue}')"
_logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2}", propertyDescripton, propertyValue, oTableName)
_firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
Next
If oMissingProperties.Count > 0 Then
Throw New MissingValueException(oFile)
End If
Next Next
If oMissingProperties.Count > 0 Then ' Check if there are no ZUGFeRD files
Throw New MissingValueException(oFile) If oZUGFeRDCount = 0 Then
Throw New NoFerdsException()
End If End If
Next
' Check if there are no ZUGFeRD files ' If no errors occurred, commit the transaction
If oZUGFeRDCount = 0 Then oTransaction.Commit()
Throw New NoFerdsException() Catch ex As TooMuchFerdsException
End If _logger.Error(ex)
' If no errors occurred, commit the transaction oMoveDirectory = args.ErrorDirectory
oTransaction.Commit()
Catch ex As TooMuchFerdsException
_logger.Error(ex)
oMoveDirectory = args.ErrorDirectory Dim oBody = "<p>Your email contained more than one ZUGFeRD-Document.</p>"
AddToEmailQueue(oFileGroupId, oBody)
Catch ex As NoFerdsException
_logger.Error(ex)
Dim oBody = "<p>Your email contained more than one ZUGFeRD-Document.</p>" oMoveDirectory = args.ErrorDirectory
AddToEmailQueue(oFileGroupId, oBody)
Catch ex As NoFerdsException
_logger.Error(ex)
oMoveDirectory = args.ErrorDirectory Dim oBody = "<p>Your email contained no ZUGFeRD-Documents.</p>"
AddToEmailQueue(oFileGroupId, oBody)
Catch ex As MissingValueException
_logger.Error(ex)
Dim oBody = "<p>Your email contained no ZUGFeRD-Documents.</p>" oMoveDirectory = args.ErrorDirectory
AddToEmailQueue(oFileGroupId, oBody)
Catch ex As MissingValueException
_logger.Error(ex)
oMoveDirectory = args.ErrorDirectory Dim oBody = CreateBodyForMissingProperties(ex.File.Name, oMissingProperties)
AddToEmailQueue(oFileGroupId, oBody)
Finally
oConnection.Close()
Dim oBody = CreateBodyForMissingProperties(ex.File.Name, oMissingProperties) ' Move all files of the current group
AddToEmailQueue(oFileGroupId, oBody) For Each oFile In oFileGroupFiles
Finally _filesystem.MoveTo(oFile.FullName, oMoveDirectory)
oConnection.Close() _logger.Info("Finished processing file {0}", oFile.Name)
_logger.Info("File moved to {0}", oMoveDirectory)
Next
' Move all files of the current group _logger.Info("Finished processing file group {0}", oFileGroupId)
For Each oFile In oFileGroupFiles _logger.EndBlock()
_filesystem.MoveTo(oFile.FullName, oMoveDirectory) End Try
_logger.Info("({1}/{2}) Finished processing file {0}", oFile.Name, oCurrentFileCount, oFileCount) Next
_logger.Info($"({1}/{2}) File moved to {0}", oMoveDirectory) End If
Next Next
_logger.Info("Finished processing file group {0}", oFileGroupId) _logger.Info("Finishing Job {0}", Me.GetType.Name)
End Try Catch ex As Exception
Next _logger.Error(ex)
End If _logger.Info("Job Failed! See error log for details")
Next End Try
End Sub End Sub

4
Jobs/Exceptions.vb
View File

@ -17,7 +17,7 @@ Public Class Exceptions
Inherits ApplicationException Inherits ApplicationException
Public Sub New() Public Sub New()
MyBase.New() MyBase.New("More than one ZUGFeRD document found")
End Sub End Sub
End Class End Class
@ -25,7 +25,7 @@ Public Class Exceptions
Inherits ApplicationException Inherits ApplicationException
Public Sub New() Public Sub New()
MyBase.New() MyBase.New("No ZUGFeRD documents found")
End Sub End Sub
End Class End Class
End Class End Class

10
Modules.Logging/Logger.vb
View File

@ -8,7 +8,15 @@ Public Class Logger
''' </summary> ''' </summary>
''' <param name="blockId">A unique Identifier for this block, eg. DocId, FullPath, ..</param> ''' <param name="blockId">A unique Identifier for this block, eg. DocId, FullPath, ..</param>
Public Sub NewBlock(blockId As String) Public Sub NewBlock(blockId As String)
Dim message As String = $"=========={vbTab}{vbTab}Start of Block {blockId}{vbTab}{vbTab}==========" Dim message As String = $"-----> Start of Block {blockId}"
Dim logEventInfo As New LogEventInfo(LogLevel.Info, Name, message)
Dim WrapperType As Type = GetType(Logger)
Log(WrapperType, logEventInfo)
End Sub
Public Sub EndBlock()
Dim message As String = $"<----- End of Block"
Dim logEventInfo As New LogEventInfo(LogLevel.Info, Name, message) Dim logEventInfo As New LogEventInfo(LogLevel.Info, Name, message)
Dim WrapperType As Type = GetType(Logger) Dim WrapperType As Type = GetType(Logger)