From 33937a41d3c30089e039e1f9011e332c0043ee09 Mon Sep 17 00:00:00 2001 From: Jonathan Jenne Date: Tue, 17 Mar 2020 16:56:39 +0100 Subject: [PATCH] start redesign for zugferd rest api --- Modules.Interfaces/Interfaces.vbproj | 13 +- Modules.Interfaces/ZUGFeRDInterface.vb | 7 +- .../ZUGFeRDInterface/FileGroups.vb | 82 ++++ .../ZUGFeRDInterface}/PDFAttachments.vb | 0 .../ZUGFeRDInterface/PropertyValues.vb | 313 +++++++++++++++ .../ZUGFeRDInterface}/XmlItemProperty.vb | 0 .../{ => pdf_zugferd_lib}/pdf_zugferd_lib.dll | Bin .../{ => pdf_zugferd_lib}/pdf_zugferd_lib.lib | Bin .../pdf_zugferd_test.exe | Bin .../EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb | 372 +++++++++--------- Modules.Jobs/EDMI/ZUGFeRD/PropertyValues.vb | 138 ------- Modules.Jobs/EDMI/ZUGFeRD/WorkerArgs.vb | 1 + Modules.Jobs/Jobs.vbproj | 6 - 13 files changed, 594 insertions(+), 338 deletions(-) create mode 100644 Modules.Interfaces/ZUGFeRDInterface/FileGroups.vb rename {Modules.Jobs/EDMI/ZUGFeRD => Modules.Interfaces/ZUGFeRDInterface}/PDFAttachments.vb (100%) create mode 100644 Modules.Interfaces/ZUGFeRDInterface/PropertyValues.vb rename {Modules.Jobs/EDMI/ZUGFeRD => Modules.Interfaces/ZUGFeRDInterface}/XmlItemProperty.vb (100%) rename Modules.Interfaces/ZUGFeRDInterface/{ => pdf_zugferd_lib}/pdf_zugferd_lib.dll (100%) rename Modules.Interfaces/ZUGFeRDInterface/{ => pdf_zugferd_lib}/pdf_zugferd_lib.lib (100%) rename Modules.Interfaces/ZUGFeRDInterface/{ => pdf_zugferd_lib}/pdf_zugferd_test.exe (100%) delete mode 100644 Modules.Jobs/EDMI/ZUGFeRD/PropertyValues.vb diff --git a/Modules.Interfaces/Interfaces.vbproj b/Modules.Interfaces/Interfaces.vbproj index 4127fe30..a83b90ee 100644 --- a/Modules.Interfaces/Interfaces.vbproj +++ b/Modules.Interfaces/Interfaces.vbproj @@ -43,6 +43,9 @@ On + + D:\ProgramFiles\GdPicture.NET 14\Redist\GdPicture.NET (.NET Framework 4.5)\GdPicture.NET.14.dll + ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll @@ -108,6 +111,10 @@ + + + + @@ -128,7 +135,7 @@ Settings.Designer.vb - + PreserveNewest @@ -143,10 +150,10 @@ - + PreserveNewest - + PreserveNewest diff --git a/Modules.Interfaces/ZUGFeRDInterface.vb b/Modules.Interfaces/ZUGFeRDInterface.vb index e8b43eff..11ed3baf 100644 --- a/Modules.Interfaces/ZUGFeRDInterface.vb +++ b/Modules.Interfaces/ZUGFeRDInterface.vb @@ -9,7 +9,7 @@ Public Class ZUGFeRDInterface Private _logConfig As LogConfig Private _logger As Logger - Private Const ZUGFERD_CONVERTER_EXE = "ZUGFeRDInterface\pdf_zugferd_test.exe" + Private Const ZUGFERD_CONVERTER_EXE = "ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_test.exe" Private Const ZUGFERD_CONVERTER_SUCCESS_MESSAGE = "Document contains ZUGFeRD data." Public Enum ErrorType @@ -18,9 +18,14 @@ Public Class ZUGFeRDInterface NoValidZugferd End Enum + Public ReadOnly Property FileGroup As FileGroups + Public ReadOnly Property PropertyValues As PropertyValues + Public Sub New(LogConfig As LogConfig) _logConfig = LogConfig _logger = _logConfig.GetLogger() + FileGroup = New FileGroups(_logConfig) + PropertyValues = New PropertyValues(_logConfig) End Sub ''' diff --git a/Modules.Interfaces/ZUGFeRDInterface/FileGroups.vb b/Modules.Interfaces/ZUGFeRDInterface/FileGroups.vb new file mode 100644 index 00000000..90cac5f5 --- /dev/null +++ b/Modules.Interfaces/ZUGFeRDInterface/FileGroups.vb @@ -0,0 +1,82 @@ +Imports System.IO +Imports System.Text.RegularExpressions +Imports DigitalData.Modules.Logging + +Public Class FileGroups + Private _logger As Logger + + Public Sub New(LogConfig As LogConfig) + _logger = LogConfig.GetLogger() + End Sub + + ''' + ''' Group files by message id. Message id is extracted from filename. + ''' Filename is expected to be in the form: 1234@subdomain.company.com + ''' The list of files to process + ''' + Public Function GroupFiles(Files As List(Of FileInfo)) As Dictionary(Of String, List(Of FileInfo)) + Dim oGrouped As New Dictionary(Of String, List(Of FileInfo)) + + If Files.Count = 0 Then + Return oGrouped + End If + + For Each oFile In Files + Dim oMessageId = GetMessageIdFromFileName(oFile.Name) + + If oMessageId Is Nothing Then + _logger.Warn("File {0} did not have the required filename-format!", oMessageId) + Continue For + End If + + If oGrouped.ContainsKey(oMessageId) Then + oGrouped.Item(oMessageId).Add(oFile) + Else + oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile}) + End If + Next + + Return oGrouped + End Function + + ''' + ''' Group files by message id. Message id is created from `FakeMessageIdDomain` and a random string + ''' + ''' The list of files to process + ''' Arbitrary domain for message id generation. Example: sub.company.com + ''' + Public Function GroupFiles(Files As List(Of FileInfo), FakeMessageIdDomain As String) As Dictionary(Of String, List(Of FileInfo)) + Dim oGrouped As New Dictionary(Of String, List(Of FileInfo)) + + If Files.Count = 0 Then + Return oGrouped + End If + + For Each oFile In Files + Dim oIdentifier = Guid.NewGuid().ToString() + Dim oMessageId = $"{oIdentifier}@{FakeMessageIdDomain}" + + If oGrouped.ContainsKey(oMessageId) Then + oGrouped.Item(oMessageId).Add(oFile) + Else + oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile}) + End If + Next + + Return oGrouped + End Function + + Private Function GetMessageIdFromFileName(Filename As String) As String + ' Regex to find MessageId + ' See also: https://stackoverflow.com/questions/3968500/regex-to-validate-a-message-id-as-per-rfc2822 + Dim oRegex = "(((([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(""(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21\x23-\x5B\x5D-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*""))@(([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(\[(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21-\x5A\x5E-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*\]))))~.+" + Dim oMatch = Regex.Match(Filename, oRegex, RegexOptions.IgnoreCase) + + If oMatch.Success Then + Dim oMessageId = oMatch.Groups(1).Value + Return oMessageId + Else + Return Nothing + End If + End Function +End Class diff --git a/Modules.Jobs/EDMI/ZUGFeRD/PDFAttachments.vb b/Modules.Interfaces/ZUGFeRDInterface/PDFAttachments.vb similarity index 100% rename from Modules.Jobs/EDMI/ZUGFeRD/PDFAttachments.vb rename to Modules.Interfaces/ZUGFeRDInterface/PDFAttachments.vb diff --git a/Modules.Interfaces/ZUGFeRDInterface/PropertyValues.vb b/Modules.Interfaces/ZUGFeRDInterface/PropertyValues.vb new file mode 100644 index 00000000..c56aa5f0 --- /dev/null +++ b/Modules.Interfaces/ZUGFeRDInterface/PropertyValues.vb @@ -0,0 +1,313 @@ +Imports System.Reflection +Imports System.Text.RegularExpressions +Imports DigitalData.Modules.Logging + +Public Class PropertyValues + Private _logger As Logger + Private _logConfig As LogConfig + + Private _indexPattern = "\((\d+)\)" + Private _indexRegex As New Regex(_indexPattern) + + Public Sub New(LogConfig As LogConfig) + _logConfig = LogConfig + _logger = LogConfig.GetLogger() + End Sub + + Public Class CheckPropertyValuesResult + Public MissingProperties As New List(Of String) + Public ValidProperties As List(Of ValidProperty) + End Class + + Public Class ValidProperty + Public MessageId As String + Public TableName As String + + Public GroupCounter As Integer = -1 + + Public Description As String + Public Value As String + End Class + + Public Function CheckPropertyValues(Document As CrossIndustryDocumentType, PropertyMap As Dictionary(Of String, XmlItemProperty), MessageId As String) As CheckPropertyValuesResult + Dim oGlobalGroupCounter = 0 + Dim oMissingProperties As New List(Of String) + Dim oResult As New CheckPropertyValuesResult() + + ' PropertyMap items with `IsGrouped = False` are handled normally + Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = PropertyMap. + Where(Function(Item) Item.Value.IsGrouped = True). + ToDictionary(Function(Item) Item.Key, + Function(Item) Item.Value) + + _logger.Debug("Found {0} default properties.", oDefaultProperties.Count) + + ' PropertyMap items with `IsGrouped = True` are grouped by group scope + Dim oGroupedProperties = PropertyMap. + Where(Function(Item) Item.Value.IsGrouped = True). + ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope + Function(Item) Item) + + _logger.Debug("Found {0} properties grouped in {1} group(s)", PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count) + ' Iterate through groups to get group scope and group items + For Each oGroup In oGroupedProperties + Dim oGroupScope As String = oGroup.Key + Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object)) + Dim oRowCount = 0 + + _logger.Debug("Fetching Property values for group {0}.", oGroupScope) + + ' get properties as a nested object, see `oPropertyList` + For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup + Dim oPropertyValues As List(Of Object) + + Try + oPropertyValues = GetPropValue(Document, oProperty.Key) + Catch ex As Exception + _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope) + _logger.Error(ex) + oPropertyValues = New List(Of Object) + End Try + + ' Flatten result value + oPropertyValues = GetFinalPropValue(oPropertyValues) + + ' Add to list + oPropertyList.Add(oProperty.Value, oPropertyValues) + + ' check the first batch of values to determine the row count + If oRowCount = 0 Then + oRowCount = oPropertyValues.Count + End If + Next + + ' Structure of oPropertyList + ' [ # Propertyname # Row 1 # Row 2 + ' PositionsMenge: [BilledQuantity1, BilledQuantity2, ...], + ' PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...], + ' ... + ' ] + For oRowIndex = 0 To oRowCount - 1 + _logger.Debug("Processing row {0}", oRowIndex) + + For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList + Dim oTableName As String = oColumn.Key.TableName + Dim oPropertyDescription As String = oColumn.Key.Description + Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1 + + ' Returns nothing if oColumn.Value contains an empty list + Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex) + + _logger.Debug("Processing property {0}.", oPropertyDescription) + + If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then + If oColumn.Key.IsRequired Then + _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription) + oResult.MissingProperties.Add(oPropertyDescription) + Else + _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription) + End If + + oPropertyValue = String.Empty + End If + + _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue) + + oResult.ValidProperties.Add(New ValidProperty() With { + .MessageId = MessageId, + .Description = oPropertyDescription, + .Value = oPropertyValue, + .GroupCounter = oRowCounter, + .TableName = oTableName + }) + Next + Next + + oGlobalGroupCounter += oRowCount + Next + + ' Iterate through default properties + For Each oItem As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties + Dim oPropertyValueList As List(Of Object) + Dim oPropertyDescription As String = oItem.Value.Description + Dim oPropertyValue As Object = Nothing + Dim oTableName = oItem.Value.TableName + + Try + oPropertyValueList = GetPropValue(Document, oItem.Key) + Catch ex As Exception + _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, oItem.Value.GroupScope) + _logger.Error(ex) + oPropertyValueList = New List(Of Object) + End Try + + Try + If IsNothing(oPropertyValueList) Then + oPropertyValue = Nothing + ElseIf TypeOf oPropertyValueList Is List(Of Object) Then + Select Case oPropertyValueList.Count + Case 0 + oPropertyValue = Nothing + Case Else + Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object)) + oPropertyValue = oList.Item(0) + + ' This should hopefully show config errors + If TypeOf oPropertyValue Is List(Of Object) Then + _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription) + oPropertyValue = Nothing + End If + End Select + End If + Catch ex As Exception + _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription) + _logger.Error(ex) + oPropertyValue = Nothing + End Try + + If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then + If oItem.Value.IsRequired Then + _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription) + oResult.MissingProperties.Add(oPropertyDescription) + Continue For + Else + _logger.Debug("Property [{0}] is empty or not found. Skipping.", oPropertyDescription) + Continue For + End If + End If + + oResult.ValidProperties.Add(New ValidProperty() With { + .MessageId = MessageId, + .Description = oPropertyDescription, + .Value = oPropertyValue, + .TableName = oTableName + }) + Next + + Return oResult + End Function + + Public Function GetPropValue(Obj As Object, PropertyName As String) As List(Of Object) + Dim oNameParts As String() = PropertyName.Split("."c) + + If IsNothing(Obj) Then + _logger.Debug("`Obj` is Nothing. Exiting.") + Return New List(Of Object) + End If + + + If oNameParts.Length = 1 Then + Dim oPropInfo As PropertyInfo = Obj.GetType().GetProperty(PropertyName) + + If IsNothing(oPropInfo) Then + _logger.Debug("Property {0} does not exist.", PropertyName) + Return New List(Of Object) + Else + Dim oPropValue = oPropInfo.GetValue(Obj, Nothing) + Return New List(Of Object) From {oPropValue} + End If + End If + + For Each oPart As String In oNameParts + Dim oType As Type = Obj.GetType() + Dim oPartName = oPart + Dim oIndex As Integer = Nothing + Dim oHasIndex As Boolean = HasIndex(oPartName) + + If oHasIndex Then + oPartName = StripIndex(oPart) + oIndex = GetIndex(oPart) + End If + + Dim oInfo As PropertyInfo = oType.GetProperty(oPartName) + + If IsNothing(oInfo) OrElse IsNothing(oInfo.GetValue(Obj, Nothing)) Then + _logger.Debug("Property {0} does not exist.", oPartName) + Return New List(Of Object) + End If + + Obj = oInfo.GetValue(Obj, Nothing) + + If oHasIndex Then + Obj = Obj(0) + End If + + If IsArray(Obj) And Not oHasIndex Then + Dim oCurrentPart As String = oPart + Dim oSplitString As String() = New String() {oCurrentPart & "."} + Dim oPathFragments = PropertyName.Split(oSplitString, StringSplitOptions.None) + Dim oResults As New List(Of Object) + + ' if path has no more subitems, return an empty list + If oPathFragments.Length = 1 Then + Return oResults + End If + + For Each oArrayItem In Obj + Dim oResult As List(Of Object) = GetPropValue(oArrayItem, oPathFragments(1)) + + If Not IsNothing(oResult) Then + oResults.Add(oResult) + End If + Next + + Return oResults + End If + Next + + Return New List(Of Object) From {Obj} + End Function + + Public Function GetFinalPropValue(List As List(Of Object)) As List(Of Object) + Dim oResult As New List(Of Object) + + For Each Item In List + Dim oItemValue = DoGetFinalPropValue(Item) + + If Not IsNothing(oItemValue) Then + oResult.Add(oItemValue) + End If + Next + + Return oResult + End Function + + Private Function DoGetFinalPropValue(Value As Object) As String + If TypeOf Value Is List(Of Object) Then + Dim oList = DirectCast(Value, List(Of Object)) + Dim oCount = oList.Count + + Select Case oCount + Case 0 + Return Nothing + Case Else + Return DoGetFinalPropValue(oList.First()) + End Select + + Return DoGetFinalPropValue(Value) + Else + Return Value.ToString + End If + End Function + + Private Function GetIndex(Prop As String) As Integer + If Regex.IsMatch(Prop, _indexPattern) Then + Dim oMatch = _indexRegex.Match(Prop) + Dim oGroup = oMatch.Groups.Item(1) + Dim oValue = oGroup.Value + + Return Integer.Parse(oValue) + End If + + Return Nothing + End Function + + Private Function StripIndex(Prop As String) As String + Return Regex.Replace(Prop, _indexPattern, "") + End Function + + Private Function HasIndex(Prop As String) As Boolean + Return Regex.IsMatch(Prop, _indexPattern) + End Function + +End Class diff --git a/Modules.Jobs/EDMI/ZUGFeRD/XmlItemProperty.vb b/Modules.Interfaces/ZUGFeRDInterface/XmlItemProperty.vb similarity index 100% rename from Modules.Jobs/EDMI/ZUGFeRD/XmlItemProperty.vb rename to Modules.Interfaces/ZUGFeRDInterface/XmlItemProperty.vb diff --git a/Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib.dll b/Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib/pdf_zugferd_lib.dll similarity index 100% rename from Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib.dll rename to Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib/pdf_zugferd_lib.dll diff --git a/Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib.lib b/Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib/pdf_zugferd_lib.lib similarity index 100% rename from Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib.lib rename to Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib/pdf_zugferd_lib.lib diff --git a/Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_test.exe b/Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib/pdf_zugferd_test.exe similarity index 100% rename from Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_test.exe rename to Modules.Interfaces/ZUGFeRDInterface/pdf_zugferd_lib/pdf_zugferd_test.exe diff --git a/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb b/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb index 7b8f889f..50218135 100644 --- a/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb +++ b/Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb @@ -287,45 +287,6 @@ Public Class ImportZUGFeRDFiles _logger.Error(ex) End Try End Sub - Private Function GetMessageIdFromFileName(Filename As String) As String - ' Regex to find MessageId - ' See also: https://stackoverflow.com/questions/3968500/regex-to-validate-a-message-id-as-per-rfc2822 - Dim oRegex = "(((([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(""(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21\x23-\x5B\x5D-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*""))@(([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(\[(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21-\x5A\x5E-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*\]))))~.+" - Dim oMatch = Regex.Match(Filename, oRegex, RegexOptions.IgnoreCase) - - If oMatch.Success Then - Dim oMessageId = oMatch.Groups(1).Value - Return oMessageId - Else - Return Nothing - End If - End Function - - Private Function GroupFiles(Files As List(Of FileInfo)) As Dictionary(Of String, List(Of FileInfo)) - Dim oGrouped As New Dictionary(Of String, List(Of FileInfo)) - - If Files.Count = 0 Then - Return oGrouped - End If - - For Each oFile In Files - Dim oMessageId = GetMessageIdFromFileName(oFile.Name) - - If oMessageId Is Nothing Then - _logger.Warn("File {0} did not have the required filename-format!", oMessageId) - Continue For - End If - - If oGrouped.ContainsKey(oMessageId) Then - oGrouped.Item(oMessageId).Add(oFile) - Else - oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile}) - End If - Next - - Return oGrouped - End Function - Public Sub Start(Arguments As Object) Implements IJob.Start Dim oArgs As WorkerArgs = Arguments @@ -374,7 +335,7 @@ Public Class ImportZUGFeRDFiles End If ' Group files by messageId - Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = GroupFiles(oFiles) + Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = _zugferd.FileGroup.GroupFiles(oFiles) _logger.Info("Found {0} file groups", oGrouped.Count) @@ -480,156 +441,191 @@ Public Class ImportZUGFeRDFiles ' Since extraction went well, increase the amount of ZUGFeRD files oZUGFeRDCount += 1 - ' PropertyMap items with `IsGrouped = False` are handled normally - Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = oArgs.PropertyMap. - Where(Function(Item) Item.Value.IsGrouped = True). - ToDictionary(Function(Item) Item.Key, - Function(Item) Item.Value) - - _logger.Debug("Found {0} default properties.", oDefaultProperties.Count) - - ' PropertyMap items with `IsGrouped = True` are grouped by group scope - Dim oGroupedProperties = oArgs.PropertyMap. - Where(Function(Item) Item.Value.IsGrouped = True). - ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope - Function(Item) Item) - - _logger.Debug("Found {0} properties grouped in {1} group(s)", oArgs.PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count) - ' Iterate through groups to get group scope and group items - For Each oGroup In oGroupedProperties - Dim oGroupScope As String = oGroup.Key - Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object)) - Dim oRowCount = 0 - - _logger.Debug("Fetching Property values for group {0}.", oGroupScope) - - ' get properties as a nested object, see `oPropertyList` - For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup - Dim oPropertyValues As List(Of Object) - - Try - oPropertyValues = oPropertyExtractor.GetPropValue(oDocument, oProperty.Key) - Catch ex As Exception - _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope) - _logger.Error(ex) - oPropertyValues = New List(Of Object) - End Try - - ' Flatten result value - oPropertyValues = oPropertyExtractor.GetFinalPropValue(oPropertyValues) - - ' Add to list - oPropertyList.Add(oProperty.Value, oPropertyValues) - - ' check the first batch of values to determine the row count - If oRowCount = 0 Then - oRowCount = oPropertyValues.Count - End If - Next - - ' Structure of oPropertyList - ' [ # Propertyname # Row 1 # Row 2 - ' PositionsMenge: [BilledQuantity1, BilledQuantity2, ...], - ' PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...], - ' ... - ' ] - For oRowIndex = 0 To oRowCount - 1 - _logger.Debug("Processing row {0}", oRowIndex) - - For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList - Dim oTableName As String = oColumn.Key.TableName - Dim oPropertyDescription As String = oColumn.Key.Description - Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1 - - ' Returns nothing if oColumn.Value contains an empty list - Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex) - - _logger.Debug("Processing property {0}.", oPropertyDescription) - - If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then - If oColumn.Key.IsRequired Then - _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription) - oMissingProperties.Add(oPropertyDescription) - Else - _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription) - End If - - oPropertyValue = String.Empty - End If - - _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue) - - Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})" - _logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2} with RowCounter {3}", oPropertyDescription, oPropertyValue, oTableName, oRowCounter) - - ' Insert into SQL Server - If oArgs.InsertIntoSQLServer = True Then - Dim oResult = _mssql.NewExecutenonQuery(oCommand) - If oResult = False Then - _logger.Warn("SQL Command was not successful. Check the log.") - End If - End If + ' --- BEGIN Check Property Values + + '' PropertyMap items with `IsGrouped = False` are handled normally + 'Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = oArgs.PropertyMap. + ' Where(Function(Item) Item.Value.IsGrouped = True). + ' ToDictionary(Function(Item) Item.Key, + ' Function(Item) Item.Value) + + '_logger.Debug("Found {0} default properties.", oDefaultProperties.Count) + + '' PropertyMap items with `IsGrouped = True` are grouped by group scope + 'Dim oGroupedProperties = oArgs.PropertyMap. + ' Where(Function(Item) Item.Value.IsGrouped = True). + ' ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope + ' Function(Item) Item) + + '_logger.Debug("Found {0} properties grouped in {1} group(s)", oArgs.PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count) + '' Iterate through groups to get group scope and group items + 'For Each oGroup In oGroupedProperties + ' Dim oGroupScope As String = oGroup.Key + ' Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object)) + ' Dim oRowCount = 0 + + ' _logger.Debug("Fetching Property values for group {0}.", oGroupScope) + + ' ' get properties as a nested object, see `oPropertyList` + ' For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup + ' Dim oPropertyValues As List(Of Object) + + ' Try + ' oPropertyValues = oPropertyExtractor.GetPropValue(oDocument, oProperty.Key) + ' Catch ex As Exception + ' _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope) + ' _logger.Error(ex) + ' oPropertyValues = New List(Of Object) + ' End Try + + ' ' Flatten result value + ' oPropertyValues = oPropertyExtractor.GetFinalPropValue(oPropertyValues) + + ' ' Add to list + ' oPropertyList.Add(oProperty.Value, oPropertyValues) + + ' ' check the first batch of values to determine the row count + ' If oRowCount = 0 Then + ' oRowCount = oPropertyValues.Count + ' End If + ' Next + + ' ' Structure of oPropertyList + ' ' [ # Propertyname # Row 1 # Row 2 + ' ' PositionsMenge: [BilledQuantity1, BilledQuantity2, ...], + ' ' PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...], + ' ' ... + ' ' ] + ' For oRowIndex = 0 To oRowCount - 1 + ' _logger.Debug("Processing row {0}", oRowIndex) + + ' For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList + ' Dim oTableName As String = oColumn.Key.TableName + ' Dim oPropertyDescription As String = oColumn.Key.Description + ' Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1 + + ' ' Returns nothing if oColumn.Value contains an empty list + ' Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex) + + ' _logger.Debug("Processing property {0}.", oPropertyDescription) + + ' If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then + ' If oColumn.Key.IsRequired Then + ' _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription) + ' oMissingProperties.Add(oPropertyDescription) + ' Else + ' _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription) + ' End If + + ' oPropertyValue = String.Empty + ' End If + + ' _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue) + + ' Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})" + ' _logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2} with RowCounter {3}", oPropertyDescription, oPropertyValue, oTableName, oRowCounter) + + ' ' Insert into SQL Server + ' If oArgs.InsertIntoSQLServer = True Then + ' Dim oResult = _mssql.NewExecutenonQuery(oCommand) + ' If oResult = False Then + ' _logger.Warn("SQL Command was not successful. Check the log.") + ' End If + ' End If + + ' ' Insert into Firebird + ' _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction) + ' Next + ' Next + + ' oGlobalGroupCounter += oRowCount + 'Next + + '' Iterate through default properties + 'For Each Item As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties + ' Dim oPropertyValueList As List(Of Object) + ' Dim oPropertyDescription As String = Item.Value.Description + ' Dim oPropertyValue As Object = Nothing + + ' Try + ' oPropertyValueList = oPropertyExtractor.GetPropValue(oDocument, Item.Key) + ' Catch ex As Exception + ' _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, Item.Value.GroupScope) + ' _logger.Error(ex) + ' oPropertyValueList = New List(Of Object) + ' End Try + + ' Try + ' If IsNothing(oPropertyValueList) Then + ' oPropertyValue = Nothing + ' ElseIf TypeOf oPropertyValueList Is List(Of Object) Then + ' Select Case oPropertyValueList.Count + ' Case 0 + ' oPropertyValue = Nothing + ' Case Else + ' Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object)) + ' oPropertyValue = oList.Item(0) + + ' ' This should hopefully show config errors + ' If TypeOf oPropertyValue Is List(Of Object) Then + ' _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription) + ' oPropertyValue = Nothing + ' End If + ' End Select + ' End If + ' Catch ex As Exception + ' _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription) + ' _logger.Error(ex) + ' oPropertyValue = Nothing + ' End Try + + ' If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then + ' If Item.Value.IsRequired Then + ' _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription) + ' oMissingProperties.Add(oPropertyDescription) + ' Continue For + ' Else + ' _logger.Debug("Property [{0}] is empty or not found. Skipping.", oPropertyDescription) + ' Continue For + ' End If + ' End If + + ' Dim oTableName = Item.Value.TableName + ' Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}')" + ' _logger.Debug("Mapping Property [{0}] to value [{1}] . Will be inserted into table {2}", oPropertyDescription, oPropertyValue, oTableName) + + ' ' Insert into SQL Server + ' If oArgs.InsertIntoSQLServer = True Then + ' Dim oResult = _mssql.NewExecutenonQuery(oCommand) + ' If oResult = False Then + ' _logger.Warn("SQL Command was not successful. Check the log.") + ' End If + ' End If + + ' ' Insert into Firebird + ' _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction) + 'Next + + '--- END Check Property Values + + ' Check the document against the configured property map and return: + ' - a List of valid properties + ' - a List of missing properties + Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument, oArgs.PropertyMap, oMessageId) + + If oCheckResult.MissingProperties.Count > 0 Then + Throw New MissingValueException(oFile) + End If - ' Insert into Firebird - _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction) - Next - Next + For Each oProperty In oCheckResult.ValidProperties + Dim oGroupCounterValue = Nothing - oGlobalGroupCounter += oRowCount - Next - - ' Iterate through default properties - For Each Item As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties - Dim oPropertyValueList As List(Of Object) - Dim oPropertyDescription As String = Item.Value.Description - Dim oPropertyValue As Object = Nothing - - Try - oPropertyValueList = oPropertyExtractor.GetPropValue(oDocument, Item.Key) - Catch ex As Exception - _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, Item.Value.GroupScope) - _logger.Error(ex) - oPropertyValueList = New List(Of Object) - End Try - - Try - If IsNothing(oPropertyValueList) Then - oPropertyValue = Nothing - ElseIf TypeOf oPropertyValueList Is List(Of Object) Then - Select Case oPropertyValueList.Count - Case 0 - oPropertyValue = Nothing - Case Else - Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object)) - oPropertyValue = oList.Item(0) - - ' This should hopefully show config errors - If TypeOf oPropertyValue Is List(Of Object) Then - _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription) - oPropertyValue = Nothing - End If - End Select - End If - Catch ex As Exception - _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription) - _logger.Error(ex) - oPropertyValue = Nothing - End Try - - If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then - If Item.Value.IsRequired Then - _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription) - oMissingProperties.Add(oPropertyDescription) - Continue For - Else - _logger.Debug("Property [{0}] is empty or not found. Skipping.", oPropertyDescription) - Continue For - End If + If oProperty.GroupCounter > -1 Then + oGroupCounterValue = oProperty.GroupCounter End If - Dim oTableName = Item.Value.TableName - Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}')" - _logger.Debug("Mapping Property [{0}] to value [{1}] . Will be inserted into table {2}", oPropertyDescription, oPropertyValue, oTableName) + Dim oCommand = $"INSERT INTO {oProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oProperty.Description}', '{oProperty.Value}', {oGroupCounterValue})" + _logger.Debug("Mapping Property [{0}] to value [{1}] . Will be inserted into table {2}", oProperty.Description, oProperty.Value, oProperty.TableName) ' Insert into SQL Server If oArgs.InsertIntoSQLServer = True Then @@ -642,10 +638,6 @@ Public Class ImportZUGFeRDFiles ' Insert into Firebird _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction) Next - - If oMissingProperties.Count > 0 Then - Throw New MissingValueException(oFile) - End If Next 'Check if there are no ZUGFeRD files diff --git a/Modules.Jobs/EDMI/ZUGFeRD/PropertyValues.vb b/Modules.Jobs/EDMI/ZUGFeRD/PropertyValues.vb deleted file mode 100644 index 31c6bfc6..00000000 --- a/Modules.Jobs/EDMI/ZUGFeRD/PropertyValues.vb +++ /dev/null @@ -1,138 +0,0 @@ -Imports System.Collections.Generic -Imports System.Linq -Imports System.Reflection -Imports System.Text.RegularExpressions -Imports DigitalData.Modules.Logging - -Public Class PropertyValues - Private _indexPattern = "\((\d+)\)" - Private _indexRegex As New Regex(_indexPattern) - Private _Logger As Logger - - Public Sub New(LogConfig As LogConfig) - _Logger = LogConfig.GetLogger() - End Sub - - Public Function GetPropValue(Obj As Object, PropertyName As String) As List(Of Object) - Dim oNameParts As String() = PropertyName.Split("."c) - - If IsNothing(Obj) Then - _Logger.Debug("`Obj` is Nothing. Exiting.") - Return New List(Of Object) - End If - - - If oNameParts.Length = 1 Then - Dim oPropInfo As PropertyInfo = Obj.GetType().GetProperty(PropertyName) - - If IsNothing(oPropInfo) Then - _Logger.Debug("Property {0} does not exist.", PropertyName) - Return New List(Of Object) - Else - Dim oPropValue = oPropInfo.GetValue(Obj, Nothing) - Return New List(Of Object) From {oPropValue} - End If - End If - - For Each oPart As String In oNameParts - Dim oType As Type = Obj.GetType() - Dim oPartName = oPart - Dim oIndex As Integer = Nothing - Dim oHasIndex As Boolean = HasIndex(oPartName) - - If oHasIndex Then - oPartName = StripIndex(oPart) - oIndex = GetIndex(oPart) - End If - - Dim oInfo As PropertyInfo = oType.GetProperty(oPartName) - - If IsNothing(oInfo) OrElse IsNothing(oInfo.GetValue(Obj, Nothing)) Then - _Logger.Debug("Property {0} does not exist.", oPartName) - Return New List(Of Object) - End If - - Obj = oInfo.GetValue(Obj, Nothing) - - If oHasIndex Then - Obj = Obj(0) - End If - - If IsArray(Obj) And Not oHasIndex Then - Dim oCurrentPart As String = oPart - Dim oSplitString As String() = New String() {oCurrentPart & "."} - Dim oPathFragments = PropertyName.Split(oSplitString, StringSplitOptions.None) - Dim oResults As New List(Of Object) - - ' if path has no more subitems, return an empty list - If oPathFragments.Length = 1 Then - Return oResults - End If - - For Each oArrayItem In Obj - Dim oResult As List(Of Object) = GetPropValue(oArrayItem, oPathFragments(1)) - - If Not IsNothing(oResult) Then - oResults.Add(oResult) - End If - Next - - Return oResults - End If - Next - - Return New List(Of Object) From {Obj} - End Function - - Public Function GetFinalPropValue(List As List(Of Object)) As List(Of Object) - Dim oResult As New List(Of Object) - - For Each Item In List - Dim oItemValue = DoGetFinalPropValue(Item) - - If Not IsNothing(oItemValue) Then - oResult.Add(oItemValue) - End If - Next - - Return oResult - End Function - - Private Function DoGetFinalPropValue(Value As Object) As String - If TypeOf Value Is List(Of Object) Then - Dim oList = DirectCast(Value, List(Of Object)) - Dim oCount = oList.Count - - Select Case oCount - Case 0 - Return Nothing - Case Else - Return DoGetFinalPropValue(oList.First()) - End Select - - Return DoGetFinalPropValue(Value) - Else - Return Value.ToString - End If - End Function - - Private Function GetIndex(Prop As String) As Integer - If Regex.IsMatch(Prop, _indexPattern) Then - Dim oMatch = _indexRegex.Match(Prop) - Dim oGroup = oMatch.Groups.Item(1) - Dim oValue = oGroup.Value - - Return Integer.Parse(oValue) - End If - - Return Nothing - End Function - - Private Function StripIndex(Prop As String) As String - Return Regex.Replace(Prop, _indexPattern, "") - End Function - - Private Function HasIndex(Prop As String) As Boolean - Return Regex.IsMatch(Prop, _indexPattern) - End Function -End Class diff --git a/Modules.Jobs/EDMI/ZUGFeRD/WorkerArgs.vb b/Modules.Jobs/EDMI/ZUGFeRD/WorkerArgs.vb index 71b3f456..46e64e63 100644 --- a/Modules.Jobs/EDMI/ZUGFeRD/WorkerArgs.vb +++ b/Modules.Jobs/EDMI/ZUGFeRD/WorkerArgs.vb @@ -1,4 +1,5 @@ Imports System.Collections.Generic +Imports DigitalData.Modules.Interfaces Public Class WorkerArgs Public WatchDirectories As List(Of String) diff --git a/Modules.Jobs/Jobs.vbproj b/Modules.Jobs/Jobs.vbproj index 3a28c163..979ebe93 100644 --- a/Modules.Jobs/Jobs.vbproj +++ b/Modules.Jobs/Jobs.vbproj @@ -88,10 +88,7 @@ - - - @@ -109,9 +106,6 @@ ..\packages\FirebirdSql.Data.FirebirdClient.6.4.0\lib\net452\FirebirdSql.Data.FirebirdClient.dll - - D:\ProgramFiles\GdPicture.NET 14\Redist\GdPicture.NET (.NET Framework 4.5)\GdPicture.NET.14.dll - ..\packages\NLog.4.6.8\lib\net45\NLog.dll