start redesign for zugferd rest api

Modules.Interfaces/Interfaces.vbproj

@@ -43,6 +43,9 @@
     <OptionInfer>On</OptionInfer>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="GdPicture.NET.14">
+      <HintPath>D:\ProgramFiles\GdPicture.NET 14\Redist\GdPicture.NET (.NET Framework 4.5)\GdPicture.NET.14.dll</HintPath>
+    </Reference>
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
@@ -108,6 +111,10 @@
     </Compile>
     <Compile Include="ZUGFeRDInterface\CrossIndustryDocumentType.vb" />
     <Compile Include="ZUGFeRDInterface.vb" />
+    <Compile Include="ZUGFeRDInterface\FileGroups.vb" />
+    <Compile Include="ZUGFeRDInterface\PDFAttachments.vb" />
+    <Compile Include="ZUGFeRDInterface\PropertyValues.vb" />
+    <Compile Include="ZUGFeRDInterface\XmlItemProperty.vb" />
   </ItemGroup>
   <ItemGroup>
     <EmbeddedResource Include="My Project\Resources.resx">
@@ -128,7 +135,7 @@
       <LastGenOutput>Settings.Designer.vb</LastGenOutput>
     </None>
     <None Include="packages.config" />
-    <None Include="ZUGFeRDInterface\pdf_zugferd_lib.lib">
+    <None Include="ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_lib.lib">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
@@ -143,10 +150,10 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
-    <Content Include="ZUGFeRDInterface\pdf_zugferd_lib.dll">
+    <Content Include="ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_lib.dll">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="ZUGFeRDInterface\pdf_zugferd_test.exe">
+    <Content Include="ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_test.exe">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>

Modules.Interfaces/ZUGFeRDInterface.vb

@@ -9,7 +9,7 @@ Public Class ZUGFeRDInterface
     Private _logConfig As LogConfig
     Private _logger As Logger
 
-    Private Const ZUGFERD_CONVERTER_EXE = "ZUGFeRDInterface\pdf_zugferd_test.exe"
+    Private Const ZUGFERD_CONVERTER_EXE = "ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_test.exe"
     Private Const ZUGFERD_CONVERTER_SUCCESS_MESSAGE = "Document contains ZUGFeRD data."
 
     Public Enum ErrorType
@@ -18,9 +18,14 @@ Public Class ZUGFeRDInterface
         NoValidZugferd
     End Enum
 
+    Public ReadOnly Property FileGroup As FileGroups
+    Public ReadOnly Property PropertyValues As PropertyValues
+
     Public Sub New(LogConfig As LogConfig)
         _logConfig = LogConfig
         _logger = _logConfig.GetLogger()
+        FileGroup = New FileGroups(_logConfig)
+        PropertyValues = New PropertyValues(_logConfig)
     End Sub
 
     ''' <summary>

Modules.Interfaces/ZUGFeRDInterface/FileGroups.vb

@@ -0,0 +1,82 @@
+Imports System.IO
+Imports System.Text.RegularExpressions
+Imports DigitalData.Modules.Logging
+
+Public Class FileGroups
+    Private _logger As Logger
+
+    Public Sub New(LogConfig As LogConfig)
+        _logger = LogConfig.GetLogger()
+    End Sub
+
+    ''' <summary>
+    ''' Group files by message id. Message id is extracted from filename.
+    ''' Filename is expected to be in the form: 1234@subdomain.company.com
+    ''' <param name="Files">The list of files to process</param>
+    ''' </summary>
+    Public Function GroupFiles(Files As List(Of FileInfo)) As Dictionary(Of String, List(Of FileInfo))
+        Dim oGrouped As New Dictionary(Of String, List(Of FileInfo))
+
+        If Files.Count = 0 Then
+            Return oGrouped
+        End If
+
+        For Each oFile In Files
+            Dim oMessageId = GetMessageIdFromFileName(oFile.Name)
+
+            If oMessageId Is Nothing Then
+                _logger.Warn("File {0} did not have the required filename-format!", oMessageId)
+                Continue For
+            End If
+
+            If oGrouped.ContainsKey(oMessageId) Then
+                oGrouped.Item(oMessageId).Add(oFile)
+            Else
+                oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile})
+            End If
+        Next
+
+        Return oGrouped
+    End Function
+
+    ''' <summary>
+    ''' Group files by message id. Message id is created from `FakeMessageIdDomain` and a random string 
+    ''' </summary>
+    ''' <param name="Files">The list of files to process</param>
+    ''' <param name="FakeMessageIdDomain">Arbitrary domain for message id generation. Example: sub.company.com</param>
+    ''' <returns></returns>
+    Public Function GroupFiles(Files As List(Of FileInfo), FakeMessageIdDomain As String) As Dictionary(Of String, List(Of FileInfo))
+        Dim oGrouped As New Dictionary(Of String, List(Of FileInfo))
+
+        If Files.Count = 0 Then
+            Return oGrouped
+        End If
+
+        For Each oFile In Files
+            Dim oIdentifier = Guid.NewGuid().ToString()
+            Dim oMessageId = $"{oIdentifier}@{FakeMessageIdDomain}"
+
+            If oGrouped.ContainsKey(oMessageId) Then
+                oGrouped.Item(oMessageId).Add(oFile)
+            Else
+                oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile})
+            End If
+        Next
+
+        Return oGrouped
+    End Function
+
+    Private Function GetMessageIdFromFileName(Filename As String) As String
+        ' Regex to find MessageId
+        ' See also: https://stackoverflow.com/questions/3968500/regex-to-validate-a-message-id-as-per-rfc2822
+        Dim oRegex = "(((([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(""(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21\x23-\x5B\x5D-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*""))@(([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(\[(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21-\x5A\x5E-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*\]))))~.+"
+        Dim oMatch = Regex.Match(Filename, oRegex, RegexOptions.IgnoreCase)
+
+        If oMatch.Success Then
+            Dim oMessageId = oMatch.Groups(1).Value
+            Return oMessageId
+        Else
+            Return Nothing
+        End If
+    End Function
+End Class

Modules.Interfaces/ZUGFeRDInterface/PropertyValues.vb

@@ -0,0 +1,313 @@
+Imports System.Reflection
+Imports System.Text.RegularExpressions
+Imports DigitalData.Modules.Logging
+
+Public Class PropertyValues
+    Private _logger As Logger
+    Private _logConfig As LogConfig
+
+    Private _indexPattern = "\((\d+)\)"
+    Private _indexRegex As New Regex(_indexPattern)
+
+    Public Sub New(LogConfig As LogConfig)
+        _logConfig = LogConfig
+        _logger = LogConfig.GetLogger()
+    End Sub
+
+    Public Class CheckPropertyValuesResult
+        Public MissingProperties As New List(Of String)
+        Public ValidProperties As List(Of ValidProperty)
+    End Class
+
+    Public Class ValidProperty
+        Public MessageId As String
+        Public TableName As String
+
+        Public GroupCounter As Integer = -1
+
+        Public Description As String
+        Public Value As String
+    End Class
+
+    Public Function CheckPropertyValues(Document As CrossIndustryDocumentType, PropertyMap As Dictionary(Of String, XmlItemProperty), MessageId As String) As CheckPropertyValuesResult
+        Dim oGlobalGroupCounter = 0
+        Dim oMissingProperties As New List(Of String)
+        Dim oResult As New CheckPropertyValuesResult()
+
+        ' PropertyMap items with `IsGrouped = False` are handled normally
+        Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = PropertyMap.
+            Where(Function(Item) Item.Value.IsGrouped = True).
+            ToDictionary(Function(Item) Item.Key,
+                         Function(Item) Item.Value)
+
+        _logger.Debug("Found {0} default properties.", oDefaultProperties.Count)
+
+        ' PropertyMap items with `IsGrouped = True` are grouped by group scope
+        Dim oGroupedProperties = PropertyMap.
+            Where(Function(Item) Item.Value.IsGrouped = True).
+            ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope
+                     Function(Item) Item)
+
+        _logger.Debug("Found {0} properties grouped in {1} group(s)", PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count)
+        ' Iterate through groups to get group scope and group items
+        For Each oGroup In oGroupedProperties
+            Dim oGroupScope As String = oGroup.Key
+            Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object))
+            Dim oRowCount = 0
+
+            _logger.Debug("Fetching Property values for group {0}.", oGroupScope)
+
+            ' get properties as a nested object, see `oPropertyList`
+            For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup
+                Dim oPropertyValues As List(Of Object)
+
+                Try
+                    oPropertyValues = GetPropValue(Document, oProperty.Key)
+                Catch ex As Exception
+                    _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope)
+                    _logger.Error(ex)
+                    oPropertyValues = New List(Of Object)
+                End Try
+
+                ' Flatten result value
+                oPropertyValues = GetFinalPropValue(oPropertyValues)
+
+                ' Add to list
+                oPropertyList.Add(oProperty.Value, oPropertyValues)
+
+                ' check the first batch of values to determine the row count
+                If oRowCount = 0 Then
+                    oRowCount = oPropertyValues.Count
+                End If
+            Next
+
+            ' Structure of oPropertyList
+            ' [ # Propertyname       # Row 1              # Row 2
+            '   PositionsMenge:      [BilledQuantity1,    BilledQuantity2, ...],
+            '   PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...],
+            '   ...
+            ' ]
+            For oRowIndex = 0 To oRowCount - 1
+                _logger.Debug("Processing row {0}", oRowIndex)
+
+                For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList
+                    Dim oTableName As String = oColumn.Key.TableName
+                    Dim oPropertyDescription As String = oColumn.Key.Description
+                    Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1
+
+                    ' Returns nothing if oColumn.Value contains an empty list
+                    Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex)
+
+                    _logger.Debug("Processing property {0}.", oPropertyDescription)
+
+                    If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
+                        If oColumn.Key.IsRequired Then
+                            _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription)
+                            oResult.MissingProperties.Add(oPropertyDescription)
+                        Else
+                            _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription)
+                        End If
+
+                        oPropertyValue = String.Empty
+                    End If
+
+                    _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue)
+
+                    oResult.ValidProperties.Add(New ValidProperty() With {
+                        .MessageId = MessageId,
+                        .Description = oPropertyDescription,
+                        .Value = oPropertyValue,
+                        .GroupCounter = oRowCounter,
+                        .TableName = oTableName
+                    })
+                Next
+            Next
+
+            oGlobalGroupCounter += oRowCount
+        Next
+
+        ' Iterate through default properties
+        For Each oItem As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties
+            Dim oPropertyValueList As List(Of Object)
+            Dim oPropertyDescription As String = oItem.Value.Description
+            Dim oPropertyValue As Object = Nothing
+            Dim oTableName = oItem.Value.TableName
+
+            Try
+                oPropertyValueList = GetPropValue(Document, oItem.Key)
+            Catch ex As Exception
+                _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, oItem.Value.GroupScope)
+                _logger.Error(ex)
+                oPropertyValueList = New List(Of Object)
+            End Try
+
+            Try
+                If IsNothing(oPropertyValueList) Then
+                    oPropertyValue = Nothing
+                ElseIf TypeOf oPropertyValueList Is List(Of Object) Then
+                    Select Case oPropertyValueList.Count
+                        Case 0
+                            oPropertyValue = Nothing
+                        Case Else
+                            Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object))
+                            oPropertyValue = oList.Item(0)
+
+                            ' This should hopefully show config errors
+                            If TypeOf oPropertyValue Is List(Of Object) Then
+                                _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription)
+                                oPropertyValue = Nothing
+                            End If
+                    End Select
+                End If
+            Catch ex As Exception
+                _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription)
+                _logger.Error(ex)
+                oPropertyValue = Nothing
+            End Try
+
+            If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
+                If oItem.Value.IsRequired Then
+                    _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription)
+                    oResult.MissingProperties.Add(oPropertyDescription)
+                    Continue For
+                Else
+                    _logger.Debug("Property [{0}]  is empty or not found. Skipping.", oPropertyDescription)
+                    Continue For
+                End If
+            End If
+
+            oResult.ValidProperties.Add(New ValidProperty() With {
+                .MessageId = MessageId,
+                .Description = oPropertyDescription,
+                .Value = oPropertyValue,
+                .TableName = oTableName
+            })
+        Next
+
+        Return oResult
+    End Function
+
+    Public Function GetPropValue(Obj As Object, PropertyName As String) As List(Of Object)
+        Dim oNameParts As String() = PropertyName.Split("."c)
+
+        If IsNothing(Obj) Then
+            _logger.Debug("`Obj` is Nothing. Exiting.")
+            Return New List(Of Object)
+        End If
+
+
+        If oNameParts.Length = 1 Then
+            Dim oPropInfo As PropertyInfo = Obj.GetType().GetProperty(PropertyName)
+
+            If IsNothing(oPropInfo) Then
+                _logger.Debug("Property {0} does not exist.", PropertyName)
+                Return New List(Of Object)
+            Else
+                Dim oPropValue = oPropInfo.GetValue(Obj, Nothing)
+                Return New List(Of Object) From {oPropValue}
+            End If
+        End If
+
+        For Each oPart As String In oNameParts
+            Dim oType As Type = Obj.GetType()
+            Dim oPartName = oPart
+            Dim oIndex As Integer = Nothing
+            Dim oHasIndex As Boolean = HasIndex(oPartName)
+
+            If oHasIndex Then
+                oPartName = StripIndex(oPart)
+                oIndex = GetIndex(oPart)
+            End If
+
+            Dim oInfo As PropertyInfo = oType.GetProperty(oPartName)
+
+            If IsNothing(oInfo) OrElse IsNothing(oInfo.GetValue(Obj, Nothing)) Then
+                _logger.Debug("Property {0} does not exist.", oPartName)
+                Return New List(Of Object)
+            End If
+
+            Obj = oInfo.GetValue(Obj, Nothing)
+
+            If oHasIndex Then
+                Obj = Obj(0)
+            End If
+
+            If IsArray(Obj) And Not oHasIndex Then
+                Dim oCurrentPart As String = oPart
+                Dim oSplitString As String() = New String() {oCurrentPart & "."}
+                Dim oPathFragments = PropertyName.Split(oSplitString, StringSplitOptions.None)
+                Dim oResults As New List(Of Object)
+
+                ' if path has no more subitems, return an empty list
+                If oPathFragments.Length = 1 Then
+                    Return oResults
+                End If
+
+                For Each oArrayItem In Obj
+                    Dim oResult As List(Of Object) = GetPropValue(oArrayItem, oPathFragments(1))
+
+                    If Not IsNothing(oResult) Then
+                        oResults.Add(oResult)
+                    End If
+                Next
+
+                Return oResults
+            End If
+        Next
+
+        Return New List(Of Object) From {Obj}
+    End Function
+
+    Public Function GetFinalPropValue(List As List(Of Object)) As List(Of Object)
+        Dim oResult As New List(Of Object)
+
+        For Each Item In List
+            Dim oItemValue = DoGetFinalPropValue(Item)
+
+            If Not IsNothing(oItemValue) Then
+                oResult.Add(oItemValue)
+            End If
+        Next
+
+        Return oResult
+    End Function
+
+    Private Function DoGetFinalPropValue(Value As Object) As String
+        If TypeOf Value Is List(Of Object) Then
+            Dim oList = DirectCast(Value, List(Of Object))
+            Dim oCount = oList.Count
+
+            Select Case oCount
+                Case 0
+                    Return Nothing
+                Case Else
+                    Return DoGetFinalPropValue(oList.First())
+            End Select
+
+            Return DoGetFinalPropValue(Value)
+        Else
+            Return Value.ToString
+        End If
+    End Function
+
+    Private Function GetIndex(Prop As String) As Integer
+        If Regex.IsMatch(Prop, _indexPattern) Then
+            Dim oMatch = _indexRegex.Match(Prop)
+            Dim oGroup = oMatch.Groups.Item(1)
+            Dim oValue = oGroup.Value
+
+            Return Integer.Parse(oValue)
+        End If
+
+        Return Nothing
+    End Function
+
+    Private Function StripIndex(Prop As String) As String
+        Return Regex.Replace(Prop, _indexPattern, "")
+    End Function
+
+    Private Function HasIndex(Prop As String) As Boolean
+        Return Regex.IsMatch(Prop, _indexPattern)
+    End Function
+
+End Class

Modules.Jobs/EDMI/ZUGFeRD/ImportZUGFeRDFiles.vb

@@ -287,45 +287,6 @@ Public Class ImportZUGFeRDFiles
             _logger.Error(ex)
         End Try
     End Sub
-    Private Function GetMessageIdFromFileName(Filename As String) As String
-        ' Regex to find MessageId
-        ' See also: https://stackoverflow.com/questions/3968500/regex-to-validate-a-message-id-as-per-rfc2822
-        Dim oRegex = "(((([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(""(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21\x23-\x5B\x5D-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*""))@(([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(\[(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21-\x5A\x5E-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*\]))))~.+"
-        Dim oMatch = Regex.Match(Filename, oRegex, RegexOptions.IgnoreCase)
-
-        If oMatch.Success Then
-            Dim oMessageId = oMatch.Groups(1).Value
-            Return oMessageId
-        Else
-            Return Nothing
-        End If
-    End Function
-
-    Private Function GroupFiles(Files As List(Of FileInfo)) As Dictionary(Of String, List(Of FileInfo))
-        Dim oGrouped As New Dictionary(Of String, List(Of FileInfo))
-
-        If Files.Count = 0 Then
-            Return oGrouped
-        End If
-
-        For Each oFile In Files
-            Dim oMessageId = GetMessageIdFromFileName(oFile.Name)
-
-            If oMessageId Is Nothing Then
-                _logger.Warn("File {0} did not have the required filename-format!", oMessageId)
-                Continue For
-            End If
-
-            If oGrouped.ContainsKey(oMessageId) Then
-                oGrouped.Item(oMessageId).Add(oFile)
-            Else
-                oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile})
-            End If
-        Next
-
-        Return oGrouped
-    End Function
-
 
     Public Sub Start(Arguments As Object) Implements IJob.Start
         Dim oArgs As WorkerArgs = Arguments
@@ -374,7 +335,7 @@ Public Class ImportZUGFeRDFiles
                     End If
 
                     ' Group files by messageId
-                    Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = GroupFiles(oFiles)
+                    Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = _zugferd.FileGroup.GroupFiles(oFiles)
 
                     _logger.Info("Found {0} file groups", oGrouped.Count)
 
@@ -480,172 +441,203 @@ Public Class ImportZUGFeRDFiles
                                 ' Since extraction went well, increase the amount of ZUGFeRD files
                                 oZUGFeRDCount += 1
 
-                                ' PropertyMap items with `IsGrouped = False` are handled normally
+                                ' --- BEGIN Check Property Values
-                                Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = oArgs.PropertyMap.
-                                    Where(Function(Item) Item.Value.IsGrouped = True).
-                                    ToDictionary(Function(Item) Item.Key,
-                                                 Function(Item) Item.Value)
 
-                                _logger.Debug("Found {0} default properties.", oDefaultProperties.Count)
+                                '' PropertyMap items with `IsGrouped = False` are handled normally
+                                'Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = oArgs.PropertyMap.
+                                '    Where(Function(Item) Item.Value.IsGrouped = True).
+                                '    ToDictionary(Function(Item) Item.Key,
+                                '                 Function(Item) Item.Value)
 
-                                ' PropertyMap items with `IsGrouped = True` are grouped by group scope
+                                '_logger.Debug("Found {0} default properties.", oDefaultProperties.Count)
-                                Dim oGroupedProperties = oArgs.PropertyMap.
-                                    Where(Function(Item) Item.Value.IsGrouped = True).
-                                    ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope
-                                             Function(Item) Item)
 
-                                _logger.Debug("Found {0} properties grouped in {1} group(s)", oArgs.PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count)
+                                '' PropertyMap items with `IsGrouped = True` are grouped by group scope
-                                ' Iterate through groups to get group scope and group items
+                                'Dim oGroupedProperties = oArgs.PropertyMap.
-                                For Each oGroup In oGroupedProperties
+                                '    Where(Function(Item) Item.Value.IsGrouped = True).
-                                    Dim oGroupScope As String = oGroup.Key
+                                '    ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope
-                                    Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object))
+                                '             Function(Item) Item)
-                                    Dim oRowCount = 0
 
-                                    _logger.Debug("Fetching Property values for group {0}.", oGroupScope)
+                                '_logger.Debug("Found {0} properties grouped in {1} group(s)", oArgs.PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count)
+                                '' Iterate through groups to get group scope and group items
+                                'For Each oGroup In oGroupedProperties
+                                '    Dim oGroupScope As String = oGroup.Key
+                                '    Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object))
+                                '    Dim oRowCount = 0
 
-                                    ' get properties as a nested object, see `oPropertyList`
+                                '    _logger.Debug("Fetching Property values for group {0}.", oGroupScope)
-                                    For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup
-                                        Dim oPropertyValues As List(Of Object)
 
-                                        Try
+                                '    ' get properties as a nested object, see `oPropertyList`
-                                            oPropertyValues = oPropertyExtractor.GetPropValue(oDocument, oProperty.Key)
+                                '    For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup
-                                        Catch ex As Exception
+                                '        Dim oPropertyValues As List(Of Object)
-                                            _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope)
-                                            _logger.Error(ex)
-                                            oPropertyValues = New List(Of Object)
-                                        End Try
 
-                                        ' Flatten result value
+                                '        Try
-                                        oPropertyValues = oPropertyExtractor.GetFinalPropValue(oPropertyValues)
+                                '            oPropertyValues = oPropertyExtractor.GetPropValue(oDocument, oProperty.Key)
+                                '        Catch ex As Exception
+                                '            _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope)
+                                '            _logger.Error(ex)
+                                '            oPropertyValues = New List(Of Object)
+                                '        End Try
 
-                                        ' Add to list
+                                '        ' Flatten result value
-                                        oPropertyList.Add(oProperty.Value, oPropertyValues)
+                                '        oPropertyValues = oPropertyExtractor.GetFinalPropValue(oPropertyValues)
 
-                                        ' check the first batch of values to determine the row count
+                                '        ' Add to list
-                                        If oRowCount = 0 Then
+                                '        oPropertyList.Add(oProperty.Value, oPropertyValues)
-                                            oRowCount = oPropertyValues.Count
-                                        End If
-                                    Next
 
-                                    ' Structure of oPropertyList
+                                '        ' check the first batch of values to determine the row count
-                                    ' [ # Propertyname       # Row 1              # Row 2
+                                '        If oRowCount = 0 Then
-                                    '   PositionsMenge:      [BilledQuantity1,    BilledQuantity2, ...],
+                                '            oRowCount = oPropertyValues.Count
-                                    '   PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...],
+                                '        End If
-                                    '   ...
+                                '    Next
-                                    ' ]
-                                    For oRowIndex = 0 To oRowCount - 1
-                                        _logger.Debug("Processing row {0}", oRowIndex)
 
-                                        For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList
+                                '    ' Structure of oPropertyList
-                                            Dim oTableName As String = oColumn.Key.TableName
+                                '    ' [ # Propertyname       # Row 1              # Row 2
-                                            Dim oPropertyDescription As String = oColumn.Key.Description
+                                '    '   PositionsMenge:      [BilledQuantity1,    BilledQuantity2, ...],
-                                            Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1
+                                '    '   PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...],
+                                '    '   ...
+                                '    ' ]
+                                '    For oRowIndex = 0 To oRowCount - 1
+                                '        _logger.Debug("Processing row {0}", oRowIndex)
 
-                                            ' Returns nothing if oColumn.Value contains an empty list
+                                '        For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList
-                                            Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex)
+                                '            Dim oTableName As String = oColumn.Key.TableName
+                                '            Dim oPropertyDescription As String = oColumn.Key.Description
+                                '            Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1
 
-                                            _logger.Debug("Processing property {0}.", oPropertyDescription)
+                                '            ' Returns nothing if oColumn.Value contains an empty list
+                                '            Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex)
 
-                                            If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
+                                '            _logger.Debug("Processing property {0}.", oPropertyDescription)
-                                                If oColumn.Key.IsRequired Then
-                                                    _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription)
-                                                    oMissingProperties.Add(oPropertyDescription)
-                                                Else
-                                                    _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription)
-                                                End If
 
-                                                oPropertyValue = String.Empty
+                                '            If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
-                                            End If
+                                '                If oColumn.Key.IsRequired Then
+                                '                    _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription)
+                                '                    oMissingProperties.Add(oPropertyDescription)
+                                '                Else
+                                '                    _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription)
+                                '                End If
 
-                                            _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue)
+                                '                oPropertyValue = String.Empty
+                                '            End If
 
-                                            Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})"
+                                '            _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue)
-                                            _logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2} with RowCounter {3}", oPropertyDescription, oPropertyValue, oTableName, oRowCounter)
 
-                                            ' Insert into SQL Server
+                                '            Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})"
-                                            If oArgs.InsertIntoSQLServer = True Then
+                                '            _logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2} with RowCounter {3}", oPropertyDescription, oPropertyValue, oTableName, oRowCounter)
-                                                Dim oResult = _mssql.NewExecutenonQuery(oCommand)
-                                                If oResult = False Then
-                                                    _logger.Warn("SQL Command was not successful. Check the log.")
-                                                End If
-                                            End If
 
-                                            ' Insert into Firebird
+                                '            ' Insert into SQL Server
-                                            _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
+                                '            If oArgs.InsertIntoSQLServer = True Then
-                                        Next
+                                '                Dim oResult = _mssql.NewExecutenonQuery(oCommand)
-                                    Next
+                                '                If oResult = False Then
+                                '                    _logger.Warn("SQL Command was not successful. Check the log.")
+                                '                End If
+                                '            End If
 
-                                    oGlobalGroupCounter += oRowCount
+                                '            ' Insert into Firebird
-                                Next
+                                '            _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
+                                '        Next
+                                '    Next
 
-                                ' Iterate through default properties
+                                '    oGlobalGroupCounter += oRowCount
-                                For Each Item As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties
+                                'Next
-                                    Dim oPropertyValueList As List(Of Object)
-                                    Dim oPropertyDescription As String = Item.Value.Description
-                                    Dim oPropertyValue As Object = Nothing
 
-                                    Try
+                                '' Iterate through default properties
-                                        oPropertyValueList = oPropertyExtractor.GetPropValue(oDocument, Item.Key)
+                                'For Each Item As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties
-                                    Catch ex As Exception
+                                '    Dim oPropertyValueList As List(Of Object)
-                                        _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, Item.Value.GroupScope)
+                                '    Dim oPropertyDescription As String = Item.Value.Description
-                                        _logger.Error(ex)
+                                '    Dim oPropertyValue As Object = Nothing
-                                        oPropertyValueList = New List(Of Object)
-                                    End Try
 
-                                    Try
+                                '    Try
-                                        If IsNothing(oPropertyValueList) Then
+                                '        oPropertyValueList = oPropertyExtractor.GetPropValue(oDocument, Item.Key)
-                                            oPropertyValue = Nothing
+                                '    Catch ex As Exception
-                                        ElseIf TypeOf oPropertyValueList Is List(Of Object) Then
+                                '        _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, Item.Value.GroupScope)
-                                            Select Case oPropertyValueList.Count
+                                '        _logger.Error(ex)
-                                                Case 0
+                                '        oPropertyValueList = New List(Of Object)
-                                                    oPropertyValue = Nothing
+                                '    End Try
-                                                Case Else
-                                                    Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object))
-                                                    oPropertyValue = oList.Item(0)
 
-                                                    ' This should hopefully show config errors
+                                '    Try
-                                                    If TypeOf oPropertyValue Is List(Of Object) Then
+                                '        If IsNothing(oPropertyValueList) Then
-                                                        _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription)
+                                '            oPropertyValue = Nothing
-                                                        oPropertyValue = Nothing
+                                '        ElseIf TypeOf oPropertyValueList Is List(Of Object) Then
-                                                    End If
+                                '            Select Case oPropertyValueList.Count
-                                            End Select
+                                '                Case 0
-                                        End If
+                                '                    oPropertyValue = Nothing
-                                    Catch ex As Exception
+                                '                Case Else
-                                        _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription)
+                                '                    Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object))
-                                        _logger.Error(ex)
+                                '                    oPropertyValue = oList.Item(0)
-                                        oPropertyValue = Nothing
-                                    End Try
 
-                                    If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
+                                '                    ' This should hopefully show config errors
-                                        If Item.Value.IsRequired Then
+                                '                    If TypeOf oPropertyValue Is List(Of Object) Then
-                                            _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription)
+                                '                        _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription)
-                                            oMissingProperties.Add(oPropertyDescription)
+                                '                        oPropertyValue = Nothing
-                                            Continue For
+                                '                    End If
-                                        Else
+                                '            End Select
-                                            _logger.Debug("Property [{0}]  is empty or not found. Skipping.", oPropertyDescription)
+                                '        End If
-                                            Continue For
+                                '    Catch ex As Exception
-                                        End If
+                                '        _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription)
-                                    End If
+                                '        _logger.Error(ex)
+                                '        oPropertyValue = Nothing
+                                '    End Try
 
-                                    Dim oTableName = Item.Value.TableName
+                                '    If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
-                                    Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}')"
+                                '        If Item.Value.IsRequired Then
-                                    _logger.Debug("Mapping Property [{0}]  to value [{1}] . Will be inserted into table {2}", oPropertyDescription, oPropertyValue, oTableName)
+                                '            _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription)
+                                '            oMissingProperties.Add(oPropertyDescription)
+                                '            Continue For
+                                '        Else
+                                '            _logger.Debug("Property [{0}]  is empty or not found. Skipping.", oPropertyDescription)
+                                '            Continue For
+                                '        End If
+                                '    End If
 
-                                    ' Insert into SQL Server
+                                '    Dim oTableName = Item.Value.TableName
-                                    If oArgs.InsertIntoSQLServer = True Then
+                                '    Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}')"
-                                        Dim oResult = _mssql.NewExecutenonQuery(oCommand)
+                                '    _logger.Debug("Mapping Property [{0}]  to value [{1}] . Will be inserted into table {2}", oPropertyDescription, oPropertyValue, oTableName)
-                                        If oResult = False Then
-                                            _logger.Warn("SQL Command was not successful. Check the log.")
-                                        End If
-                                    End If
 
-                                    ' Insert into Firebird
+                                '    ' Insert into SQL Server
-                                    _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
+                                '    If oArgs.InsertIntoSQLServer = True Then
-                                Next
+                                '        Dim oResult = _mssql.NewExecutenonQuery(oCommand)
+                                '        If oResult = False Then
+                                '            _logger.Warn("SQL Command was not successful. Check the log.")
+                                '        End If
+                                '    End If
 
-                                If oMissingProperties.Count > 0 Then
+                                '    ' Insert into Firebird
+                                '    _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
+                                'Next
+
+                                '--- END Check Property Values
+
+                                ' Check the document against the configured property map and return:
+                                ' - a List of valid properties 
+                                ' - a List of missing properties
+                                Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument, oArgs.PropertyMap, oMessageId)
+
+                                If oCheckResult.MissingProperties.Count > 0 Then
                                     Throw New MissingValueException(oFile)
                                 End If
+
+                                For Each oProperty In oCheckResult.ValidProperties
+                                    Dim oGroupCounterValue = Nothing
+
+                                    If oProperty.GroupCounter > -1 Then
+                                        oGroupCounterValue = oProperty.GroupCounter
+                                    End If
+
+                                    Dim oCommand = $"INSERT INTO {oProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oProperty.Description}', '{oProperty.Value}', {oGroupCounterValue})"
+                                    _logger.Debug("Mapping Property [{0}]  to value [{1}] . Will be inserted into table {2}", oProperty.Description, oProperty.Value, oProperty.TableName)
+
+                                    ' Insert into SQL Server
+                                    If oArgs.InsertIntoSQLServer = True Then
+                                        Dim oResult = _mssql.NewExecutenonQuery(oCommand)
+                                        If oResult = False Then
+                                            _logger.Warn("SQL Command was not successful. Check the log.")
+                                        End If
+                                    End If
+
+                                    ' Insert into Firebird
+                                    _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
+                                Next
                             Next
 
                             'Check if there are no ZUGFeRD files

Modules.Jobs/EDMI/ZUGFeRD/PropertyValues.vb

@@ -1,138 +0,0 @@
-Imports System.Collections.Generic
-Imports System.Linq
-Imports System.Reflection
-Imports System.Text.RegularExpressions
-Imports DigitalData.Modules.Logging
-
-Public Class PropertyValues
-    Private _indexPattern = "\((\d+)\)"
-    Private _indexRegex As New Regex(_indexPattern)
-    Private _Logger As Logger
-
-    Public Sub New(LogConfig As LogConfig)
-        _Logger = LogConfig.GetLogger()
-    End Sub
-
-    Public Function GetPropValue(Obj As Object, PropertyName As String) As List(Of Object)
-        Dim oNameParts As String() = PropertyName.Split("."c)
-
-        If IsNothing(Obj) Then
-            _Logger.Debug("`Obj` is Nothing. Exiting.")
-            Return New List(Of Object)
-        End If
-
-
-        If oNameParts.Length = 1 Then
-            Dim oPropInfo As PropertyInfo = Obj.GetType().GetProperty(PropertyName)
-
-            If IsNothing(oPropInfo) Then
-                _Logger.Debug("Property {0} does not exist.", PropertyName)
-                Return New List(Of Object)
-            Else
-                Dim oPropValue = oPropInfo.GetValue(Obj, Nothing)
-                Return New List(Of Object) From {oPropValue}
-            End If
-        End If
-
-        For Each oPart As String In oNameParts
-            Dim oType As Type = Obj.GetType()
-            Dim oPartName = oPart
-            Dim oIndex As Integer = Nothing
-            Dim oHasIndex As Boolean = HasIndex(oPartName)
-
-            If oHasIndex Then
-                oPartName = StripIndex(oPart)
-                oIndex = GetIndex(oPart)
-            End If
-
-            Dim oInfo As PropertyInfo = oType.GetProperty(oPartName)
-
-            If IsNothing(oInfo) OrElse IsNothing(oInfo.GetValue(Obj, Nothing)) Then
-                _Logger.Debug("Property {0} does not exist.", oPartName)
-                Return New List(Of Object)
-            End If
-
-            Obj = oInfo.GetValue(Obj, Nothing)
-
-            If oHasIndex Then
-                Obj = Obj(0)
-            End If
-
-            If IsArray(Obj) And Not oHasIndex Then
-                Dim oCurrentPart As String = oPart
-                Dim oSplitString As String() = New String() {oCurrentPart & "."}
-                Dim oPathFragments = PropertyName.Split(oSplitString, StringSplitOptions.None)
-                Dim oResults As New List(Of Object)
-
-                ' if path has no more subitems, return an empty list
-                If oPathFragments.Length = 1 Then
-                    Return oResults
-                End If
-
-                For Each oArrayItem In Obj
-                    Dim oResult As List(Of Object) = GetPropValue(oArrayItem, oPathFragments(1))
-
-                    If Not IsNothing(oResult) Then
-                        oResults.Add(oResult)
-                    End If
-                Next
-
-                Return oResults
-            End If
-        Next
-
-        Return New List(Of Object) From {Obj}
-    End Function
-
-    Public Function GetFinalPropValue(List As List(Of Object)) As List(Of Object)
-        Dim oResult As New List(Of Object)
-
-        For Each Item In List
-            Dim oItemValue = DoGetFinalPropValue(Item)
-
-            If Not IsNothing(oItemValue) Then
-                oResult.Add(oItemValue)
-            End If
-        Next
-
-        Return oResult
-    End Function
-
-    Private Function DoGetFinalPropValue(Value As Object) As String
-        If TypeOf Value Is List(Of Object) Then
-            Dim oList = DirectCast(Value, List(Of Object))
-            Dim oCount = oList.Count
-
-            Select Case oCount
-                Case 0
-                    Return Nothing
-                Case Else
-                    Return DoGetFinalPropValue(oList.First())
-            End Select
-
-            Return DoGetFinalPropValue(Value)
-        Else
-            Return Value.ToString
-        End If
-    End Function
-
-    Private Function GetIndex(Prop As String) As Integer
-        If Regex.IsMatch(Prop, _indexPattern) Then
-            Dim oMatch = _indexRegex.Match(Prop)
-            Dim oGroup = oMatch.Groups.Item(1)
-            Dim oValue = oGroup.Value
-
-            Return Integer.Parse(oValue)
-        End If
-
-        Return Nothing
-    End Function
-
-    Private Function StripIndex(Prop As String) As String
-        Return Regex.Replace(Prop, _indexPattern, "")
-    End Function
-
-    Private Function HasIndex(Prop As String) As Boolean
-        Return Regex.IsMatch(Prop, _indexPattern)
-    End Function
-End Class

Modules.Jobs/EDMI/ZUGFeRD/WorkerArgs.vb

@@ -1,4 +1,5 @@
 Imports System.Collections.Generic
+Imports DigitalData.Modules.Interfaces
 
 Public Class WorkerArgs
     Public WatchDirectories As List(Of String)

Modules.Jobs/Jobs.vbproj

@@ -88,10 +88,7 @@
     <Compile Include="EDMI\GraphQL\GraphQLJob.vb" />
     <Compile Include="EDMI\ZUGFeRD\EmailData.vb" />
     <Compile Include="EDMI\ZUGFeRD\ImportZUGFeRDFiles.vb" />
-    <Compile Include="EDMI\ZUGFeRD\PDFAttachments.vb" />
-    <Compile Include="EDMI\ZUGFeRD\PropertyValues.vb" />
     <Compile Include="EDMI\ZUGFeRD\WorkerArgs.vb" />
-    <Compile Include="EDMI\ZUGFeRD\XmlItemProperty.vb" />
     <Compile Include="Exceptions.vb" />
     <Compile Include="JobInterface.vb" />
     <Compile Include="JobBase.vb" />
@@ -109,9 +106,6 @@
     <Reference Include="FirebirdSql.Data.FirebirdClient, Version=6.4.0.0, Culture=neutral, PublicKeyToken=3750abcc3150b00c, processorArchitecture=MSIL">
       <HintPath>..\packages\FirebirdSql.Data.FirebirdClient.6.4.0\lib\net452\FirebirdSql.Data.FirebirdClient.dll</HintPath>
     </Reference>
-    <Reference Include="GdPicture.NET.14">
-      <HintPath>D:\ProgramFiles\GdPicture.NET 14\Redist\GdPicture.NET (.NET Framework 4.5)\GdPicture.NET.14.dll</HintPath>
-    </Reference>
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="NLog, Version=4.0.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
       <HintPath>..\packages\NLog.4.6.8\lib\net45\NLog.dll</HintPath>