start redesign for zugferd rest api

Modules.Interfaces/Interfaces.vbproj

@@ -43,6 +43,9 @@
     <OptionInfer>On</OptionInfer>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="GdPicture.NET.14">
+      <HintPath>D:\ProgramFiles\GdPicture.NET 14\Redist\GdPicture.NET (.NET Framework 4.5)\GdPicture.NET.14.dll</HintPath>
+    </Reference>
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
@@ -108,6 +111,10 @@
     </Compile>
     <Compile Include="ZUGFeRDInterface\CrossIndustryDocumentType.vb" />
     <Compile Include="ZUGFeRDInterface.vb" />
+    <Compile Include="ZUGFeRDInterface\FileGroups.vb" />
+    <Compile Include="ZUGFeRDInterface\PDFAttachments.vb" />
+    <Compile Include="ZUGFeRDInterface\PropertyValues.vb" />
+    <Compile Include="ZUGFeRDInterface\XmlItemProperty.vb" />
   </ItemGroup>
   <ItemGroup>
     <EmbeddedResource Include="My Project\Resources.resx">
@@ -128,7 +135,7 @@
       <LastGenOutput>Settings.Designer.vb</LastGenOutput>
     </None>
     <None Include="packages.config" />
-    <None Include="ZUGFeRDInterface\pdf_zugferd_lib.lib">
+    <None Include="ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_lib.lib">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>
@@ -143,10 +150,10 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
-    <Content Include="ZUGFeRDInterface\pdf_zugferd_lib.dll">
+    <Content Include="ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_lib.dll">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="ZUGFeRDInterface\pdf_zugferd_test.exe">
+    <Content Include="ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_test.exe">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>

Modules.Interfaces/ZUGFeRDInterface.vb

@@ -9,7 +9,7 @@ Public Class ZUGFeRDInterface
     Private _logConfig As LogConfig
     Private _logger As Logger
 
-    Private Const ZUGFERD_CONVERTER_EXE = "ZUGFeRDInterface\pdf_zugferd_test.exe"
+    Private Const ZUGFERD_CONVERTER_EXE = "ZUGFeRDInterface\pdf_zugferd_lib\pdf_zugferd_test.exe"
     Private Const ZUGFERD_CONVERTER_SUCCESS_MESSAGE = "Document contains ZUGFeRD data."
 
     Public Enum ErrorType
@@ -18,9 +18,14 @@ Public Class ZUGFeRDInterface
         NoValidZugferd
     End Enum
 
+    Public ReadOnly Property FileGroup As FileGroups
+    Public ReadOnly Property PropertyValues As PropertyValues
+
     Public Sub New(LogConfig As LogConfig)
         _logConfig = LogConfig
         _logger = _logConfig.GetLogger()
+        FileGroup = New FileGroups(_logConfig)
+        PropertyValues = New PropertyValues(_logConfig)
     End Sub
 
     ''' <summary>

Modules.Interfaces/ZUGFeRDInterface/FileGroups.vb

@@ -0,0 +1,82 @@
+Imports System.IO
+Imports System.Text.RegularExpressions
+Imports DigitalData.Modules.Logging
+
+Public Class FileGroups
+    Private _logger As Logger
+
+    Public Sub New(LogConfig As LogConfig)
+        _logger = LogConfig.GetLogger()
+    End Sub
+
+    ''' <summary>
+    ''' Group files by message id. Message id is extracted from filename.
+    ''' Filename is expected to be in the form: 1234@subdomain.company.com
+    ''' <param name="Files">The list of files to process</param>
+    ''' </summary>
+    Public Function GroupFiles(Files As List(Of FileInfo)) As Dictionary(Of String, List(Of FileInfo))
+        Dim oGrouped As New Dictionary(Of String, List(Of FileInfo))
+
+        If Files.Count = 0 Then
+            Return oGrouped
+        End If
+
+        For Each oFile In Files
+            Dim oMessageId = GetMessageIdFromFileName(oFile.Name)
+
+            If oMessageId Is Nothing Then
+                _logger.Warn("File {0} did not have the required filename-format!", oMessageId)
+                Continue For
+            End If
+
+            If oGrouped.ContainsKey(oMessageId) Then
+                oGrouped.Item(oMessageId).Add(oFile)
+            Else
+                oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile})
+            End If
+        Next
+
+        Return oGrouped
+    End Function
+
+    ''' <summary>
+    ''' Group files by message id. Message id is created from `FakeMessageIdDomain` and a random string 
+    ''' </summary>
+    ''' <param name="Files">The list of files to process</param>
+    ''' <param name="FakeMessageIdDomain">Arbitrary domain for message id generation. Example: sub.company.com</param>
+    ''' <returns></returns>
+    Public Function GroupFiles(Files As List(Of FileInfo), FakeMessageIdDomain As String) As Dictionary(Of String, List(Of FileInfo))
+        Dim oGrouped As New Dictionary(Of String, List(Of FileInfo))
+
+        If Files.Count = 0 Then
+            Return oGrouped
+        End If
+
+        For Each oFile In Files
+            Dim oIdentifier = Guid.NewGuid().ToString()
+            Dim oMessageId = $"{oIdentifier}@{FakeMessageIdDomain}"
+
+            If oGrouped.ContainsKey(oMessageId) Then
+                oGrouped.Item(oMessageId).Add(oFile)
+            Else
+                oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile})
+            End If
+        Next
+
+        Return oGrouped
+    End Function
+
+    Private Function GetMessageIdFromFileName(Filename As String) As String
+        ' Regex to find MessageId
+        ' See also: https://stackoverflow.com/questions/3968500/regex-to-validate-a-message-id-as-per-rfc2822
+        Dim oRegex = "(((([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(""(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21\x23-\x5B\x5D-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*""))@(([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(\[(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21-\x5A\x5E-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*\]))))~.+"
+        Dim oMatch = Regex.Match(Filename, oRegex, RegexOptions.IgnoreCase)
+
+        If oMatch.Success Then
+            Dim oMessageId = oMatch.Groups(1).Value
+            Return oMessageId
+        Else
+            Return Nothing
+        End If
+    End Function
+End Class

Modules.Interfaces/ZUGFeRDInterface/PDFAttachments.vb

@@ -0,0 +1,100 @@
+Imports System.Collections.Generic
+Imports System.IO
+Imports DigitalData.Modules.Logging
+Imports GdPicture14
+
+Public Class PDFAttachments
+    Private Logger As Logger
+
+    Private Const ZUGFERD_XML_FILENAME = "ZUGFeRD-invoice.xml"
+
+    Public Class AttachmentResult
+        Public FileName As String
+        Public FileContents As Byte()
+    End Class
+
+    Public Sub New(LogConfig As LogConfig)
+        Logger = LogConfig.GetLogger
+    End Sub
+
+    Public Function Extract(FileName As String, AllowedExtensions As List(Of String)) As List(Of AttachmentResult)
+        Dim oResults As New List(Of AttachmentResult)
+        Dim oExtensions = AllowedExtensions.ConvertAll(Of String)(New Converter(Of String, String)(Function(ext) ext.ToUpper))
+
+        Try
+            Using oGDPicturePDF As New GdPicturePDF()
+                If oGDPicturePDF.LoadFromFile(FileName, False) = GdPictureStatus.OK Then
+                    Dim oEmbeddedFileCount As Integer = oGDPicturePDF.GetEmbeddedFileCount()
+                    If oGDPicturePDF.GetStat() = GdPictureStatus.OK Then
+                        If oEmbeddedFileCount > 1 Then
+                            For index = 0 To oEmbeddedFileCount - 1
+                                Dim oFileName As String = oGDPicturePDF.GetEmbeddedFileName(index)
+
+                                If oGDPicturePDF.GetStat() = GdPictureStatus.OK Then
+                                    Dim oExtension = New FileInfo(oFileName).Extension.ToUpper.Substring(1)
+                                    If oFileName.ToUpper <> ZUGFERD_XML_FILENAME.ToUpper Then
+                                        If oExtensions.Contains(oExtension) Then
+                                            Dim FileSize As Integer = oGDPicturePDF.GetEmbeddedFileSize(index)
+
+                                            If oGDPicturePDF.GetStat() = GdPictureStatus.OK Then
+                                                Dim oFileData As Byte() = New Byte(FileSize) {}
+                                                Dim status As GdPictureStatus = oGDPicturePDF.ExtractEmbeddedFile(index, oFileData)
+
+                                                If status = GdPictureStatus.OK Then
+                                                    oResults.Add(New AttachmentResult() With {
+                                                                    .FileContents = oFileData,
+                                                                    .FileName = oFileName
+                                                                 })
+                                                Else
+                                                    Logger.Error("The embedded file [{0}] has failed to extract. Status: {1}", oFileName, oGDPicturePDF.GetStat().ToString())
+                                                    Continue For
+                                                End If
+
+                                                'If status = GdPictureStatus.OK Then
+                                                '    Dim oVersionedName = Filesystem.GetVersionedFilename(oFileName)
+                                                '    Dim oTempName As String = Path.Combine(Path.GetTempPath(), oVersionedName)
+                                                '    Using oFileStream As New FileStream(oTempName, FileMode.OpenOrCreate)
+                                                '        oFileStream.Write(oFileData, 0, oFileData.Length)
+                                                '    End Using
+
+                                                '    oResults.Add(New FileInfo(oTempName))
+                                                'Else
+                                                '    Logger.Error("The embedded file [{0}] has failed to extract. Status: {1}", oFileName, oGDPicturePDF.GetStat().ToString())
+                                                '    Continue For
+                                                'End If
+                                            Else
+                                                Logger.Error("An error occurred getting the file size for [{0}]. Status: {1}", oFileName, oGDPicturePDF.GetStat().ToString())
+                                                Continue For
+                                            End If
+                                        Else
+                                            Logger.Warn("File [{0}] was skipped because its extension [{1}] is not allowed.", oFileName, oExtension)
+                                            Continue For
+                                        End If
+                                    Else
+                                        Logger.Debug("File [{0}] was skipped because its name indicates the invoice data file.", oFileName)
+                                        Continue For
+                                    End If
+                                Else
+                                    Logger.Error("An error occurred getting the file name for [{0}]. Status: {1}", oFileName, oGDPicturePDF.GetStat().ToString())
+                                    Continue For
+                                End If
+                            Next
+                        End If
+                    Else
+                        Logger.Error("An error occurred getting the number of embedded files. Status: {0}", oGDPicturePDF.GetStat().ToString())
+                        Return Nothing
+                    End If
+                Else
+                    Logger.Error("The file [{0}] can't be loaded.", FileName)
+                    Return Nothing
+                End If
+            End Using
+
+            Return oResults
+        Catch ex As Exception
+            Logger.Warn("Unexpected Error while Extracting attachments from File [{0}]", FileName)
+            Logger.Error(ex)
+            Return Nothing
+        End Try
+    End Function
+End Class

Modules.Interfaces/ZUGFeRDInterface/PropertyValues.vb

@@ -0,0 +1,313 @@
+Imports System.Reflection
+Imports System.Text.RegularExpressions
+Imports DigitalData.Modules.Logging
+
+Public Class PropertyValues
+    Private _logger As Logger
+    Private _logConfig As LogConfig
+
+    Private _indexPattern = "\((\d+)\)"
+    Private _indexRegex As New Regex(_indexPattern)
+
+    Public Sub New(LogConfig As LogConfig)
+        _logConfig = LogConfig
+        _logger = LogConfig.GetLogger()
+    End Sub
+
+    Public Class CheckPropertyValuesResult
+        Public MissingProperties As New List(Of String)
+        Public ValidProperties As List(Of ValidProperty)
+    End Class
+
+    Public Class ValidProperty
+        Public MessageId As String
+        Public TableName As String
+
+        Public GroupCounter As Integer = -1
+
+        Public Description As String
+        Public Value As String
+    End Class
+
+    Public Function CheckPropertyValues(Document As CrossIndustryDocumentType, PropertyMap As Dictionary(Of String, XmlItemProperty), MessageId As String) As CheckPropertyValuesResult
+        Dim oGlobalGroupCounter = 0
+        Dim oMissingProperties As New List(Of String)
+        Dim oResult As New CheckPropertyValuesResult()
+
+        ' PropertyMap items with `IsGrouped = False` are handled normally
+        Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = PropertyMap.
+            Where(Function(Item) Item.Value.IsGrouped = True).
+            ToDictionary(Function(Item) Item.Key,
+                         Function(Item) Item.Value)
+
+        _logger.Debug("Found {0} default properties.", oDefaultProperties.Count)
+
+        ' PropertyMap items with `IsGrouped = True` are grouped by group scope
+        Dim oGroupedProperties = PropertyMap.
+            Where(Function(Item) Item.Value.IsGrouped = True).
+            ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope
+                     Function(Item) Item)
+
+        _logger.Debug("Found {0} properties grouped in {1} group(s)", PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count)
+        ' Iterate through groups to get group scope and group items
+        For Each oGroup In oGroupedProperties
+            Dim oGroupScope As String = oGroup.Key
+            Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object))
+            Dim oRowCount = 0
+
+            _logger.Debug("Fetching Property values for group {0}.", oGroupScope)
+
+            ' get properties as a nested object, see `oPropertyList`
+            For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup
+                Dim oPropertyValues As List(Of Object)
+
+                Try
+                    oPropertyValues = GetPropValue(Document, oProperty.Key)
+                Catch ex As Exception
+                    _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope)
+                    _logger.Error(ex)
+                    oPropertyValues = New List(Of Object)
+                End Try
+
+                ' Flatten result value
+                oPropertyValues = GetFinalPropValue(oPropertyValues)
+
+                ' Add to list
+                oPropertyList.Add(oProperty.Value, oPropertyValues)
+
+                ' check the first batch of values to determine the row count
+                If oRowCount = 0 Then
+                    oRowCount = oPropertyValues.Count
+                End If
+            Next
+
+            ' Structure of oPropertyList
+            ' [ # Propertyname       # Row 1              # Row 2
+            '   PositionsMenge:      [BilledQuantity1,    BilledQuantity2, ...],
+            '   PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...],
+            '   ...
+            ' ]
+            For oRowIndex = 0 To oRowCount - 1
+                _logger.Debug("Processing row {0}", oRowIndex)
+
+                For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList
+                    Dim oTableName As String = oColumn.Key.TableName
+                    Dim oPropertyDescription As String = oColumn.Key.Description
+                    Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1
+
+                    ' Returns nothing if oColumn.Value contains an empty list
+                    Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex)
+
+                    _logger.Debug("Processing property {0}.", oPropertyDescription)
+
+                    If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
+                        If oColumn.Key.IsRequired Then
+                            _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription)
+                            oResult.MissingProperties.Add(oPropertyDescription)
+                        Else
+                            _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription)
+                        End If
+
+                        oPropertyValue = String.Empty
+                    End If
+
+                    _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue)
+
+                    oResult.ValidProperties.Add(New ValidProperty() With {
+                        .MessageId = MessageId,
+                        .Description = oPropertyDescription,
+                        .Value = oPropertyValue,
+                        .GroupCounter = oRowCounter,
+                        .TableName = oTableName
+                    })
+                Next
+            Next
+
+            oGlobalGroupCounter += oRowCount
+        Next
+
+        ' Iterate through default properties
+        For Each oItem As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties
+            Dim oPropertyValueList As List(Of Object)
+            Dim oPropertyDescription As String = oItem.Value.Description
+            Dim oPropertyValue As Object = Nothing
+            Dim oTableName = oItem.Value.TableName
+
+            Try
+                oPropertyValueList = GetPropValue(Document, oItem.Key)
+            Catch ex As Exception
+                _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, oItem.Value.GroupScope)
+                _logger.Error(ex)
+                oPropertyValueList = New List(Of Object)
+            End Try
+
+            Try
+                If IsNothing(oPropertyValueList) Then
+                    oPropertyValue = Nothing
+                ElseIf TypeOf oPropertyValueList Is List(Of Object) Then
+                    Select Case oPropertyValueList.Count
+                        Case 0
+                            oPropertyValue = Nothing
+                        Case Else
+                            Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object))
+                            oPropertyValue = oList.Item(0)
+
+                            ' This should hopefully show config errors
+                            If TypeOf oPropertyValue Is List(Of Object) Then
+                                _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription)
+                                oPropertyValue = Nothing
+                            End If
+                    End Select
+                End If
+            Catch ex As Exception
+                _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription)
+                _logger.Error(ex)
+                oPropertyValue = Nothing
+            End Try
+
+            If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
+                If oItem.Value.IsRequired Then
+                    _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription)
+                    oResult.MissingProperties.Add(oPropertyDescription)
+                    Continue For
+                Else
+                    _logger.Debug("Property [{0}]  is empty or not found. Skipping.", oPropertyDescription)
+                    Continue For
+                End If
+            End If
+
+            oResult.ValidProperties.Add(New ValidProperty() With {
+                .MessageId = MessageId,
+                .Description = oPropertyDescription,
+                .Value = oPropertyValue,
+                .TableName = oTableName
+            })
+        Next
+
+        Return oResult
+    End Function
+
+    Public Function GetPropValue(Obj As Object, PropertyName As String) As List(Of Object)
+        Dim oNameParts As String() = PropertyName.Split("."c)
+
+        If IsNothing(Obj) Then
+            _logger.Debug("`Obj` is Nothing. Exiting.")
+            Return New List(Of Object)
+        End If
+
+
+        If oNameParts.Length = 1 Then
+            Dim oPropInfo As PropertyInfo = Obj.GetType().GetProperty(PropertyName)
+
+            If IsNothing(oPropInfo) Then
+                _logger.Debug("Property {0} does not exist.", PropertyName)
+                Return New List(Of Object)
+            Else
+                Dim oPropValue = oPropInfo.GetValue(Obj, Nothing)
+                Return New List(Of Object) From {oPropValue}
+            End If
+        End If
+
+        For Each oPart As String In oNameParts
+            Dim oType As Type = Obj.GetType()
+            Dim oPartName = oPart
+            Dim oIndex As Integer = Nothing
+            Dim oHasIndex As Boolean = HasIndex(oPartName)
+
+            If oHasIndex Then
+                oPartName = StripIndex(oPart)
+                oIndex = GetIndex(oPart)
+            End If
+
+            Dim oInfo As PropertyInfo = oType.GetProperty(oPartName)
+
+            If IsNothing(oInfo) OrElse IsNothing(oInfo.GetValue(Obj, Nothing)) Then
+                _logger.Debug("Property {0} does not exist.", oPartName)
+                Return New List(Of Object)
+            End If
+
+            Obj = oInfo.GetValue(Obj, Nothing)
+
+            If oHasIndex Then
+                Obj = Obj(0)
+            End If
+
+            If IsArray(Obj) And Not oHasIndex Then
+                Dim oCurrentPart As String = oPart
+                Dim oSplitString As String() = New String() {oCurrentPart & "."}
+                Dim oPathFragments = PropertyName.Split(oSplitString, StringSplitOptions.None)
+                Dim oResults As New List(Of Object)
+
+                ' if path has no more subitems, return an empty list
+                If oPathFragments.Length = 1 Then
+                    Return oResults
+                End If
+
+                For Each oArrayItem In Obj
+                    Dim oResult As List(Of Object) = GetPropValue(oArrayItem, oPathFragments(1))
+
+                    If Not IsNothing(oResult) Then
+                        oResults.Add(oResult)
+                    End If
+                Next
+
+                Return oResults
+            End If
+        Next
+
+        Return New List(Of Object) From {Obj}
+    End Function
+
+    Public Function GetFinalPropValue(List As List(Of Object)) As List(Of Object)
+        Dim oResult As New List(Of Object)
+
+        For Each Item In List
+            Dim oItemValue = DoGetFinalPropValue(Item)
+
+            If Not IsNothing(oItemValue) Then
+                oResult.Add(oItemValue)
+            End If
+        Next
+
+        Return oResult
+    End Function
+
+    Private Function DoGetFinalPropValue(Value As Object) As String
+        If TypeOf Value Is List(Of Object) Then
+            Dim oList = DirectCast(Value, List(Of Object))
+            Dim oCount = oList.Count
+
+            Select Case oCount
+                Case 0
+                    Return Nothing
+                Case Else
+                    Return DoGetFinalPropValue(oList.First())
+            End Select
+
+            Return DoGetFinalPropValue(Value)
+        Else
+            Return Value.ToString
+        End If
+    End Function
+
+    Private Function GetIndex(Prop As String) As Integer
+        If Regex.IsMatch(Prop, _indexPattern) Then
+            Dim oMatch = _indexRegex.Match(Prop)
+            Dim oGroup = oMatch.Groups.Item(1)
+            Dim oValue = oGroup.Value
+
+            Return Integer.Parse(oValue)
+        End If
+
+        Return Nothing
+    End Function
+
+    Private Function StripIndex(Prop As String) As String
+        Return Regex.Replace(Prop, _indexPattern, "")
+    End Function
+
+    Private Function HasIndex(Prop As String) As Boolean
+        Return Regex.IsMatch(Prop, _indexPattern)
+    End Function
+
+End Class

Modules.Interfaces/ZUGFeRDInterface/XmlItemProperty.vb

@@ -0,0 +1,7 @@
+Public Class XmlItemProperty
+    Public TableName As String
+    Public Description As String
+    Public IsRequired As Boolean
+    Public IsGrouped As Boolean
+    Public GroupScope As String
+End Class