Zugferd: prepare loading different specs per document schema version

Interfaces/ZUGFeRDInterface.vb

@@ -12,6 +12,13 @@ Public Class ZUGFeRDInterface
     Private ReadOnly _logger As Logger
     Private ReadOnly _Options As ZugferdOptions
 
+    ' These constants define the specification markers for the different
+    ' zugferd document schema versions. These markers need to be used to 
+    ' define the property map in the database (column SPECIFICATION).
+    Public Const ZUGFERD_SPEC_DEFAULT = "DEFAULT"
+    Public Const ZUGFERD_SPEC_10 = "ZUGFERD_10"
+    Public Const ZUGFERD_SPEC_2x = "ZUGFERD_2x"
+
     Private ReadOnly ValidFilenames As New List(Of String) From {
         PDFEmbeds.ZUGFERD_XML_FILENAME.ToUpper,
         PDFEmbeds.FACTUR_X_XML_FILENAME_DE.ToUpper,
@@ -39,6 +46,13 @@ Public Class ZUGFeRDInterface
         Public Property AllowZugferd_2_x_Schema As Boolean = True
     End Class
 
+    Public Class ZugferdResult
+        Public Property DataFileName As String
+        Public Property XPathObject As XPathDocument
+        Public Property SchemaObject As Object
+        Public Property Specification As String
+    End Class
+
     ''' <summary>
     ''' Create a new instance of ZUGFeRDInterface
     ''' </summary>
@@ -91,14 +105,14 @@ Public Class ZUGFeRDInterface
     ''' </summary>
     ''' <param name="Path"></param>
     ''' <exception cref="ZUGFeRDExecption"></exception>
-    Public Function ExtractZUGFeRDFileWithGDPicture(Path As String) As Tuple(Of String, Object)
+    Public Function ExtractZUGFeRDFileWithGDPicture(Path As String) As ZugferdResult
-        Dim oXmlDocument = ValidateZUGFeRDFileWithGDPicture(Path)
+        Dim oResult = ValidateZUGFeRDFileWithGDPicture(Path)
 
-        If IsNothing(oXmlDocument.Item2) Then
+        If IsNothing(oResult.SchemaObject) Then
             Throw New ZUGFeRDExecption(ErrorType.NoZugferd, "Datei ist keine ZUGFeRD Datei.")
         End If
 
-        Return SerializeZUGFeRDDocument(oXmlDocument)
+        Return SerializeZUGFeRDDocument(oResult)
     End Function
 
     ''' <summary>
@@ -106,14 +120,14 @@ Public Class ZUGFeRDInterface
     ''' </summary>
     ''' <param name="Stream"></param>
     ''' <exception cref="ZUGFeRDExecption"></exception>
-    Public Function ExtractZUGFeRDFileWithGDPicture(Stream As Stream) As Tuple(Of String, Object)
+    Public Function ExtractZUGFeRDFileWithGDPicture(Stream As Stream) As ZugferdResult
-        Dim oXmlDocument = ValidateZUGFeRDFileWithGDPicture(Stream)
+        Dim oResult = ValidateZUGFeRDFileWithGDPicture(Stream)
 
-        If IsNothing(oXmlDocument.Item2) Then
+        If IsNothing(oResult.SchemaObject) Then
             Throw New ZUGFeRDExecption(ErrorType.NoZugferd, "Datei ist keine ZUGFeRD Datei.")
         End If
 
-        Return SerializeZUGFeRDDocument(oXmlDocument)
+        Return SerializeZUGFeRDDocument(oResult)
     End Function
 
     ''' <summary>
@@ -122,7 +136,7 @@ Public Class ZUGFeRDInterface
     ''' <param name="pStream"></param>
     ''' <exception cref="ZUGFeRDExecption"></exception>
     ''' <returns>The embedded xml data as an XPath document</returns>
-    Public Function ValidateZUGFeRDFileWithGDPicture(pStream As Stream) As Tuple(Of String, XPathDocument)
+    Public Function ValidateZUGFeRDFileWithGDPicture(pStream As Stream) As ZugferdResult
         Dim oEmbedExtractor = New PDFEmbeds(_logConfig)
 
         Try
@@ -150,7 +164,7 @@ Public Class ZUGFeRDInterface
     ''' <param name="pPath"></param>
     ''' <exception cref="ZUGFeRDExecption"></exception>
     ''' <returns>The embedded xml data as an XPath document</returns>
-    Public Function ValidateZUGFeRDFileWithGDPicture(pPath As String) As Tuple(Of String, XPathDocument)
+    Public Function ValidateZUGFeRDFileWithGDPicture(pPath As String) As ZugferdResult
         Dim oEmbedExtractor = New PDFEmbeds(_logConfig)
 
         Try
@@ -172,7 +186,7 @@ Public Class ZUGFeRDInterface
         End Try
     End Function
 
-    Private Function HandleEmbeddedFiles(pResults As List(Of PDFEmbeds.EmbeddedFile)) As Tuple(Of String, XPathDocument)
+    Private Function HandleEmbeddedFiles(pResults As List(Of PDFEmbeds.EmbeddedFile)) As ZugferdResult
         If pResults Is Nothing Then
             Throw New ZUGFeRDExecption(ErrorType.NoZugferd, "Datei ist keine ZUGFeRD Datei, weil die Attachments nicht gelesen werden konnten.")
         End If
@@ -202,7 +216,10 @@ Public Class ZUGFeRDInterface
 
         Try
             Using oStream As New MemoryStream(oAllowedResult.FileContents)
-                Return New Tuple(Of String, XPathDocument)(oAllowedResult.FileName, New XPathDocument(oStream))
+                Return New ZugferdResult With {
+                    .DataFileName = oAllowedResult.FileName,
+                    .XPathObject = New XPathDocument(oStream)
+                }
             End Using
 
         Catch ex As ZUGFeRDExecption
@@ -216,47 +233,54 @@ Public Class ZUGFeRDInterface
         End Try
     End Function
 
-    Public Function SerializeZUGFeRDDocument(pDocument As Tuple(Of String, XPathDocument)) As Tuple(Of String, Object)
+    Public Function SerializeZUGFeRDDocument(pResult As ZugferdResult) As ZugferdResult
         Try
-            Dim oNavigator As XPathNavigator = pDocument.Item2.CreateNavigator()
+            Dim oNavigator As XPathNavigator = pResult.XPathObject.CreateNavigator()
             Dim oReader As XmlReader
-            Dim oResult = Nothing
 
-            Dim oAllowedTypes As New List(Of Type)
+            Dim oObject As Object = Nothing
+            Dim oSpecification As String = Nothing
+
+            Dim oAllowedTypes As New Dictionary(Of String, Type)
 
             If _Options.AllowZugferd_1_0_Schema Then
-                oAllowedTypes.Add(GetType(ZUGFeRD.Version1_0.CrossIndustryDocumentType))
+                oAllowedTypes.Add(ZUGFERD_SPEC_10, GetType(ZUGFeRD.Version1_0.CrossIndustryDocumentType))
             End If
 
             If _Options.AllowZugferd_2_x_Schema Then
-                oAllowedTypes.AddRange(New List(Of Type) From {
+                oAllowedTypes.Add(ZUGFERD_SPEC_2x, GetType(ZUGFeRD.Version2_0.CrossIndustryInvoiceType))
-                    GetType(ZUGFeRD.Version2_0.CrossIndustryInvoiceType),
+                oAllowedTypes.Add(ZUGFERD_SPEC_2x, GetType(ZUGFeRD.Version2_1_1.CrossIndustryInvoiceType))
-                    GetType(ZUGFeRD.Version2_1_1.CrossIndustryInvoiceType),
+                oAllowedTypes.Add(ZUGFERD_SPEC_2x, GetType(ZUGFeRD.Version2_2_FacturX.CrossIndustryInvoiceType))
-                    GetType(ZUGFeRD.Version2_2_FacturX.CrossIndustryInvoiceType)
-                })
             End If
 
             For Each oType In oAllowedTypes
-                _logger.Debug("Trying Type [{0}]", oType.FullName)
+                Dim oTypeName As String = oType.Value.FullName
-                Dim oSerializer As New XmlSerializer(oType)
+                Dim oSerializer As New XmlSerializer(oType.Value)
+                _logger.Debug("Trying Type [{0}]", oTypeName)
 
                 Try
                     oReader = oNavigator.ReadSubtree()
-                    oResult = oSerializer.Deserialize(oReader)
+
-                    _logger.Debug("Serializing with type [{0}] succeeded", oType.FullName)
+                    oObject = oSerializer.Deserialize(oReader)
+                    oSpecification = oType.Key
+
+                    _logger.Debug("Serializing with type [{0}] succeeded", oTypeName)
                     Exit For
                 Catch ex As Exception
-                    _logger.Debug("Serializing with type [{0}] failed", oType.FullName)
+                    _logger.Debug("Serializing with type [{0}] failed", oTypeName)
                     _logger.Debug(ex.Message)
                     _logger.Error(ex.InnerException?.Message)
                 End Try
             Next
 
-            If oResult Is Nothing Then
+            If oObject Is Nothing Then
                 Throw New ApplicationException("No Types matched the given document. Document could not be serialized.")
             End If
 
-            Return oResult
+            pResult.Specification = oSpecification
+            pResult.SchemaObject = oObject
+
+            Return pResult
 
         Catch ex As Exception
             _logger.Error(ex)

Interfaces/ZUGFeRDInterface/XmlItemProperty.vb

@@ -1,8 +1,14 @@
 Public Class XmlItemProperty
+    Public IsRequired As Boolean
+    Public IsGrouped As Boolean
+
     Public TableName As String
     Public TableColumn As String
     Public Description As String
-    Public IsRequired As Boolean
-    Public IsGrouped As Boolean
     Public GroupScope As String
+
+    ''' <summary>
+    ''' Document version, eg. ZUGFeRD Schema version
+    ''' </summary>
+    Public Specification As String
 End Class

Jobs/Jobs.vbproj

@@ -116,9 +116,9 @@
     </Compile>
   </ItemGroup>
   <ItemGroup>
-    <Reference Include="DigitalData.Modules.Language, Version=1.6.2.0, Culture=neutral, processorArchitecture=MSIL">
+    <Reference Include="DigitalData.Modules.Language, Version=1.7.0.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\Interfaces\bin\Debug\DigitalData.Modules.Language.dll</HintPath>
+      <HintPath>..\Language\bin\Debug\DigitalData.Modules.Language.dll</HintPath>
     </Reference>
     <Reference Include="FirebirdSql.Data.FirebirdClient, Version=7.5.0.0, Culture=neutral, PublicKeyToken=3750abcc3150b00c, processorArchitecture=MSIL">
       <HintPath>..\packages\FirebirdSql.Data.FirebirdClient.7.5.0\lib\net452\FirebirdSql.Data.FirebirdClient.dll</HintPath>

Jobs/ZUGFeRD/EmailFunctions.vb

@@ -1,5 +1,6 @@
 Imports DigitalData.Modules.Logging
 Imports DigitalData.Modules.Database
+Imports DigitalData.Modules.Language
 Imports System.Data
 Imports System.IO
 Imports System.Data.SqlClient
@@ -164,10 +165,19 @@ Public Class EmailFunctions
             _logger.Debug("Got Email Data for FileId {0}", MessageId)
             oRow = oDatatable.Rows.Item(oDatatable.Rows.Count - 1)
 
+            Dim oFromDefault = String.Format("No Sender found for ({0})", MessageId)
+            Dim oFrom = oRow.ItemEx("EMAIL_FROM", oFromDefault)
+
+            Dim oAttachmentDefault = String.Format("No Attachment found for ({0})", MessageId)
+            Dim oAttachment = oRow.ItemEx("EMAIL_ATTMT1", oAttachmentDefault)
+
+            Dim oSubjectDefault = String.Format("No Subject found for ({0})", MessageId)
+            Dim oSubject = oRow.ItemEx("EMAIL_SUBJECT", oSubjectDefault)
+
             Return New EmailData() With {
-                .From = oRow.Item("EMAIL_FROM"),
+                .From = oFrom,
-                .Attachment = oRow.Item("EMAIL_ATTMT1"),
+                .Attachment = oAttachment,
-                .Subject = oRow.Item("EMAIL_SUBJECT")
+                .Subject = oSubject
             }
         Catch ex As Exception
             _logger.Warn("Could not fetch Email Data for FileId {0}", MessageId)

Jobs/ZUGFeRD/EmailStrings.vb

@@ -20,9 +20,9 @@
 
     Public Const EMAIL_MD5_ERROR = "<p>Die von Ihnen gesendete Rechnung wurde bereits von unserem System verarbeitet.</p>"
 
-    Public Const EMAIL_TOO_MUCH_FERDS = "<p>Ihre Email enthielt mehr als ein ZUGFeRD-Dokument.</p>"
+    Public Const EMAIL_TOO_MUCH_FERDS = "<p>In Ihrer Email ({0}) sind mehr als ein ZUGFeRD Dokument enthalten. Bitte prüfen Sie Rechnung an Anhänge. Nur eine Rechnung darf das ZUGFeRD-Format enthalten</p>"
 
-    Public Const EMAIL_NO_FERDS = "<p>Ihre Email enthielt keine ZUGFeRD-Dokumente.</p>"
+    Public Const EMAIL_NO_FERDS = "<p>Ihre Email ({0}) enthielt keine ZUGFeRD-Dokumente.</p>"
 
     Public Const EMAIL_FILE_SIZE_REACHED = "
         <p>Die von Ihnen gesendete Rechnung oder einer der Rechnungs-Anhänge überschreitet die erlaubte Größe von <strong>{0} MB</strong>.</p>
@@ -32,13 +32,13 @@
     "
 
     Public Const EMAIL_INVALID_DOCUMENT = "
-        <p>Ihre Email enthielt ein ZUGFeRD Dokument, welches aber inkorrekt formatiert wurde.</p>
+        <p>Ihre Email ({0}) enthielt ein ZUGFeRD Dokument, welches aber inkorrekt formatiert wurde.</p>
         <p>Mögliche Gründe für ein inkorrektes Format:<ul>
             <li>Betrags-Werte weisen ungültiges Format auf (25,01 anstatt 25.01)</li>
         </ul></p>
     "
 
     Public Const EMAIL_UNSUPPORTED_DOCUMENT = "
-        <p>Ihre Email enthielt ein ZUGFeRD Dokument ({0}), welches zur Zeit noch nicht unsterstützt wird.</p>
+        <p>Ihre Email ({0}) enthielt ein ZUGFeRD Dokument ({1}), welches zur Zeit noch nicht freigeschaltet ist.</p>
     "
 End Class

Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb

@@ -206,7 +206,9 @@ Public Class ImportZUGFeRDFiles
                                 '   different versions of ZUGFeRD and the type is unknown at compile-time.
                                 ' 17.11.2022: oDocument is now a Tuple of (String, Object), to be able to return the filename
                                 '   of the extracted xml file.
-                                Dim oDocument As Tuple(Of String, Object)
+                                ' 21.12.2022: oDocument is now an object of type ZugferdResult to be able to save
+                                '   the new meta data, ie. the type of schema (zugferd version)
+                                Dim oDocument As ZUGFeRDInterface.ZugferdResult
 
                                 ' Start a global group counter for each file
                                 Dim oGlobalGroupCounter = 0
@@ -283,7 +285,8 @@ Public Class ImportZUGFeRDFiles
                                 ' Check the document against the configured property map and return:
                                 ' - a List of valid properties 
                                 ' - a List of missing properties
-                                Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument, oArgs.PropertyMap, oMessageId)
+                                Dim oPropertyMap = GetPropertyMapFor(oArgs, oDocument)
+                                Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument.SchemaObject, oPropertyMap, oMessageId)
 
                                 _logger.Info("Properties checked: [{0}] missing properties / [{1}] valid properties found.", oCheckResult.MissingProperties.Count, oCheckResult.ValidProperties.Count)
 
@@ -398,8 +401,9 @@ Public Class ImportZUGFeRDFiles
                             ' That 's why we set it to String.Empty here.
                             Create_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but unsupported format", oFBTransaction)
 
-                            Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, ex.XmlFile)
                             Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
+                            Dim oBody As String = String.Format(EmailStrings.EMAIL_UNSUPPORTED_DOCUMENT, oEmailData.Subject, ex.XmlFile)
+
                             _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "UnsupportedFerdException", _EmailOutAccountId, oArgs.NamePortal)
                             AddRejectedState(oMessageId, "UnsupportedFerdException", "Nicht unterstütztes Datenformat", "", oSQLTransaction)
 
@@ -410,8 +414,9 @@ Public Class ImportZUGFeRDFiles
                             ' That 's why we set it to String.Empty here.
                             Create_HistoryEntry(oMessageId, String.Empty, "REJECTED - ZUGFeRD yes but incorrect format", oFBTransaction)
 
-                            Dim oBody = EmailStrings.EMAIL_INVALID_DOCUMENT
                             Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
+                            Dim oBody = String.Format(EmailStrings.EMAIL_INVALID_DOCUMENT, oEmailData.Subject)
+
                             _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "InvalidFerdException", _EmailOutAccountId, oArgs.NamePortal)
                             AddRejectedState(oMessageId, "InvalidFerdException", "Inkorrekte Formate", "", oSQLTransaction)
 
@@ -420,8 +425,9 @@ Public Class ImportZUGFeRDFiles
 
                             Create_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - More than one ZUGFeRD-document in email", oFBTransaction)
 
-                            Dim oBody = EmailStrings.EMAIL_TOO_MUCH_FERDS
                             Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
+                            Dim oBody = String.Format(EmailStrings.EMAIL_TOO_MUCH_FERDS, oEmailData.Subject)
+
                             _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "TooMuchFerdsException", _EmailOutAccountId, oArgs.NamePortal)
                             AddRejectedState(oMessageId, "TooMuchFerdsException", "Email enthielt mehr als ein ZUGFeRD-Dokument", "", oSQLTransaction)
 
@@ -430,8 +436,9 @@ Public Class ImportZUGFeRDFiles
 
                             Create_HistoryEntry(oMessageId, oMD5CheckSum, "REJECTED - no ZUGFeRD-Document in email", oFBTransaction)
 
-                            Dim oBody = EmailStrings.EMAIL_NO_FERDS
                             Dim oEmailData = MoveAndRenameEmailToRejected(oArgs, oMessageId)
+                            Dim oBody = String.Format(EmailStrings.EMAIL_NO_FERDS, oEmailData.Subject)
+
                             _email.AddToEmailQueueMSSQL(oMessageId, oBody, oEmailData, "NoFerdsException", _EmailOutAccountId, oArgs.NamePortal)
                             AddRejectedState(oMessageId, "NoFerdsException", " Email enthielt keine ZUGFeRD-Dokumente", "", oSQLTransaction)
 
@@ -576,6 +583,26 @@ Public Class ImportZUGFeRDFiles
         End Try
     End Sub
 
+    Private Function GetPropertyMapFor(pWorkerArgs As WorkerArgs, pSpecification As String) As Dictionary(Of String, XmlItemProperty)
+        Dim oPropertyMap = DoGetPropertyMapFor(pWorkerArgs, pSpecification)
+
+        _logger.Debug("Found [{0}] Properties for Specification [{1}].", oPropertyMap.Count, pSpecification)
+
+        ' If no properties were found, fall back to the default specification before giving up
+        If oPropertyMap.Count = 0 Then
+            _logger.Warn("No Properties found for Specification [{0}]. Loading default property map!", pSpecification)
+            Return DoGetPropertyMapFor(pWorkerArgs, ZUGFeRDInterface.ZUGFERD_SPEC_DEFAULT)
+        End If
+
+        Return oPropertyMap
+    End Function
+
+    Private Function DoGetPropertyMapFor(pWorkerArgs As WorkerArgs, pSpecification As String) As Dictionary(Of String, XmlItemProperty)
+        Return pWorkerArgs.PropertyMap.
+            Where(Function(kv) kv.Value.Specification = pSpecification).
+            ToDictionary(Function(kv) kv.Key, Function(kv) kv.Value)
+    End Function
+
     Private Sub MoveFiles(
                          Args As WorkerArgs,
                          MessageId As String,