start redesign for zugferd rest api
This commit is contained in:
Jonathan Jenne
@@ -287,45 +287,6 @@ Public Class ImportZUGFeRDFiles
|
||||
_logger.Error(ex)
|
||||
End Try
|
||||
End Sub
|
||||
Private Function GetMessageIdFromFileName(Filename As String) As String
|
||||
' Regex to find MessageId
|
||||
' See also: https://stackoverflow.com/questions/3968500/regex-to-validate-a-message-id-as-per-rfc2822
|
||||
Dim oRegex = "(((([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(""(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21\x23-\x5B\x5D-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*""))@(([a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*)|(\[(([\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21-\x5A\x5E-\x7E])|(\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*\]))))~.+"
|
||||
Dim oMatch = Regex.Match(Filename, oRegex, RegexOptions.IgnoreCase)
|
||||
|
||||
If oMatch.Success Then
|
||||
Dim oMessageId = oMatch.Groups(1).Value
|
||||
Return oMessageId
|
||||
Else
|
||||
Return Nothing
|
||||
End If
|
||||
End Function
|
||||
|
||||
Private Function GroupFiles(Files As List(Of FileInfo)) As Dictionary(Of String, List(Of FileInfo))
|
||||
Dim oGrouped As New Dictionary(Of String, List(Of FileInfo))
|
||||
|
||||
If Files.Count = 0 Then
|
||||
Return oGrouped
|
||||
End If
|
||||
|
||||
For Each oFile In Files
|
||||
Dim oMessageId = GetMessageIdFromFileName(oFile.Name)
|
||||
|
||||
If oMessageId Is Nothing Then
|
||||
_logger.Warn("File {0} did not have the required filename-format!", oMessageId)
|
||||
Continue For
|
||||
End If
|
||||
|
||||
If oGrouped.ContainsKey(oMessageId) Then
|
||||
oGrouped.Item(oMessageId).Add(oFile)
|
||||
Else
|
||||
oGrouped.Add(oMessageId, New List(Of FileInfo) From {oFile})
|
||||
End If
|
||||
Next
|
||||
|
||||
Return oGrouped
|
||||
End Function
|
||||
|
||||
|
||||
Public Sub Start(Arguments As Object) Implements IJob.Start
|
||||
Dim oArgs As WorkerArgs = Arguments
|
||||
@@ -374,7 +335,7 @@ Public Class ImportZUGFeRDFiles
|
||||
End If
|
||||
|
||||
' Group files by messageId
|
||||
Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = GroupFiles(oFiles)
|
||||
Dim oGrouped As Dictionary(Of String, List(Of FileInfo)) = _zugferd.FileGroup.GroupFiles(oFiles)
|
||||
|
||||
_logger.Info("Found {0} file groups", oGrouped.Count)
|
||||
|
||||
@@ -480,156 +441,191 @@ Public Class ImportZUGFeRDFiles
|
||||
' Since extraction went well, increase the amount of ZUGFeRD files
|
||||
oZUGFeRDCount += 1
|
||||
|
||||
' PropertyMap items with `IsGrouped = False` are handled normally
|
||||
Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = oArgs.PropertyMap.
|
||||
Where(Function(Item) Item.Value.IsGrouped = True).
|
||||
ToDictionary(Function(Item) Item.Key,
|
||||
Function(Item) Item.Value)
|
||||
' --- BEGIN Check Property Values
|
||||
|
||||
_logger.Debug("Found {0} default properties.", oDefaultProperties.Count)
|
||||
'' PropertyMap items with `IsGrouped = False` are handled normally
|
||||
'Dim oDefaultProperties As Dictionary(Of String, XmlItemProperty) = oArgs.PropertyMap.
|
||||
' Where(Function(Item) Item.Value.IsGrouped = True).
|
||||
' ToDictionary(Function(Item) Item.Key,
|
||||
' Function(Item) Item.Value)
|
||||
|
||||
' PropertyMap items with `IsGrouped = True` are grouped by group scope
|
||||
Dim oGroupedProperties = oArgs.PropertyMap.
|
||||
Where(Function(Item) Item.Value.IsGrouped = True).
|
||||
ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope
|
||||
Function(Item) Item)
|
||||
'_logger.Debug("Found {0} default properties.", oDefaultProperties.Count)
|
||||
|
||||
_logger.Debug("Found {0} properties grouped in {1} group(s)", oArgs.PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count)
|
||||
' Iterate through groups to get group scope and group items
|
||||
For Each oGroup In oGroupedProperties
|
||||
Dim oGroupScope As String = oGroup.Key
|
||||
Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object))
|
||||
Dim oRowCount = 0
|
||||
'' PropertyMap items with `IsGrouped = True` are grouped by group scope
|
||||
'Dim oGroupedProperties = oArgs.PropertyMap.
|
||||
' Where(Function(Item) Item.Value.IsGrouped = True).
|
||||
' ToLookup(Function(Item) Item.Value.GroupScope, ' Lookup key is group scope
|
||||
' Function(Item) Item)
|
||||
|
||||
_logger.Debug("Fetching Property values for group {0}.", oGroupScope)
|
||||
'_logger.Debug("Found {0} properties grouped in {1} group(s)", oArgs.PropertyMap.Count - oDefaultProperties.Count, oGroupedProperties.Count)
|
||||
'' Iterate through groups to get group scope and group items
|
||||
'For Each oGroup In oGroupedProperties
|
||||
' Dim oGroupScope As String = oGroup.Key
|
||||
' Dim oPropertyList As New Dictionary(Of XmlItemProperty, List(Of Object))
|
||||
' Dim oRowCount = 0
|
||||
|
||||
' get properties as a nested object, see `oPropertyList`
|
||||
For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup
|
||||
Dim oPropertyValues As List(Of Object)
|
||||
' _logger.Debug("Fetching Property values for group {0}.", oGroupScope)
|
||||
|
||||
Try
|
||||
oPropertyValues = oPropertyExtractor.GetPropValue(oDocument, oProperty.Key)
|
||||
Catch ex As Exception
|
||||
_logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope)
|
||||
_logger.Error(ex)
|
||||
oPropertyValues = New List(Of Object)
|
||||
End Try
|
||||
' ' get properties as a nested object, see `oPropertyList`
|
||||
' For Each oProperty As KeyValuePair(Of String, XmlItemProperty) In oGroup
|
||||
' Dim oPropertyValues As List(Of Object)
|
||||
|
||||
' Flatten result value
|
||||
oPropertyValues = oPropertyExtractor.GetFinalPropValue(oPropertyValues)
|
||||
' Try
|
||||
' oPropertyValues = oPropertyExtractor.GetPropValue(oDocument, oProperty.Key)
|
||||
' Catch ex As Exception
|
||||
' _logger.Warn("Unknown error occurred while fetching property [{0}] in group [{1}]:", oProperty.Value.Description, oGroupScope)
|
||||
' _logger.Error(ex)
|
||||
' oPropertyValues = New List(Of Object)
|
||||
' End Try
|
||||
|
||||
' Add to list
|
||||
oPropertyList.Add(oProperty.Value, oPropertyValues)
|
||||
' ' Flatten result value
|
||||
' oPropertyValues = oPropertyExtractor.GetFinalPropValue(oPropertyValues)
|
||||
|
||||
' check the first batch of values to determine the row count
|
||||
If oRowCount = 0 Then
|
||||
oRowCount = oPropertyValues.Count
|
||||
End If
|
||||
Next
|
||||
' ' Add to list
|
||||
' oPropertyList.Add(oProperty.Value, oPropertyValues)
|
||||
|
||||
' Structure of oPropertyList
|
||||
' [ # Propertyname # Row 1 # Row 2
|
||||
' PositionsMenge: [BilledQuantity1, BilledQuantity2, ...],
|
||||
' PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...],
|
||||
' ...
|
||||
' ]
|
||||
For oRowIndex = 0 To oRowCount - 1
|
||||
_logger.Debug("Processing row {0}", oRowIndex)
|
||||
' ' check the first batch of values to determine the row count
|
||||
' If oRowCount = 0 Then
|
||||
' oRowCount = oPropertyValues.Count
|
||||
' End If
|
||||
' Next
|
||||
|
||||
For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList
|
||||
Dim oTableName As String = oColumn.Key.TableName
|
||||
Dim oPropertyDescription As String = oColumn.Key.Description
|
||||
Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1
|
||||
' ' Structure of oPropertyList
|
||||
' ' [ # Propertyname # Row 1 # Row 2
|
||||
' ' PositionsMenge: [BilledQuantity1, BilledQuantity2, ...],
|
||||
' ' PositionsSteuersatz: [ApplicablePercent1, ApplicablePercent2, ...],
|
||||
' ' ...
|
||||
' ' ]
|
||||
' For oRowIndex = 0 To oRowCount - 1
|
||||
' _logger.Debug("Processing row {0}", oRowIndex)
|
||||
|
||||
' Returns nothing if oColumn.Value contains an empty list
|
||||
Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex)
|
||||
' For Each oColumn As KeyValuePair(Of XmlItemProperty, List(Of Object)) In oPropertyList
|
||||
' Dim oTableName As String = oColumn.Key.TableName
|
||||
' Dim oPropertyDescription As String = oColumn.Key.Description
|
||||
' Dim oRowCounter = oRowIndex + oGlobalGroupCounter + 1
|
||||
|
||||
_logger.Debug("Processing property {0}.", oPropertyDescription)
|
||||
' ' Returns nothing if oColumn.Value contains an empty list
|
||||
' Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex)
|
||||
|
||||
If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
|
||||
If oColumn.Key.IsRequired Then
|
||||
_logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription)
|
||||
oMissingProperties.Add(oPropertyDescription)
|
||||
Else
|
||||
_logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription)
|
||||
End If
|
||||
' _logger.Debug("Processing property {0}.", oPropertyDescription)
|
||||
|
||||
oPropertyValue = String.Empty
|
||||
End If
|
||||
' If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
|
||||
' If oColumn.Key.IsRequired Then
|
||||
' _logger.Warn("Property [{0}] is empty or not found but is required. Continuing with Empty String.", oPropertyDescription)
|
||||
' oMissingProperties.Add(oPropertyDescription)
|
||||
' Else
|
||||
' _logger.Debug("Property [{0}] is empty or not found. Continuing with Empty String.", oPropertyDescription)
|
||||
' End If
|
||||
|
||||
_logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue)
|
||||
' oPropertyValue = String.Empty
|
||||
' End If
|
||||
|
||||
Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})"
|
||||
_logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2} with RowCounter {3}", oPropertyDescription, oPropertyValue, oTableName, oRowCounter)
|
||||
' _logger.Debug("Property {0} has value '{1}'", oPropertyDescription, oPropertyValue)
|
||||
|
||||
' Insert into SQL Server
|
||||
If oArgs.InsertIntoSQLServer = True Then
|
||||
Dim oResult = _mssql.NewExecutenonQuery(oCommand)
|
||||
If oResult = False Then
|
||||
_logger.Warn("SQL Command was not successful. Check the log.")
|
||||
End If
|
||||
End If
|
||||
' Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}', {oRowCounter})"
|
||||
' _logger.Debug("Mapping Property {0} to value {1}. Will be inserted into table {2} with RowCounter {3}", oPropertyDescription, oPropertyValue, oTableName, oRowCounter)
|
||||
|
||||
' Insert into Firebird
|
||||
_firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
|
||||
Next
|
||||
Next
|
||||
' ' Insert into SQL Server
|
||||
' If oArgs.InsertIntoSQLServer = True Then
|
||||
' Dim oResult = _mssql.NewExecutenonQuery(oCommand)
|
||||
' If oResult = False Then
|
||||
' _logger.Warn("SQL Command was not successful. Check the log.")
|
||||
' End If
|
||||
' End If
|
||||
|
||||
oGlobalGroupCounter += oRowCount
|
||||
Next
|
||||
' ' Insert into Firebird
|
||||
' _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
|
||||
' Next
|
||||
' Next
|
||||
|
||||
' Iterate through default properties
|
||||
For Each Item As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties
|
||||
Dim oPropertyValueList As List(Of Object)
|
||||
Dim oPropertyDescription As String = Item.Value.Description
|
||||
Dim oPropertyValue As Object = Nothing
|
||||
' oGlobalGroupCounter += oRowCount
|
||||
'Next
|
||||
|
||||
Try
|
||||
oPropertyValueList = oPropertyExtractor.GetPropValue(oDocument, Item.Key)
|
||||
Catch ex As Exception
|
||||
_logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, Item.Value.GroupScope)
|
||||
_logger.Error(ex)
|
||||
oPropertyValueList = New List(Of Object)
|
||||
End Try
|
||||
'' Iterate through default properties
|
||||
'For Each Item As KeyValuePair(Of String, XmlItemProperty) In oDefaultProperties
|
||||
' Dim oPropertyValueList As List(Of Object)
|
||||
' Dim oPropertyDescription As String = Item.Value.Description
|
||||
' Dim oPropertyValue As Object = Nothing
|
||||
|
||||
Try
|
||||
If IsNothing(oPropertyValueList) Then
|
||||
oPropertyValue = Nothing
|
||||
ElseIf TypeOf oPropertyValueList Is List(Of Object) Then
|
||||
Select Case oPropertyValueList.Count
|
||||
Case 0
|
||||
oPropertyValue = Nothing
|
||||
Case Else
|
||||
Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object))
|
||||
oPropertyValue = oList.Item(0)
|
||||
' Try
|
||||
' oPropertyValueList = oPropertyExtractor.GetPropValue(oDocument, Item.Key)
|
||||
' Catch ex As Exception
|
||||
' _logger.Warn("Unknown error occurred while fetching property {0} in group {1}:", oPropertyDescription, Item.Value.GroupScope)
|
||||
' _logger.Error(ex)
|
||||
' oPropertyValueList = New List(Of Object)
|
||||
' End Try
|
||||
|
||||
' This should hopefully show config errors
|
||||
If TypeOf oPropertyValue Is List(Of Object) Then
|
||||
_logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription)
|
||||
oPropertyValue = Nothing
|
||||
End If
|
||||
End Select
|
||||
End If
|
||||
Catch ex As Exception
|
||||
_logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription)
|
||||
_logger.Error(ex)
|
||||
oPropertyValue = Nothing
|
||||
End Try
|
||||
' Try
|
||||
' If IsNothing(oPropertyValueList) Then
|
||||
' oPropertyValue = Nothing
|
||||
' ElseIf TypeOf oPropertyValueList Is List(Of Object) Then
|
||||
' Select Case oPropertyValueList.Count
|
||||
' Case 0
|
||||
' oPropertyValue = Nothing
|
||||
' Case Else
|
||||
' Dim oList As List(Of Object) = DirectCast(oPropertyValueList, List(Of Object))
|
||||
' oPropertyValue = oList.Item(0)
|
||||
|
||||
If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
|
||||
If Item.Value.IsRequired Then
|
||||
_logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription)
|
||||
oMissingProperties.Add(oPropertyDescription)
|
||||
Continue For
|
||||
Else
|
||||
_logger.Debug("Property [{0}] is empty or not found. Skipping.", oPropertyDescription)
|
||||
Continue For
|
||||
End If
|
||||
' ' This should hopefully show config errors
|
||||
' If TypeOf oPropertyValue Is List(Of Object) Then
|
||||
' _logger.Warn("Property with Description {0} may be configured incorrectly", oPropertyDescription)
|
||||
' oPropertyValue = Nothing
|
||||
' End If
|
||||
' End Select
|
||||
' End If
|
||||
' Catch ex As Exception
|
||||
' _logger.Warn("Unknown error occurred while processing property {0}:", oPropertyDescription)
|
||||
' _logger.Error(ex)
|
||||
' oPropertyValue = Nothing
|
||||
' End Try
|
||||
|
||||
' If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then
|
||||
' If Item.Value.IsRequired Then
|
||||
' _logger.Warn("Property {0} is empty but marked as required! Skipping.", oPropertyDescription)
|
||||
' oMissingProperties.Add(oPropertyDescription)
|
||||
' Continue For
|
||||
' Else
|
||||
' _logger.Debug("Property [{0}] is empty or not found. Skipping.", oPropertyDescription)
|
||||
' Continue For
|
||||
' End If
|
||||
' End If
|
||||
|
||||
' Dim oTableName = Item.Value.TableName
|
||||
' Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}')"
|
||||
' _logger.Debug("Mapping Property [{0}] to value [{1}] . Will be inserted into table {2}", oPropertyDescription, oPropertyValue, oTableName)
|
||||
|
||||
' ' Insert into SQL Server
|
||||
' If oArgs.InsertIntoSQLServer = True Then
|
||||
' Dim oResult = _mssql.NewExecutenonQuery(oCommand)
|
||||
' If oResult = False Then
|
||||
' _logger.Warn("SQL Command was not successful. Check the log.")
|
||||
' End If
|
||||
' End If
|
||||
|
||||
' ' Insert into Firebird
|
||||
' _firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
|
||||
'Next
|
||||
|
||||
'--- END Check Property Values
|
||||
|
||||
' Check the document against the configured property map and return:
|
||||
' - a List of valid properties
|
||||
' - a List of missing properties
|
||||
Dim oCheckResult = _zugferd.PropertyValues.CheckPropertyValues(oDocument, oArgs.PropertyMap, oMessageId)
|
||||
|
||||
If oCheckResult.MissingProperties.Count > 0 Then
|
||||
Throw New MissingValueException(oFile)
|
||||
End If
|
||||
|
||||
For Each oProperty In oCheckResult.ValidProperties
|
||||
Dim oGroupCounterValue = Nothing
|
||||
|
||||
If oProperty.GroupCounter > -1 Then
|
||||
oGroupCounterValue = oProperty.GroupCounter
|
||||
End If
|
||||
|
||||
Dim oTableName = Item.Value.TableName
|
||||
Dim oCommand = $"INSERT INTO {oTableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE) VALUES ('{oMessageId}', '{oPropertyDescription}', '{oPropertyValue}')"
|
||||
_logger.Debug("Mapping Property [{0}] to value [{1}] . Will be inserted into table {2}", oPropertyDescription, oPropertyValue, oTableName)
|
||||
Dim oCommand = $"INSERT INTO {oProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER) VALUES ('{oMessageId}', '{oProperty.Description}', '{oProperty.Value}', {oGroupCounterValue})"
|
||||
_logger.Debug("Mapping Property [{0}] to value [{1}] . Will be inserted into table {2}", oProperty.Description, oProperty.Value, oProperty.TableName)
|
||||
|
||||
' Insert into SQL Server
|
||||
If oArgs.InsertIntoSQLServer = True Then
|
||||
@@ -642,10 +638,6 @@ Public Class ImportZUGFeRDFiles
|
||||
' Insert into Firebird
|
||||
_firebird.ExecuteNonQueryWithConnection(oCommand, oConnection, Firebird.TransactionMode.ExternalTransaction, oTransaction)
|
||||
Next
|
||||
|
||||
If oMissingProperties.Count > 0 Then
|
||||
Throw New MissingValueException(oFile)
|
||||
End If
|
||||
Next
|
||||
|
||||
'Check if there are no ZUGFeRD files
|
||||
|
||||
Reference in New Issue
Block a user