Imports System.IO
Imports System.Text.RegularExpressions
Imports DigitalData.Modules.Logging
Imports DigitalData.Modules.Language
Imports MultiTool.Common.Exceptions
Imports MultiTool.Common.Templates
Imports MultiTool.Common.Winline
Imports MultiTool.Common.Winline.Entities
Imports MultiTool.Common.Constants
Namespace Documents
Public Class DocumentLoader
Inherits BaseClass
Private ReadOnly Winline As WinlineData
Private ReadOnly MappingConfig As MappingConfig
Private ReadOnly TemplateConfig As TemplateConfig
Public Property Files As New List(Of Document)
Public Property FilesTotal As Integer = 0
Public Property FilesLoaded As Integer = 0
Public Event FileLoadComplete As EventHandler(Of FileLoadInfo)
Public Event FileLoadProgress As EventHandler(Of FileLoadProgressInfo)
Public Class FileLoadInfo
Public FilesLoaded As Integer
Public FilesTotal As Integer
Public Sub New(pTotal As Integer, pLoaded As Integer)
FilesTotal = pTotal
FilesLoaded = pLoaded
End Sub
End Class
Public Class FileLoadProgressInfo
Inherits FileLoadInfo
Public RunningFunction As String
Public Sub New(pTotal As Integer, pLoaded As Integer)
MyBase.New(pTotal, pLoaded)
End Sub
End Class
Public Sub New(pLogConfig As LogConfig, pWinline As WinlineData, pMappingConfig As MappingConfig, pTemplateConfig As TemplateConfig)
MyBase.New(pLogConfig)
Winline = pWinline
MappingConfig = pMappingConfig
TemplateConfig = pTemplateConfig
End Sub
Public Async Function LoadFiles(pTemplate As Template, pMandator As Mandator) As Task(Of Boolean)
Logger.Info("Loading files from directory [{0}]", pTemplate.InputDirectory)
Files.Clear()
Try
Dim oDirectory As New DirectoryInfo(pTemplate.InputDirectory)
Dim oFiles = oDirectory.GetFiles()
FilesTotal = oFiles.Count
Logger.Debug("Found [{0}] files in directory [{1}]", oFiles.Count, oDirectory)
For Each oFile In oFiles
Try
Dim oDocument = Await LoadFile(oFile, pTemplate, pMandator)
Files.Add(oDocument)
FilesLoaded = Files.Count
Dim oInfo As New FileLoadInfo(FilesTotal, FilesLoaded)
RaiseEvent FileLoadComplete(Me, oInfo)
Catch ex As MissingAttributeException
Logger.Error(ex)
Throw New DocumentLoaderException($"Missing Attribute '{ex.Message}' in File '{oFile.Name}'")
Catch ex As Exception
Logger.Error(ex)
Throw ex
End Try
Next
Return True
Catch ex As Exception
Logger.Error(ex)
Throw ex
End Try
End Function
Public Async Function LoadFile(pFileInfo As FileInfo, pTemplate As Template, pMandator As Mandator) As Task(Of Document)
Dim oFileList As New List(Of FileInfo) From {pFileInfo}
Logger.Info("Loading file [{0}]", pFileInfo.Name)
Try
Return Await oFileList.
Select(AddressOf WrapFileInfo).
Select(Function(d) IncludeSchema(d, pTemplate)).
Select(Function(d) LoadDocumentData(d, pTemplate, TemplateConfig)).
Select(Async Function(d) Await MatchDataFromWinLine(d, Winline.Mandators, pMandator, pTemplate)).
SingleOrDefault()
Catch ex As Exception
Logger.Error(ex)
Throw ex
End Try
End Function
Public Sub ReplaceDocument(pDocument As Document)
Dim oIndex = Files.IndexOf(pDocument)
Files.Item(oIndex) = pDocument
End Sub
Private Function IncludeSchema(pDocument As Document, pTemplate As Template) As Document
pDocument.Schema = pTemplate
Return pDocument
End Function
'''
''' Loads a single document from the FullName Property in the Document Object
'''
'''
''' A document might look like this:
'''
'''
'''
'''
'''
'''
Private Function LoadDocumentData(pDocument As Document, pTemplate As Template, pTemplateConfig As TemplateConfig) As Document
Dim oText As String = IO.File.ReadAllText(pDocument.FullName)
Dim oDoc = XDocument.Parse(oText)
Dim oRootElement As XElement = XmlData.GetElement(oDoc, "MESOWebService")
If oRootElement Is Nothing Then
Throw New MalformedXmlException("Datei enthält kein MESOWebService-Element")
End If
Dim oTemplateName = XmlData.GetElementAttribute(oRootElement, "Template")
If oTemplateName Is Nothing Then
Throw New MalformedXmlException("Datei enthält kein Template-Attribut")
End If
Dim oTemplateType = XmlData.GetElementAttribute(oRootElement, "TemplateType")
If oTemplateType Is Nothing Then
Throw New MalformedXmlException("Datei enthält kein TemplateType-Attribut")
End If
Dim oOption = XmlData.GetElementAttribute(oRootElement, "option")
If oOption Is Nothing Then
Throw New MalformedXmlException("Datei enthält kein option-Attribut")
End If
Dim oPrintVoucher = XmlData.GetElementAttribute(oRootElement, "printVoucher")
If oPrintVoucher Is Nothing Then
Throw New MalformedXmlException("Datei enthält kein printVoucher-Attribut")
End If
' The first level of Elements are the document Rows
Dim oTopLevelElements As List(Of XElement) = oRootElement.Elements.ToList
Dim oDocumentRows As New List(Of DocumentRow)
Dim oRowSortKey As Integer = 0
' TODO: Somehow add all fields in the correct order
'
' Right now, the method of
' - first the filled field from xml
' - then the rest from schema
'
' leads to unordered fields.
For Each oTopLevelElement As XElement In oTopLevelElements
Dim oColumnSortKey = 0
Dim oFields As New Dictionary(Of String, DocumentRow.FieldValue)
Dim oSubElements = oTopLevelElement.Descendants().ToList()
Dim oTable = pTemplate.Tables.
Where(Function(t) t.Name = oTopLevelElement.Name).
FirstOrDefault()
For Each oColumn In oTable.Columns
Dim oSubElement = oSubElements.
Where(Function(e) e.Name = oColumn.Name).
SingleOrDefault()
If oSubElement IsNot Nothing Then
Dim oRequired = oColumn.IsRequired
Dim oValue = oSubElement.Value.Trim()
' TODO: Needed when we have time for date times
'If oTemplateField.DataType = Constants.ColumnType.Date Then
' Dim oDate = Date.ParseExact(oValue, "yyyy-MM-dd", CultureInfo.InvariantCulture)
' oValue = oDate.ToString("d")
'End If
oFields.Add(oSubElement.Name.ToString, New DocumentRow.FieldValue With {
.Original = oValue,
.Final = oValue,
.DataType = oColumn.DataType,
.IsRequired = oRequired,
.IsVirtual = oColumn.Config.IsVirtual,
.SortKey = oColumnSortKey
})
Else
Dim oColumnError = FieldError.None
If oColumn.Config?.IsRequired Then
oColumnError = FieldError.MissingValue
End If
oFields.Add(oColumn.Name, New DocumentRow.FieldValue With {
.[Error] = oColumnError,
.SortKey = oColumnSortKey,
.IsVirtual = oColumn.Config.IsVirtual
})
End If
oColumnSortKey += 1
Next
' Create a DocumentRow object for each Top Level Element
Dim oRow = New DocumentRow With {
.SortKey = oRowSortKey,
.TableName = oTopLevelElement.Name.ToString,
.Fields = oFields
}
oRowSortKey += 1
oDocumentRows.Add(oRow)
Next
' Update the document
pDocument.TemplateName = oTemplateName
pDocument.TemplateType = oTemplateType
pDocument.Option = oOption
pDocument.PrintVoucher = oPrintVoucher
pDocument.Rows = oDocumentRows
Return pDocument
End Function
Private Async Function MatchDataFromWinLine(pDocument As Document, pMandators As List(Of Mandator), pMandator As Mandator, pTemplate As Template) As Task(Of Document)
Dim oMandators As List(Of Mandator) = pMandators.
Where(Function(m) m.IsWhitelisted = True).
OrderBy(Function(m) m.Order).
ToList()
Dim oInfo As New FileLoadProgressInfo(FilesTotal, FilesLoaded) With {.RunningFunction = "Mandant finden"}
RaiseEvent FileLoadProgress(Me, oInfo)
Dim oMandator As Mandator = Nothing
If pMandator IsNot Nothing Then
oMandator = pMandator
Else
oMandator = Winline.FindMatchingMandatorFromOrder(pDocument)
End If
If oMandator Is Nothing Then
Logger.Warn("Mandator not found for File [{0}]", pDocument.File.Name)
Else
' Set mandator befor applying any functions that depend on a valid mandator
pDocument.Mandator = oMandator
RaiseEvent FileLoadProgress(Me, New FileLoadProgressInfo(FilesTotal, FilesLoaded) With {
.RunningFunction = "Winline-Funktionen"
})
pDocument = ApplyDefinedItemFunctionsForImport(pDocument, oMandator, pTemplate)
pDocument = ApplyDynamicItemFunctionsForImport(pDocument, oMandator)
RaiseEvent FileLoadProgress(Me, New FileLoadProgressInfo(FilesTotal, FilesLoaded) With {
.RunningFunction = "Preis-Funktionen"
})
' These functions will only be applied if the document does not have errors
pDocument = Await MaybeApplyPriceFunctions(pDocument, oMandator, pTemplate)
RaiseEvent FileLoadProgress(Me, New FileLoadProgressInfo(FilesTotal, FilesLoaded) With {
.RunningFunction = "Feld-Funktionen"
})
' This function needs to be the last one because
' it can relate to any previously set value
ApplyFieldFunctionForImport(pDocument, oMandator, pTemplate)
End If
Return pDocument
End Function
'''
''' Apply price calculation to the documents products
'''
''' This needs to be strictly seperated from `ApplyDefinedItemFunctionsForImport`
''' because prices can only be calculated if GLN and EAN functions were already (successfully) processed
'''
Public Async Function MaybeApplyPriceFunctions(pDocument As Document, pMandator As Mandator, pTemplate As Template) As Task(Of Document)
If pDocument.HasErrors Then
Return pDocument
End If
For Each oRow As DocumentRow In pDocument.Rows
Dim oTable = pTemplate.Tables.Where(Function(table) table.Name = oRow.TableName).SingleOrDefault()
For Each oField In oRow.Fields
If oTable Is Nothing Then
Exit For
End If
Dim oColumn = oTable.Columns.Where(Function(c) c.Name = oField.Key).SingleOrDefault()
If oColumn Is Nothing Then
Continue For
End If
Dim oFunctionName = oColumn.Config.FunctionName
Dim oFunctionParams = oColumn.Config.FunctionParams
Logger.Debug("Running Function: [{0}]", oFunctionName)
Logger.Debug("With Parameters: [{0}]", oFunctionParams)
Dim oParamsDict = ParseFunctionParamsAsDict(oFunctionParams)
If oFunctionName = Constants.FUNCTION_PRICE Then
Await SetPrice(oRow, oField.Key, oParamsDict, pDocument, pMandator, pTemplate)
End If
Next
Next
Return pDocument
End Function
Private Function ApplyDefinedItemFunctionsForImport(pDocument As Document, pMandator As Mandator, pTemplate As Template) As Document
For Each oRow As DocumentRow In pDocument.Rows
Dim oTable = pTemplate.Tables.Where(Function(table) table.Name = oRow.TableName).SingleOrDefault()
For Each oField In oRow.Fields
If oTable Is Nothing Then
Exit For
End If
Dim oColumn = oTable.Columns.Where(Function(c) c.Name = oField.Key).SingleOrDefault()
If oColumn Is Nothing Then
Continue For
End If
Dim oFunctionName = oColumn.Config.FunctionName
Dim oFunctionParams = oColumn.Config.FunctionParams
Dim oParamsDict = ParseFunctionParamsAsDict(oFunctionParams)
If oFunctionName = Constants.FUNCTION_GLN Then
SetAccountByGLN(oRow, pMandator, oField.Key, Nothing, oParamsDict)
End If
If oFunctionName = Constants.FUNCTION_EAN Then
SetArticleByEAN(oRow, pMandator, oField.Key)
End If
Next
Next
Return pDocument
End Function
Private Function ApplyFieldFunctionForImport(pDocument As Document, pMandator As Mandator, pTemplate As Template) As Document
For Each oRow As DocumentRow In pDocument.Rows
Dim oTable = pDocument.Schema.Tables.Where(Function(table) table.Name = oRow.TableName).SingleOrDefault()
For Each oField In oRow.Fields
If oTable Is Nothing Then
Logger.Warn("Table [{0}] was not found in the Schema. Exiting.", oRow.TableName)
Exit For
End If
Dim oColumn = oTable.Columns.Where(Function(c) c.Name = oField.Key).SingleOrDefault()
If oColumn Is Nothing Then
Logger.Warn("Column [{0}] was not found in Table [{0}]. Skipping.", oField.Key, oTable.Name)
Continue For
End If
Dim oFunctionName = oColumn.Config.FunctionName
Dim oFunctionParams = oColumn.Config.FunctionParams
Dim oParamsDict = ParseFunctionParamsAsDict(oFunctionParams)
If oFunctionName = Constants.FUNCTION_FIELD Then
Try
Logger.Debug("Applying function FIELD to field [{0}]", oField.Key)
Dim oParam = oParamsDict.FirstOrDefault()
If IsNothing(oParam) Then
Logger.Warn("Function FIELD needs exactly one parameter. Skipping")
Continue For
End If
Dim oFieldName = oParam.Key
Dim oSubKey = oParam.Value
Dim oReferencedField = oRow.Fields.
Where(Function(field) field.Key = oFieldName).
FirstOrDefault()
If IsNothing(oReferencedField) = False Then
Dim oRawValue = oReferencedField.Value?.GetValue(oSubKey)
Dim oValue As String = Utils.NotNull(oRawValue, String.Empty)
If oValue <> String.Empty Then
oField.Value.Final = oValue
End If
Else
Logger.Warn("Referenced Field [{0}] was not found. Skipping.", oFieldName)
Continue For
End If
Catch ex As Exception
Logger.Warn("Function FIELD could not be applied to field [{0}]. Skipping.", oField.Key)
Continue For
End Try
End If
Next
Next
Return pDocument
End Function
'''
''' Execute Mappings defined in TBMT_MAPPING_CONFIG,
''' for example mapping Article Numbers to Winline Mandators
'''
Private Function ApplyDynamicItemFunctionsForImport(pDocument As Document, pMandator As Mandator) As Document
' We only want the mapping config for things in the xml file.
' that excludes things like setting the mandator.
Dim oFilteredMappingConfig = MappingConfig.Items.
Where(Function(item) item.DestinationItem <> String.Empty).
ToList()
For Each oMapping As MappingConfigItem In oFilteredMappingConfig
' Get Source Value
Dim oField As KeyValuePair(Of String, DocumentRow.FieldValue) = pDocument.Rows.
SelectMany(Function(row) row.Fields).
Where(Function(field) field.Key = oMapping.SourceItem).
FirstOrDefault()
' Test on Regex
Dim oRegex As New Regex(oMapping.SourceRegex)
If oRegex.IsMatch(oField.Value.Final) Then
pDocument.Rows.
SelectMany(Function(row) row.Fields).
Where(Function(field) field.Key = oMapping.DestinationItem).
SetValue(Sub(field)
field.Value.Final = oMapping.DestinationValue
End Sub)
Else
' don't do anything
End If
Next
Return pDocument
End Function
Private Async Function SetPrice(pRow As DocumentRow, pPriceField As String, oFieldMap As Dictionary(Of String, String), pDocument As Document, pMandator As Mandator, pTemplate As Template) As Task
Dim oPriceItem As DocumentRow.FieldValue = pRow.Fields.GetOrDefault(pPriceField)
' These fields are fetched from the current row
Const PARAMETER_ARTICLE = "Article"
Dim oArticleNumberField As String = oFieldMap.GetOrDefault(PARAMETER_ARTICLE, Nothing)
If oArticleNumberField Is Nothing Then
Logger.Warn("Parameter '{0}' not found for Function PRICE", PARAMETER_ARTICLE)
End If
If pRow.Fields.ContainsKey(oArticleNumberField) = False Then
Logger.Warn("Value '{0}' for Parameter '{1}' not found for Function PRICE", oArticleNumberField, PARAMETER_ARTICLE)
End If
Dim oArticleNumber As String = pRow.Fields.Item(oArticleNumberField).Final
Const PARAMETER_QUANTITY = "Quantity"
Dim oQuantityField As String = oFieldMap.GetOrDefault(PARAMETER_QUANTITY, Nothing)
If oQuantityField Is Nothing Then
Logger.Warn("Parameter '{0}' not found for Function PRICE", PARAMETER_QUANTITY)
End If
Dim oQuantity As Integer = 1
If Integer.TryParse(pRow.Fields.Item(oQuantityField).Final, oQuantity) = False Then
Logger.Warn("Value for parameter '{0}' could not be parsed. Setting to 1.", PARAMETER_QUANTITY)
End If
' These fields a fetched from the head row, ie. the first row
Dim oAccountNumberField As String = oFieldMap.GetOrDefault("Account", Nothing)
Dim oAccountNumber = pDocument.GetFieldValue(oAccountNumberField)
Dim oDocumentDateField As String = oFieldMap.GetOrDefault("DocumentDate", Nothing)
Dim oDocumentDate = pDocument.GetFieldValue(oDocumentDateField)
Dim oDocumentKindField As String = oFieldMap.GetOrDefault("DocumentKind", Nothing)
Dim oDocumentKind As Integer = 0
If Integer.TryParse(pDocument.GetFieldValue(oDocumentKindField), oDocumentKind) = False Then
Logger.Warn("Value for parameter DocumentKind could not be parsed. Setting to 0.")
End If
' TODO: Add Field Names as Constants
' TODO: Check for missing values
' TODO: This function should not be hardcoded, but be replaced with virtual field or something..
' It is related to customer SCHAUM and nothing general
Dim oWaitingDays As Integer = Await Winline.TryGetWaitingDays(oDocumentKind, pMandator)
' END TODO
Dim oArticlePrice As Double = Await Winline.TryGetArticlePrice(oArticleNumber, oAccountNumber, oQuantity, oDocumentDate, pMandator, pTemplate, oWaitingDays)
If oArticlePrice > 0 Then
oPriceItem.External = oArticlePrice
oPriceItem.Final = oArticlePrice
Logger.Info("Price for Item [{0}] set to [{1}]", pPriceField, oArticlePrice)
Else
Logger.Warn("Price for Item [{0}] could not be found!", pPriceField)
End If
End Function
Private Sub SetArticleByEAN(pRow As DocumentRow, pMandator As Mandator, pArticleField As String)
Dim oNumberItem As DocumentRow.FieldValue = pRow.Fields.GetOrDefault(pArticleField)
Dim oArticleNumber = Winline.TryGetArticleNumber(oNumberItem.Original, pMandator)
If oArticleNumber IsNot Nothing Then
oNumberItem.External = oArticleNumber
oNumberItem.Final = oArticleNumber
Else
oNumberItem.Error = FieldError.ArticleNotFound
End If
End Sub
Private Sub SetAccountByGLN(oRow As DocumentRow, pMandator As Mandator, pNumberField As String, pNameField As String, pParams As Dictionary(Of String, String))
' Try to read the Account number (which is a GLN really) and account Name
Dim oNumberItem As DocumentRow.FieldValue = oRow.Fields.GetOrDefault(pNumberField)
Dim oNameItem As DocumentRow.FieldValue = oRow.Fields.GetOrDefault(pNameField)
Dim oContainsAccountName As Boolean = Not IsNothing(oNameItem)
If oNumberItem Is Nothing Then
Exit Sub
End If
' Try to find an account that matches the GLN
Dim oAlternateField = pParams.GetOrDefault("AltField", String.Empty)
Dim oAccount = Winline.TryGetAccount(oNumberItem.Original, pMandator, "c260", oAlternateField)
' If an account was found, set it for External and Final value
If oAccount IsNot Nothing Then
oNumberItem.External = oAccount.Id
oNumberItem.Final = oAccount.Id
If oContainsAccountName Then
oNameItem.External = oAccount.Name
oNameItem.Final = oAccount.Name
Else
' TODO: What to to if name field is missing or not set?
'oRow.Fields.Add(pNameField, New DocumentRow.FieldValue() With {
' .External = oAccount.Name,
' .Final = oAccount.Name
'})
End If
Else
oNumberItem.Error = FieldError.AccountNotFound
End If
End Sub
Private Function WrapFileInfo(pFileInfo As FileInfo) As Document
Return New Document With {.File = pFileInfo}
End Function
Private Function ParseFunctionParamsAsDict(pParams As String) As Dictionary(Of String, String)
Dim oParamsDict As New Dictionary(Of String, String)
If pParams <> String.Empty Then
Dim oParamList = pParams.Split("|").ToList()
For Each oParam In oParamList
Dim oParamSplit = oParam.Split("=")
oParamsDict.Add(oParamSplit(0), oParamSplit(1))
Next
End If
Return oParamsDict
End Function
End Class
End Namespace