Imports System.IO Imports System.Text.RegularExpressions Imports DigitalData.Modules.Logging Imports DigitalData.Modules.Language Imports MultiTool.Common.Exceptions Imports MultiTool.Common.Templates Imports MultiTool.Common.Winline Imports MultiTool.Common.Winline.Entities Imports MultiTool.Common.Constants Namespace Documents Public Class DocumentLoader Inherits BaseClass Private ReadOnly Winline As WinlineData Private ReadOnly MappingConfig As MappingConfig Private ReadOnly TemplateConfig As TemplateConfig Public Property Files As New List(Of Document) Public Property FilesTotal As Integer = 0 Public Property FilesLoaded As Integer = 0 Public Event FileLoadComplete As EventHandler(Of FileLoadInfo) Public Event FileLoadProgress As EventHandler(Of FileLoadProgressInfo) Public Class FileLoadInfo Public FilesLoaded As Integer Public FilesTotal As Integer Public Sub New(pTotal As Integer, pLoaded As Integer) FilesTotal = pTotal FilesLoaded = pLoaded End Sub End Class Public Class FileLoadProgressInfo Inherits FileLoadInfo Public RunningFunction As String Public Sub New(pTotal As Integer, pLoaded As Integer) MyBase.New(pTotal, pLoaded) End Sub End Class Public Sub New(pLogConfig As LogConfig, pWinline As WinlineData, pMappingConfig As MappingConfig, pTemplateConfig As TemplateConfig) MyBase.New(pLogConfig) Winline = pWinline MappingConfig = pMappingConfig TemplateConfig = pTemplateConfig End Sub Public Async Function LoadFiles(pTemplate As Template, pMandator As Mandator) As Task(Of Boolean) Logger.Info("Loading files from directory [{0}]", pTemplate.InputDirectory) Files.Clear() Try Dim oDirectory As New DirectoryInfo(pTemplate.InputDirectory) Dim oFiles = oDirectory.GetFiles() FilesTotal = oFiles.Count Logger.Debug("Found [{0}] files in directory [{1}]", oFiles.Count, oDirectory) For Each oFile In oFiles Try Dim oDocument = Await LoadFile(oFile, pTemplate, pMandator) Files.Add(oDocument) FilesLoaded = Files.Count Dim oInfo As New FileLoadInfo(FilesTotal, FilesLoaded) RaiseEvent FileLoadComplete(Me, oInfo) Catch ex As MissingAttributeException Logger.Error(ex) Throw New DocumentLoaderException($"Missing Attribute '{ex.Message}' in File '{oFile.Name}'") Catch ex As Exception Logger.Error(ex) Throw ex End Try Next Return True Catch ex As Exception Logger.Error(ex) Throw ex End Try End Function Public Async Function LoadFile(pFileInfo As FileInfo, pTemplate As Template, pMandator As Mandator) As Task(Of Document) Dim oFileList As New List(Of FileInfo) From {pFileInfo} Logger.Info("Loading file [{0}]", pFileInfo.Name) Try Return Await oFileList. Select(AddressOf WrapFileInfo). Select(Function(d) IncludeSchema(d, pTemplate)). Select(Function(d) LoadDocumentData(d, pTemplate, TemplateConfig)). Select(Async Function(d) Await MatchDataFromWinLine(d, Winline.Mandators, pMandator, pTemplate)). SingleOrDefault() Catch ex As Exception Logger.Error(ex) Throw ex End Try End Function Public Sub ReplaceDocument(pDocument As Document) Dim oIndex = Files.IndexOf(pDocument) Files.Item(oIndex) = pDocument End Sub Private Function IncludeSchema(pDocument As Document, pTemplate As Template) As Document pDocument.Schema = pTemplate Return pDocument End Function ''' ''' Loads a single document from the FullName Property in the Document Object ''' ''' ''' A document might look like this: ''' ''' ''' ''' ''' ''' Private Function LoadDocumentData(pDocument As Document, pTemplate As Template, pTemplateConfig As TemplateConfig) As Document Dim oText As String = IO.File.ReadAllText(pDocument.FullName) Dim oDoc = XDocument.Parse(oText) Dim oRootElement As XElement = XmlData.GetElement(oDoc, "MESOWebService") If oRootElement Is Nothing Then Throw New MalformedXmlException("Datei enthält kein MESOWebService-Element") End If Dim oTemplateName = XmlData.GetElementAttribute(oRootElement, "Template") If oTemplateName Is Nothing Then Throw New MalformedXmlException("Datei enthält kein Template-Attribut") End If Dim oTemplateType = XmlData.GetElementAttribute(oRootElement, "TemplateType") If oTemplateType Is Nothing Then Throw New MalformedXmlException("Datei enthält kein TemplateType-Attribut") End If Dim oOption = XmlData.GetElementAttribute(oRootElement, "option") If oOption Is Nothing Then Throw New MalformedXmlException("Datei enthält kein option-Attribut") End If Dim oPrintVoucher = XmlData.GetElementAttribute(oRootElement, "printVoucher") If oPrintVoucher Is Nothing Then Throw New MalformedXmlException("Datei enthält kein printVoucher-Attribut") End If ' The first level of Elements are the document Rows Dim oTopLevelElements As List(Of XElement) = oRootElement.Elements.ToList Dim oDocumentRows As New List(Of DocumentRow) Dim oRowSortKey As Integer = 0 ' TODO: Somehow add all fields in the correct order ' ' Right now, the method of ' - first the filled field from xml ' - then the rest from schema ' ' leads to unordered fields. For Each oTopLevelElement As XElement In oTopLevelElements Dim oColumnSortKey = 0 Dim oFields As New Dictionary(Of String, DocumentRow.FieldValue) Dim oSubElements = oTopLevelElement.Descendants().ToList() Dim oTable = pTemplate.Tables. Where(Function(t) t.Name = oTopLevelElement.Name). FirstOrDefault() For Each oColumn In oTable.Columns Dim oSubElement = oSubElements. Where(Function(e) e.Name = oColumn.Name). SingleOrDefault() If oSubElement IsNot Nothing Then Dim oRequired = oColumn.IsRequired Dim oValue = oSubElement.Value.Trim() ' TODO: Needed when we have time for date times 'If oTemplateField.DataType = Constants.ColumnType.Date Then ' Dim oDate = Date.ParseExact(oValue, "yyyy-MM-dd", CultureInfo.InvariantCulture) ' oValue = oDate.ToString("d") 'End If oFields.Add(oSubElement.Name.ToString, New DocumentRow.FieldValue With { .Original = oValue, .Final = oValue, .DataType = oColumn.DataType, .IsRequired = oRequired, .IsVirtual = oColumn.Config.IsVirtual, .SortKey = oColumnSortKey }) Else Dim oColumnError = FieldError.None If oColumn.Config?.IsRequired Then oColumnError = FieldError.MissingValue End If oFields.Add(oColumn.Name, New DocumentRow.FieldValue With { .[Error] = oColumnError, .SortKey = oColumnSortKey, .IsVirtual = oColumn.Config.IsVirtual }) End If oColumnSortKey += 1 Next ' Create a DocumentRow object for each Top Level Element Dim oRow = New DocumentRow With { .SortKey = oRowSortKey, .TableName = oTopLevelElement.Name.ToString, .Fields = oFields } oRowSortKey += 1 oDocumentRows.Add(oRow) Next ' Update the document pDocument.TemplateName = oTemplateName pDocument.TemplateType = oTemplateType pDocument.Option = oOption pDocument.PrintVoucher = oPrintVoucher pDocument.Rows = oDocumentRows Return pDocument End Function Private Async Function MatchDataFromWinLine(pDocument As Document, pMandators As List(Of Mandator), pMandator As Mandator, pTemplate As Template) As Task(Of Document) Dim oMandators As List(Of Mandator) = pMandators. Where(Function(m) m.IsWhitelisted = True). OrderBy(Function(m) m.Order). ToList() Dim oInfo As New FileLoadProgressInfo(FilesTotal, FilesLoaded) With {.RunningFunction = "Mandant finden"} RaiseEvent FileLoadProgress(Me, oInfo) Dim oMandator As Mandator = Nothing If pMandator IsNot Nothing Then oMandator = pMandator Else oMandator = Winline.FindMatchingMandatorFromOrder(pDocument) End If If oMandator Is Nothing Then Logger.Warn("Mandator not found for File [{0}]", pDocument.File.Name) Else ' Set mandator befor applying any functions that depend on a valid mandator pDocument.Mandator = oMandator RaiseEvent FileLoadProgress(Me, New FileLoadProgressInfo(FilesTotal, FilesLoaded) With { .RunningFunction = "Winline-Funktionen" }) pDocument = ApplyDefinedItemFunctionsForImport(pDocument, oMandator, pTemplate) pDocument = ApplyDynamicItemFunctionsForImport(pDocument, oMandator) RaiseEvent FileLoadProgress(Me, New FileLoadProgressInfo(FilesTotal, FilesLoaded) With { .RunningFunction = "Preis-Funktionen" }) ' These functions will only be applied if the document does not have errors pDocument = Await MaybeApplyPriceFunctions(pDocument, oMandator, pTemplate) RaiseEvent FileLoadProgress(Me, New FileLoadProgressInfo(FilesTotal, FilesLoaded) With { .RunningFunction = "Feld-Funktionen" }) ' This function needs to be the last one because ' it can relate to any previously set value ApplyFieldFunctionForImport(pDocument, oMandator, pTemplate) End If Return pDocument End Function ''' ''' Apply price calculation to the documents products ''' ''' This needs to be strictly seperated from `ApplyDefinedItemFunctionsForImport` ''' because prices can only be calculated if GLN and EAN functions were already (successfully) processed ''' Public Async Function MaybeApplyPriceFunctions(pDocument As Document, pMandator As Mandator, pTemplate As Template) As Task(Of Document) If pDocument.HasErrors Then Return pDocument End If For Each oRow As DocumentRow In pDocument.Rows Dim oTable = pTemplate.Tables.Where(Function(table) table.Name = oRow.TableName).SingleOrDefault() For Each oField In oRow.Fields If oTable Is Nothing Then Exit For End If Dim oColumn = oTable.Columns.Where(Function(c) c.Name = oField.Key).SingleOrDefault() If oColumn Is Nothing Then Continue For End If Dim oFunctionName = oColumn.Config.FunctionName Dim oFunctionParams = oColumn.Config.FunctionParams Dim oParamsDict = ParseFunctionParamsAsDict(oFunctionParams) If oFunctionName = Constants.FUNCTION_PRICE Then Await SetPrice(oRow, oField.Key, oParamsDict, pDocument, pMandator, pTemplate) End If Next Next Return pDocument End Function Private Function ApplyDefinedItemFunctionsForImport(pDocument As Document, pMandator As Mandator, pTemplate As Template) As Document For Each oRow As DocumentRow In pDocument.Rows Dim oTable = pTemplate.Tables.Where(Function(table) table.Name = oRow.TableName).SingleOrDefault() For Each oField In oRow.Fields If oTable Is Nothing Then Exit For End If Dim oColumn = oTable.Columns.Where(Function(c) c.Name = oField.Key).SingleOrDefault() If oColumn Is Nothing Then Continue For End If Dim oFunctionName = oColumn.Config.FunctionName Dim oFunctionParams = oColumn.Config.FunctionParams Dim oParamsDict = ParseFunctionParamsAsDict(oFunctionParams) If oFunctionName = Constants.FUNCTION_GLN Then SetAccountByGLN(oRow, pMandator, oField.Key, Nothing, oParamsDict) End If If oFunctionName = Constants.FUNCTION_EAN Then SetArticleByEAN(oRow, pMandator, oField.Key) End If Next Next Return pDocument End Function Private Function ApplyFieldFunctionForImport(pDocument As Document, pMandator As Mandator, pTemplate As Template) As Document For Each oRow As DocumentRow In pDocument.Rows Dim oTable = pDocument.Schema.Tables.Where(Function(table) table.Name = oRow.TableName).SingleOrDefault() For Each oField In oRow.Fields If oTable Is Nothing Then Logger.Warn("Table [{0}] was not found in the Schema. Exiting.", oRow.TableName) Exit For End If Dim oColumn = oTable.Columns.Where(Function(c) c.Name = oField.Key).SingleOrDefault() If oColumn Is Nothing Then Logger.Warn("Column [{0}] was not found in Table [{0}]. Skipping.", oField.Key, oTable.Name) Continue For End If Dim oFunctionName = oColumn.Config.FunctionName Dim oFunctionParams = oColumn.Config.FunctionParams Dim oParamsDict = ParseFunctionParamsAsDict(oFunctionParams) If oFunctionName = Constants.FUNCTION_FIELD Then Try Logger.Debug("Applying function FIELD to field [{0}]", oField.Key) Dim oParam = oParamsDict.FirstOrDefault() If IsNothing(oParam) Then Logger.Warn("Function FIELD needs exactly one parameter. Skipping") Continue For End If Dim oFieldName = oParam.Key Dim oSubKey = oParam.Value Dim oReferencedField = oRow.Fields. Where(Function(field) field.Key = oFieldName). FirstOrDefault() If IsNothing(oReferencedField) = False Then Dim oRawValue = oReferencedField.Value?.GetValue(oSubKey) Dim oValue As String = Utils.NotNull(oRawValue, String.Empty) If oValue <> String.Empty Then oField.Value.Final = oValue End If Else Logger.Warn("Referenced Field [{0}] was not found. Skipping.", oFieldName) Continue For End If Catch ex As Exception Logger.Warn("Function FIELD could not be applied to field [{0}]. Skipping.", oField.Key) Continue For End Try End If Next Next Return pDocument End Function ''' ''' Execute Mappings defined in TBMT_MAPPING_CONFIG, ''' for example mapping Article Numbers to Winline Mandators ''' Private Function ApplyDynamicItemFunctionsForImport(pDocument As Document, pMandator As Mandator) As Document ' We only want the mapping config for things in the xml file. ' that excludes things like setting the mandator. Dim oFilteredMappingConfig = MappingConfig.Items. Where(Function(item) item.DestinationItem <> String.Empty). ToList() For Each oMapping As MappingConfigItem In oFilteredMappingConfig ' Get Source Value Dim oField As KeyValuePair(Of String, DocumentRow.FieldValue) = pDocument.Rows. SelectMany(Function(row) row.Fields). Where(Function(field) field.Key = oMapping.SourceItem). FirstOrDefault() ' Test on Regex Dim oRegex As New Regex(oMapping.SourceRegex) If oRegex.IsMatch(oField.Value.Final) Then pDocument.Rows. SelectMany(Function(row) row.Fields). Where(Function(field) field.Key = oMapping.DestinationItem). SetValue(Sub(field) field.Value.Final = oMapping.DestinationValue End Sub) Else ' don't do anything End If Next Return pDocument End Function Private Async Function SetPrice(pRow As DocumentRow, pPriceField As String, oFieldMap As Dictionary(Of String, String), pDocument As Document, pMandator As Mandator, pTemplate As Template) As Task Dim oPriceItem As DocumentRow.FieldValue = pRow.Fields.GetOrDefault(pPriceField) ' These fields are fetched from the current row Dim oArticleNumberField As String = oFieldMap.GetOrDefault("Article", Nothing) Dim oArticleNumber As String = pRow.Fields.Item(oArticleNumberField).Final Dim oQuantityField As String = oFieldMap.GetOrDefault("Quantity", Nothing) Dim oQuantity As Integer = 1 If Integer.TryParse(pRow.Fields.Item(oQuantityField).Final, oQuantity) = False Then Logger.Warn("Value for parameter Quantity could not be parsed. Setting to 1.") End If ' These fields a fetched from the head row, ie. the first row Dim oAccountNumberField As String = oFieldMap.GetOrDefault("Account", Nothing) Dim oAccountNumber = pDocument.GetFieldValue(oAccountNumberField) Dim oDocumentDateField As String = oFieldMap.GetOrDefault("DocumentDate", Nothing) Dim oDocumentDate = pDocument.GetFieldValue(oDocumentDateField) Dim oDocumentKindField As String = oFieldMap.GetOrDefault("DocumentKind", Nothing) Dim oDocumentKind As Integer = 0 If Integer.TryParse(pDocument.GetFieldValue(oDocumentKindField), oDocumentKind) = False Then Logger.Warn("Value for parameter DocumentKind could not be parsed. Setting to 0.") End If ' TODO: Add Field Names as Constants ' TODO: Check for missing values ' TODO: This function should not be hardcoded, but be replaced with virtual field or something.. ' It is related to customer SCHAUM and nothing general Dim oWaitingDays As Integer = Await Winline.TryGetWaitingDays(oDocumentKind, pMandator) ' END TODO Dim oArticlePrice As Double = Await Winline.TryGetArticlePrice(oArticleNumber, oAccountNumber, oQuantity, oDocumentDate, pMandator, pTemplate, oWaitingDays) If oArticlePrice > 0 Then oPriceItem.External = oArticlePrice oPriceItem.Final = oArticlePrice Logger.Info("Price for Item [{0}] set to [{1}]", pPriceField, oArticlePrice) Else Logger.Warn("Price for Item [{0}] could not be found!", pPriceField) End If End Function Private Sub SetArticleByEAN(pRow As DocumentRow, pMandator As Mandator, pArticleField As String) Dim oNumberItem As DocumentRow.FieldValue = pRow.Fields.GetOrDefault(pArticleField) Dim oArticleNumber = Winline.TryGetArticleNumber(oNumberItem.Original, pMandator) If oArticleNumber IsNot Nothing Then oNumberItem.External = oArticleNumber oNumberItem.Final = oArticleNumber Else oNumberItem.Error = FieldError.ArticleNotFound End If End Sub Private Sub SetAccountByGLN(oRow As DocumentRow, pMandator As Mandator, pNumberField As String, pNameField As String, pParams As Dictionary(Of String, String)) ' Try to read the Account number (which is a GLN really) and account Name Dim oNumberItem As DocumentRow.FieldValue = oRow.Fields.GetOrDefault(pNumberField) Dim oNameItem As DocumentRow.FieldValue = oRow.Fields.GetOrDefault(pNameField) Dim oContainsAccountName As Boolean = Not IsNothing(oNameItem) If oNumberItem Is Nothing Then Exit Sub End If ' Try to find an account that matches the GLN Dim oAlternateField = pParams.GetOrDefault("AltField", String.Empty) Dim oAccount = Winline.TryGetAccount(oNumberItem.Original, pMandator, "c260", oAlternateField) ' If an account was found, set it for External and Final value If oAccount IsNot Nothing Then oNumberItem.External = oAccount.Id oNumberItem.Final = oAccount.Id If oContainsAccountName Then oNameItem.External = oAccount.Name oNameItem.Final = oAccount.Name Else ' TODO: What to to if name field is missing or not set? 'oRow.Fields.Add(pNameField, New DocumentRow.FieldValue() With { ' .External = oAccount.Name, ' .Final = oAccount.Name '}) End If Else oNumberItem.Error = FieldError.AccountNotFound End If End Sub Private Function WrapFileInfo(pFileInfo As FileInfo) As Document Return New Document With {.File = pFileInfo} End Function Private Function ParseFunctionParamsAsDict(pParams As String) As Dictionary(Of String, String) Dim oParamsDict As New Dictionary(Of String, String) If pParams <> String.Empty Then Dim oParamList = pParams.Split("|").ToList() For Each oParam In oParamList Dim oParamSplit = oParam.Split("=") oParamsDict.Add(oParamSplit(0), oParamSplit(1)) Next End If Return oParamsDict End Function End Class End Namespace