Imports System.IO Imports DigitalData.Modules.Database Imports DigitalData.Modules.Logging Imports DigitalData.Modules.Language Namespace Templates Public Class TemplateLoader Inherits BaseClass Private ReadOnly ns As XNamespace = "http://www.w3.org/2001/XMLSchema" Public Property TemplateList As List(Of Template) Public Property TemplateConfiguration As New TemplateConfig Public Property MappingConfiguration As New MappingConfig Public Property MandatorConfiguration As New MandatorConfig Public Property GeneralConfiguration As New GeneralConfig Public Property FilterConfiguration As New FilterConfig Private Const SQL_TBMT_FILTERS = "SELECT * FROM [DD_ECM].[dbo].[VWMT_FILTERS]" Private Const SQL_VWMT_ITEMS = "SELECT * FROM [DD_ECM].[dbo].[VWMT_ITEMS] ORDER BY TEMPLATE_NAME, TABLE_NAME" Private Const SQL_VWMT_FUNCTIONS = "SELECT * FROM [DD_ECM].[dbo].[VWMT_FUNCTIONS] ORDER BY ITEM_ID, SEQUENCE" Private Const SQL_VWMT_MAPPING = "SELECT * FROM [DD_ECM].[dbo].[VWMT_MAPPING]" Private Const SQL_TBMT_MANDATORS = "SELECT * FROM [DD_ECM].[dbo].[TBMT_MANDATORS] ORDER BY ORDER_KEY" Private Const SQL_TBMT_CONFIG = "SELECT * FROM [DD_ECM].[dbo].[TBMT_CONFIG]" Private Const SQL_TBMT_TEMPLATES = "SELECT * FROM [DD_ECM].[dbo].[TBMT_TEMPLATES] WHERE ACTIVE = 1" Public Sub New(pLogConfig As LogConfig, pMSSQL As MSSQLServer) MyBase.New(pLogConfig, pMSSQL) End Sub Public Async Function LoadFilterConfiguration() As Task(Of Boolean) Try Dim oTable As DataTable = Await Database.GetDatatableAsync(SQL_TBMT_FILTERS) Dim oFilters As New List(Of FilterConfigItem) For Each oRow As DataRow In oTable.Rows Dim oFilter As New FilterConfigItem With { .Guid = oRow.Item("GUID"), .SQLCommand = oRow.ItemEx("SQL_COMMAND", String.Empty), .TableId = oRow.ItemEx("TABLE_ID", 0), .TableName = oRow.ItemEx("TABLE_NAME", String.Empty), .ColumnId = oRow.ItemEx("ITEM_ID", 0), .ColumnName = oRow.ItemEx("ITEM_NAME", String.Empty) } oFilters.Add(oFilter) Next FilterConfiguration.Items = oFilters Return True Catch ex As Exception Logger.Error(ex) Return False End Try End Function Public Async Function LoadTemplates() As Task(Of Boolean) Try Dim oTable As DataTable = Await Database.GetDatatableAsync(SQL_TBMT_TEMPLATES) Dim oTemplates As New List(Of Template) For Each oRow As DataRow In oTable.Rows Dim oTemplate As New Template With { .Guid = oRow.Item("GUID"), .Name = oRow.ItemEx("NAME", String.Empty), .Description = oRow.ItemEx("DESCRIPTION", String.Empty), .FileName = oRow.ItemEx("FILE_NAME", String.Empty), .IsImport = oRow.ItemEx("IS_IMPORT", True), .FinalSQL = oRow.ItemEx("FINAL_SQL", String.Empty), .Parameter1 = oRow.ItemEx("PARAMETER_1", String.Empty), .Parameter2 = oRow.ItemEx("PARAMETER_2", String.Empty) } Logger.Debug("Loading Template [{0}]", oTemplate.Name) oTemplates.Add(oTemplate) Next TemplateList = oTemplates Return True Catch ex As Exception Logger.Error(ex) Return False End Try End Function Public Async Function LoadGeneralConfiguration() As Task(Of Boolean) Try Dim oTable As DataTable = Await Database.GetDatatableAsync(SQL_TBMT_CONFIG) Dim oGeneralConfig As New GeneralConfig For Each oRow As DataRow In oTable.Rows Dim oValue As String = oRow.ItemEx("VALUE", String.Empty) Select Case oRow.Item("KEY") Case "YEAR_OVERRIDE" oGeneralConfig.DefaultYearOverride = oRow.ItemEx("VALUE", 0) Case "WEBSERVICE_BASEURL" oGeneralConfig.Webservice.BaseUrl = oValue Case "WEBSERVICE_USERNAME" oGeneralConfig.Webservice.Username = oValue Case "WEBSERIVCE_PASSWORD" oGeneralConfig.Webservice.Password = oValue Case "WEBSERVICE_IMPORT_BASE_PATH" oGeneralConfig.Webservice.ImportBasePath = oValue Case "WEBSERVICE_IMPORT_RELATIVE_PATH" oGeneralConfig.Webservice.ImportRelativePath = oValue Case "TEMPLATE_DIRECTORY" oGeneralConfig.TemplateDirectory = oValue End Select Next GeneralConfiguration = oGeneralConfig Return True Catch ex As Exception Logger.Error(ex) Return False End Try End Function Public Async Function LoadMappingConfiguration() As Task(Of Boolean) Try Dim oTable As DataTable = Await Database.GetDatatableAsync(SQL_VWMT_MAPPING) Dim oMappingConfig As New MappingConfig For Each oRow As DataRow In oTable.Rows Dim oTemplate As New MappingConfigItem With { .OrderKey = oRow.ItemEx("ORDER_KEY", String.Empty), .SourceName = oRow.ItemEx("SOURCE_NAME", String.Empty), .SourceItem = oRow.ItemEx("SOURCE_ITEM", String.Empty), .SourceRegex = oRow.ItemEx("SOURCE_REGEX", String.Empty), .DestinationName = oRow.ItemEx("DESTINATION_NAME", String.Empty), .DestinationItem = oRow.ItemEx("DESTINATION_ITEM", String.Empty), .DestinationValue = oRow.ItemEx("DESTINATION_VALUE", String.Empty) } oMappingConfig.Items.Add(oTemplate) Next MappingConfiguration = oMappingConfig Return True Catch ex As Exception Logger.Error(ex) Return False End Try End Function Public Async Function LoadMandatorConfiguration() As Task(Of Boolean) Try Dim oTable As DataTable = Await Database.GetDatatableAsync(SQL_TBMT_MANDATORS) Dim oMandatorConfig As New MandatorConfig For Each oRow As DataRow In oTable.Rows Dim oMandator As New MandatorConfigItem With { .OrderKey = oRow.ItemEx("ORDER_KEY", String.Empty), .Name = oRow.ItemEx("NAME", String.Empty) } oMandatorConfig.Items.Add(oMandator) Next MandatorConfiguration = oMandatorConfig Return True Catch ex As Exception Logger.Error(ex) Return False End Try End Function Public Async Function LoadTemplateConfiguration() As Task(Of Boolean) Try Dim oTable As DataTable = Await Database.GetDatatableAsync(SQL_VWMT_ITEMS) Dim oItems As New List(Of FieldConfig) For Each oRow As DataRow In oTable.Rows Dim oColumn As New FieldConfig() With { .Id = oRow.ItemEx("ITEM_ID", 0), .Template = oRow.ItemEx("TEMPLATE_NAME", String.Empty), .Table = oRow.ItemEx("TABLE_NAME", String.Empty), .Name = oRow.ItemEx("ITEM_NAME", String.Empty), .Type = FieldConfig.ConvertType(ItemEx(oRow, "DATA_TYPE", String.Empty)), .OrderKey = oRow.ItemEx("ORDER_KEY", 0), .IsReadOnly = oRow.ItemEx("IS_READ_ONLY", False), .IsVisible = oRow.ItemEx("IS_VISIBLE", True), .IsRequired = oRow.ItemEx("IS_REQUIRED", False), .IsVirtual = oRow.ItemEx("IS_VIRTUAL", False), .IsHead = oRow.ItemEx("IS_HEAD", True), .PreferExternalValue = oRow.ItemEx("PREFER_EXTERNAL", True), .Functions = New List(Of FieldConfig.ColumnFunction) } Logger.Debug("Creating Template Item for Table [{0}]: [{1}]", oColumn.Table, oColumn.Name) oItems.Add(oColumn) Next TemplateConfiguration = New TemplateConfig With { .Items = oItems } Return True Catch ex As Exception Logger.Error(ex) Return False End Try End Function Public Async Function LoadTemplateFunctions() As Task(Of Boolean) Try Dim oTable As DataTable = Await Database.GetDatatableAsync(SQL_VWMT_FUNCTIONS) Dim oItems As New List(Of FieldConfig) For Each oRow As DataRow In oTable.Rows Dim oTemplateItemId = oRow.ItemEx("ITEM_ID", 0) Dim oTemplateItem = TemplateConfiguration.Items.SingleOrDefault(Function(i) i.Id = oTemplateItemId) If oTemplateItem Is Nothing Then Logger.Warn("Function configuration could not be assigned to an existing template item, item id was [{0}]", oTemplateItemId) Continue For End If oTemplateItem.Functions.Add(New FieldConfig.ColumnFunction With { .Id = oRow.ItemEx("FUNCTION_ID", 0), .Name = oRow.ItemEx("FUNCTION_NAME", String.Empty), .Params = oRow.ItemEx("FUNCTION_PARAMETERS", String.Empty) }) Next Return True Catch ex As Exception Logger.Error(ex) Return False End Try End Function Public Function UpdateTemplateFromFile(pTemplate As Template, pInputDirectory As String) As Template Dim oFullPath = Path.Combine(pInputDirectory, pTemplate.FileName) If Not IO.File.Exists(oFullPath) Then Throw New FileNotFoundException(oFullPath) End If Dim oElements = GetTemplateElements(oFullPath) For Each oElement In oElements Dim oColumns = GetElementColumns(oElement) Dim oTemplateColumns As New List(Of Template.Column) For Each oColumn As XElement In oColumns Dim oName = XmlData.GetElementAttribute(oColumn, "name") Dim oMinOccurs = XmlData.GetElementAttribute(oColumn, "minOccurs") Dim oMaxOccurs = XmlData.GetElementAttribute(oColumn, "maxOccurs") Dim oType = GetElementType(oColumn) Dim oRequired = False If oMinOccurs = 1 And oMaxOccurs = 1 Then oRequired = True End If Dim oTemplateColumn As New Template.Column With { .Name = oName, .DataType = oType, .IsRequired = oRequired } oTemplateColumns.Add(oTemplateColumn) Next pTemplate.Tables.Add(New Template.Table With { .Name = XmlData.GetElementAttribute(oElement, "name"), .Columns = oTemplateColumns }) Next Return pTemplate End Function Public Function UpdateTemplateTablesFromDatabase(pTemplate As Template, pTemplateConfig As TemplateConfig) As Template If pTemplateConfig Is Nothing Then Logger.Warn("Template config is empty. Skipping.") Return pTemplate End If Dim oTemplate = CreateVirtualColumns(pTemplate, pTemplateConfig) For Each oTable In oTemplate.Tables For Each oColumn As Template.Column In oTable.Columns Dim oConfig As FieldConfig = pTemplateConfig.GetColumn(oColumn.Name, oTable.Name) If oConfig Is Nothing Then oConfig = New FieldConfig With { .IsRequired = oColumn.IsRequired, .Name = oColumn.Name } End If oColumn.Config = oConfig Next Next Return oTemplate End Function Private Function CreateVirtualColumns(pTemplate As Template, pTemplateConfig As TemplateConfig) As Template Dim oTemplateItems = pTemplateConfig.Items. Where(Function(item) item.Template = pTemplate.Name). ToList() For Each oItem In oTemplateItems ' Find the table that relates to this config item Dim oTable = pTemplate.Tables.Where(Function(table) table.Name = oItem.Table).FirstOrDefault() If oTable Is Nothing Then Logger.Warn("Table [{0}] for item [{1}] does not exist in this Template!", oItem.Table, oItem.Name) Continue For End If Dim oColumnExists = oTable.Columns.Any(Function(column) column.Name = oItem.Name) If oColumnExists = False And oItem.IsVirtual = True Then oTable.Columns.Add(New Template.Column() With { .Name = oItem.Name, .Config = oItem, .DataType = Constants.ColumnType.String, .IsRequired = False }) End If Next Return pTemplate End Function Public Function UpdateTemplateFromDatabase(pTemplate As Template) As Template Try Dim oTable As DataTable = Database.GetDatatable($"{SQL_TBMT_CONFIG} WHERE TEMPLATE_ID = {pTemplate.Guid}") Dim oGeneralConfig As New GeneralConfig For Each oRow As DataRow In oTable.Rows Dim oValue As String = oRow.ItemEx("VALUE", String.Empty) Select Case oRow.Item("KEY") Case "INPUT_DIRECTORY" pTemplate.InputDirectory = oValue Case "OUTPUT_DIRECTORY" pTemplate.OutputDirectory = oValue Case "ARCHIVE_DIRECTORY" pTemplate.ArchiveDirectory = oValue End Select Next Return pTemplate Catch ex As Exception Logger.Error(ex) Return pTemplate End Try End Function Public Function GetElementType(pElement As XElement) As Constants.ColumnType Dim oTypeString = XmlData.GetElementAttribute(pElement, "type") If oTypeString Is Nothing Then Dim oRestrictionElement As XElement = pElement. Descendants(ns + "restriction"). FirstOrDefault() oTypeString = XmlData.GetElementAttribute(oRestrictionElement, "base") End If Select Case oTypeString Case Constants.TEMPLATE_TYPE_DATE Return Constants.ColumnType.Date Case Constants.TEMPLATE_TYPE_INTEGER Return Constants.ColumnType.Integer Case Constants.TEMPLATE_TYPE_DECIMAL Return Constants.ColumnType.Decimal Case Constants.TEMPLATE_TYPE_BOOLEAN Return Constants.ColumnType.Boolean Case Else Return Constants.ColumnType.String End Select End Function Public Function GetTemplateElements(pTemplateFilePath As String) As List(Of XElement) Dim oText As String = IO.File.ReadAllText(pTemplateFilePath) Dim oDoc = XDocument.Parse(oText) Return XmlData.GetElementsFromElement(oDoc, "choice", ns) End Function Public Function GetElementColumns(pElement As XElement) As List(Of XElement) Return XmlData.GetElementsFromElement(pElement, "sequence", ns) End Function End Class End Namespace