From 42db951c93b0d204e9a2e3c5d2eb3d17996484c1 Mon Sep 17 00:00:00 2001 From: Jonathan Jenne Date: Thu, 9 Dec 2021 13:31:25 +0100 Subject: [PATCH] EDMIService: Add dynamic file path --- Service.EDMIService/BaseClass.vb | 8 + Service.EDMIService/EDMIService.vbproj | 4 +- Service.EDMIService/IDB/Helpers.vb | 4 + ...serAttributeValue.vb => AttributeValue.vb} | 17 +- .../ImportFile/ImportFileMethod.vb | 189 +++--------------- .../Methods/GlobalIndexer/Loader.vb | 181 +++++++++++++++++ .../Methods/GlobalIndexer/Profile.vb | 11 + Service.EDMIService/Scheduler/Scheduler.vb | 2 - 8 files changed, 255 insertions(+), 161 deletions(-) rename Service.EDMIService/Methods/{UserAttributeValue.vb => AttributeValue.vb} (53%) create mode 100644 Service.EDMIService/Methods/GlobalIndexer/Loader.vb create mode 100644 Service.EDMIService/Methods/GlobalIndexer/Profile.vb diff --git a/Service.EDMIService/BaseClass.vb b/Service.EDMIService/BaseClass.vb index 6823af2d..42edda75 100644 --- a/Service.EDMIService/BaseClass.vb +++ b/Service.EDMIService/BaseClass.vb @@ -11,11 +11,19 @@ Public Class BaseClass Logger = pLogConfig.GetLogger(oClassName) End Sub + ''' + ''' + ''' + ''' Public Sub LogAndThrow(pMessage As String) Logger.Warn(pMessage) Throw New ApplicationException(pMessage) End Sub + ''' + ''' + ''' + ''' Public Sub LogAndThrow(pException As Exception, pMessage As String) Logger.Error(pException) Throw New ApplicationException(pMessage, pException) diff --git a/Service.EDMIService/EDMIService.vbproj b/Service.EDMIService/EDMIService.vbproj index 67fa652f..e629449f 100644 --- a/Service.EDMIService/EDMIService.vbproj +++ b/Service.EDMIService/EDMIService.vbproj @@ -145,7 +145,9 @@ + + @@ -160,7 +162,7 @@ - + diff --git a/Service.EDMIService/IDB/Helpers.vb b/Service.EDMIService/IDB/Helpers.vb index af19315a..2d72aadb 100644 --- a/Service.EDMIService/IDB/Helpers.vb +++ b/Service.EDMIService/IDB/Helpers.vb @@ -37,6 +37,10 @@ Namespace IDB End Function Public Function UserAttributesToDictionary(pUserAttributes As List(Of UserAttributeValue)) As Dictionary(Of String, List(Of String)) + If pUserAttributes Is Nothing OrElse pUserAttributes.Count = 0 Then + Return New Dictionary(Of String, List(Of String)) + End If + Return pUserAttributes.ToDictionary( Function(attr) attr.AttributeName, Function(attr) attr.AttributeValues) diff --git a/Service.EDMIService/Methods/UserAttributeValue.vb b/Service.EDMIService/Methods/AttributeValue.vb similarity index 53% rename from Service.EDMIService/Methods/UserAttributeValue.vb rename to Service.EDMIService/Methods/AttributeValue.vb index 4c5dfed8..7f46a092 100644 --- a/Service.EDMIService/Methods/UserAttributeValue.vb +++ b/Service.EDMIService/Methods/AttributeValue.vb @@ -1,9 +1,16 @@ Namespace Methods + Public MustInherit Class BaseAttributeValue + + End Class + + ''' + ''' Attribute values supplied by the user + ''' Public Class UserAttributeValue - Public Property AttributeId As Integer Public Property AttributeName As String Public Property AttributeValues As List(Of String) + Public Property AttributeId As Integer Public Property ControlName As String Public Overrides Function ToString() As String @@ -11,4 +18,12 @@ End Function End Class + Public Class AutoAttributeValue + Inherits BaseAttributeValue + End Class + + Public Class SystemAttributeValue + Inherits BaseAttributeValue + End Class + End Namespace \ No newline at end of file diff --git a/Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileMethod.vb b/Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileMethod.vb index 423c5dce..f066722b 100644 --- a/Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileMethod.vb +++ b/Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileMethod.vb @@ -10,24 +10,19 @@ Namespace Methods.GlobalIndexer.ImportFile Public Class ImportFileMethod Inherits BaseMethod + Private ReadOnly Loader As Loader Private ReadOnly Patterns As Patterns2 Private ReadOnly GetDatatable As GetDatatableFromCacheMethod Private ReadOnly Connection As SqlClient.SqlConnection Private ReadOnly Transaction As SqlClient.SqlTransaction Private User As UserState - Private Profile As DataRow - - Private Const VIEW_PROFILE = "VWGI_DOCTYPE_IDB" - Private Const VIEW_INDEX_MANUAL = "VWDDINDEX_MAN" - Private Const VIEW_INDEX_AUTOMATIC = "VWDDINDEX_AUTOM" - Private Const TABLE_POST_PROCESSING = "TBDD_INDEX_MAN_POSTPROCESSING" Public Sub New(pLogConfig As LogConfig, pMSSQLServer As MSSQLServer, pGlobalState As GlobalState) MyBase.New(pLogConfig, pMSSQLServer, pGlobalState) Patterns = New Patterns2(pLogConfig) - GetDatatable = New GetDatatableFromCacheMethod(LogConfig, Database, GlobalState) + Loader = New Loader(pLogConfig, Database, pGlobalState) Connection = Database.GetConnection() Transaction = Connection.BeginTransaction() End Sub @@ -47,10 +42,10 @@ Namespace Methods.GlobalIndexer.ImportFile ' TODO: Add missing user properties in UserState from TBDD_USER 'pData.User = ResolveUserFromUserName(pData.User.UserName) - Dim oManualIndexes = LoadManualIndexes(pData.ProfileId) - Dim oAutomaticIndexes = LoadAutomaticIndexes(pData.ProfileId) - Dim oPostProcessingSteps As DataTable = LoadPostProcessingSteps(oManualIndexes) - LoadProfile(pData.ProfileId) + Dim oManualIndexes = Loader.LoadManualIndexes(pData.ProfileId) + Dim oAutomaticIndexes = Loader.LoadAutomaticIndexes(pData.ProfileId) + Dim oPostProcessingSteps As DataTable = Loader.LoadPostProcessingSteps(oManualIndexes) + Dim oProfile = Loader.LoadProfile(pData.ProfileId) Dim oUserAttributes = pData.AttributeValues Dim oAutoAttributes As List(Of UserAttributeValue) = Nothing @@ -79,15 +74,19 @@ Namespace Methods.GlobalIndexer.ImportFile Throw New ApplicationException(oResponse.ErrorMessage) End If - Logger.Info("Generating display filename for file [{0}]", pData.File.FileName) + ' Generate display Filename from nameconvention + Dim oDisplayFilename = GetFilenameByNameconvention( + pData.File.FileInfoRaw, oProfile.NameConvention, User, oUserAttributes, oAutoAttributes) - Dim oNameconvention As String = Profile.ItemEx(Of String)("NAMENKONVENTION") - Dim oDisplayFilename = GetFilenameByNameconvention(pData.File.FileInfoRaw, oNameconvention, User, oUserAttributes, oAutoAttributes) + ' Generate virtual path from profile + Dim oDynamicFilePath = GetVirtualPath( + pData.File.FileInfoRaw, oProfile.DynamicPath, User, oUserAttributes, oAutoAttributes) Logger.Info("Collecting Attributes for ObjectId [{0}]", oResponse.ObjectId) Dim oFinalAttributes As New Dictionary(Of String, List(Of String)) From { - {"DisplayFileName", New List(Of String) From {oDisplayFilename}} + {"DisplayFileName", New List(Of String) From {oDisplayFilename}}, + {"Dynamic Folder", New List(Of String) From {oDynamicFilePath}} } oFinalAttributes = oFinalAttributes. Concat(Helpers.UserAttributesToDictionary(oUserAttributes)). @@ -113,7 +112,25 @@ Namespace Methods.GlobalIndexer.ImportFile End Try End Function + Private Function GetVirtualPath(pFileInfo As FileInfo, pPathConvention As String, pUser As UserState, pAttributes As List(Of UserAttributeValue), pAutoAttributes As List(Of UserAttributeValue)) + Logger.Info("Generating virtual path for file [{0}]", pFileInfo.Name) + + Dim oAttributeDict = Helpers.UserAttributesToDictionary(pAttributes) + Dim oAutoAttributeDict = Helpers.UserAttributesToDictionary(pAutoAttributes) + + If pPathConvention Is Nothing OrElse pPathConvention = String.Empty Then + Logger.Warn("Virtual path template for File [{0}] was empty. Returning empty string.", pFileInfo.Name) + Return String.Empty + + End If + + Dim oVirtualPath As String = Helpers.GetPlaceholderValue(pPathConvention, pFileInfo, pUser, oAttributeDict, oAutoAttributeDict) + Return oVirtualPath + End Function + Private Function GetFilenameByNameconvention(pFileInfo As FileInfo, pNameconvention As String, pUser As UserState, pAttributes As List(Of UserAttributeValue), pAutoAttributes As List(Of UserAttributeValue)) As String + Logger.Info("Generating display filename for file [{0}]", pFileInfo.Name) + Dim oAttributeDict = Helpers.UserAttributesToDictionary(pAttributes) Dim oAutoAttributeDict = Helpers.UserAttributesToDictionary(pAutoAttributes) @@ -150,148 +167,6 @@ Namespace Methods.GlobalIndexer.ImportFile End Try Next End Sub - - ''' - ''' Load Profiles for this Import - ''' - Private Sub LoadProfile(pProfileId As Integer) - Logger.Debug("Start of Method [LoadAutomaticIndexes]") - - Try - Dim oProfile = GetDatatable.Run( - New GetDatatableFromCacheRequest With { - .DataTable = VIEW_PROFILE, - .FilterExpression = $"DOCTYPE_ID = {pProfileId}" - }) - - If oProfile.OK = False Then - LogAndThrow(oProfile.ErrorMessage) - End If - - Profile = oProfile.Table.Rows.Item(0) - Catch ex As Exception - LogAndThrow(ex, "Error while automatic loading indexes!") - End Try - End Sub - - ''' - ''' Load automatic indexes for this Import - ''' - Private Function LoadAutomaticIndexes(pProfileId As Integer) As List(Of AutomaticIndex) - Logger.Debug("Start of Method [LoadAutomaticIndexes]") - - Try - ' Load automatic Indexes for this Import - Dim oAutomaticIndexes = GetDatatable.Run( - New GetDatatableFromCacheRequest With { - .DataTable = VIEW_INDEX_AUTOMATIC, - .FilterExpression = $"DOCTYPE_ID = {pProfileId}" - }) - - If oAutomaticIndexes.OK = False Then - LogAndThrow(oAutomaticIndexes.ErrorMessage) - End If - - Dim oIndexes As New List(Of AutomaticIndex) - - For Each oRow As DataRow In oAutomaticIndexes.Table.Rows - Dim oAutomaticIndex As New AutomaticIndex With { - .Id = oRow.ItemEx(Of Integer)("GUID"), - .Name = oRow.ItemEx(Of String)("INDEXNAME"), - .ProfileId = oRow.ItemEx(Of Integer)("DOCTYPE_ID"), - .SQLCommand = oRow.ItemEx(Of String)("SQL_RESULT"), - .SQLConnectionId = oRow.ItemEx(Of Integer)("CONNECTION_ID"), - .Sequence = oRow.ItemEx(Of String)("SEQUENCE"), - .Value = oRow.ItemEx(Of String)("VALUE") - } - - oIndexes.Add(oAutomaticIndex) - Next - - Logger.Info("Automatic indexes loaded: [{0}]", oIndexes.Count) - - Return oIndexes - Catch ex As Exception - LogAndThrow(ex, "Error while automatic loading indexes!") - End Try - End Function - - ''' - ''' Load manual indexes for this Import - ''' - Private Function LoadManualIndexes(pProfileId As Integer) As List(Of ManualIndex) - Logger.Debug("Start of Method [LoadManualIndexes]") - - Try - ' Load manual Indexes for this Import - Dim oManualIndexes = GetDatatable.Run( - New GetDatatableFromCacheRequest With { - .DataTable = VIEW_INDEX_MANUAL, - .FilterExpression = $"DOK_ID = {pProfileId}" - }) - - If oManualIndexes.OK = False Then - LogAndThrow(oManualIndexes.ErrorMessage) - End If - - Dim oIndexes As New List(Of ManualIndex) - - For Each oRow As DataRow In oManualIndexes.Table.Rows - Dim oManualIndex As New ManualIndex With { - .Id = oRow.ItemEx(Of Integer)("GUID"), - .Name = oRow.ItemEx(Of String)("INDEXNAME"), - .ProfileId = oRow.ItemEx(Of Integer)("DOCTYPE_ID"), - .IsOptional = oRow.ItemEx(Of Boolean)("OPTIONAL"), - .IsMultiselect = oRow.ItemEx(Of String)("MULTISELECT"), - .SQLCommand = oRow.ItemEx(Of String)("SQL_RESULT"), - .SQLConnectionId = oRow.ItemEx(Of Integer)("CONNECTION_ID"), - .DefaultValue = oRow.ItemEx(Of String)("DEFAULT_VALUE"), - .DataType = oRow.ItemEx(Of String)("DATA_TYPE") - } - - oIndexes.Add(oManualIndex) - Next - - Logger.Info("Manual indexes loaded: [{0}]", oIndexes.Count) - - Return oIndexes - - Catch ex As Exception - LogAndThrow(ex, "Error while loading indexes!") - End Try - End Function - - Private Function LoadPostProcessingSteps(pManualIndexes As List(Of ManualIndex)) As DataTable - Logger.Debug("Start of Method [LoadPostProcessingSteps]") - - Try - ' Generate a string containing all index ids joined into a string - Dim oIndexIdList As List(Of Integer) = pManualIndexes. - Select(Function(index) index.Id). - ToList() - Dim oIndexIds As String = String.Join(",", oIndexIdList) - - If oIndexIdList.Count = 0 Then - Logger.Debug("No Postprocessing steps found for this profile. Exiting.") - Return Nothing - End If - - ' Load all relevant postprocessing steps - Dim oPostProcessingSteps = GetDatatable.Run( - New GetDatatableFromCacheRequest With { - .DataTable = TABLE_POST_PROCESSING, - .FilterExpression = $"IDXMAN_ID IN ({oIndexIds})" - }) - - If oPostProcessingSteps.OK = False Then - LogAndThrow(oPostProcessingSteps.ErrorMessage) - End If - - Return oPostProcessingSteps.Table - Catch ex As Exception - LogAndThrow(ex, "Error while loading post processing steps!") - End Try - End Function End Class End Namespace \ No newline at end of file diff --git a/Service.EDMIService/Methods/GlobalIndexer/Loader.vb b/Service.EDMIService/Methods/GlobalIndexer/Loader.vb new file mode 100644 index 00000000..99265b03 --- /dev/null +++ b/Service.EDMIService/Methods/GlobalIndexer/Loader.vb @@ -0,0 +1,181 @@ +Imports DigitalData.Modules.Database +Imports DigitalData.Modules.Language +Imports DigitalData.Modules.Logging +Imports DigitalData.Services.EDMIService.Methods.GetDatatableFromCache + +Namespace Methods.GlobalIndexer + Public Class Loader + Inherits BaseClass + + Private Const VIEW_PROFILE = "VWGI_DOCTYPE_IDB" + Private Const VIEW_INDEX_MANUAL = "VWDDINDEX_MAN" + Private Const VIEW_INDEX_AUTOMATIC = "VWDDINDEX_AUTOM" + Private Const TABLE_POST_PROCESSING = "TBDD_INDEX_MAN_POSTPROCESSING" + + Private GetDatatable As GetDatatableFromCacheMethod + + Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pGlobalState As GlobalState) + MyBase.New(pLogConfig) + GetDatatable = New GetDatatableFromCacheMethod(pLogConfig, pDatabase, pGlobalState) + End Sub + + ''' + ''' Load Profiles for this Import + ''' + Public Function LoadProfile(pProfileId As Integer) As Profile + Logger.Debug("Start of Method [LoadAutomaticIndexes]") + + Try + Dim oProfile = GetDatatable.Run( + New GetDatatableFromCacheRequest With { + .DataTable = VIEW_PROFILE, + .FilterExpression = $"DOCTYPE_ID = {pProfileId}" + }) + + If oProfile.OK = False Then + LogAndThrow(oProfile.ErrorMessage) + End If + + Dim oRow As DataRow = oProfile.Table.Rows.Item(0) + Dim oProfileObject As New Profile With { + .Id = oRow.ItemEx("DOCTYPE_ID", 0), + .Name = oRow.ItemEx("DOCTYPE", "Missing Profile Name"), + .IsActive = oRow.ItemEx("AKTIV", False), + .NameConvention = oRow.ItemEx("NAMENKONVENTION", ""), + .ObjectStore = oRow.ItemEx("OBJECT_STORE", "Work"), + .ShortName = oRow.ItemEx("KURZNAME", ""), + .DynamicPath = oRow.ItemEx("DYNAMIC_PATH", "") + } + + Return oProfileObject + Catch ex As Exception + LogAndThrow(ex, "Error while automatic loading indexes!") + Return Nothing + End Try + End Function + + ''' + ''' Load automatic indexes for this Import + ''' + Public Function LoadAutomaticIndexes(pProfileId As Integer) As List(Of AutomaticIndex) + Logger.Debug("Start of Method [LoadAutomaticIndexes]") + + Try + ' Load automatic Indexes for this Import + Dim oAutomaticIndexes = GetDatatable.Run( + New GetDatatableFromCacheRequest With { + .DataTable = VIEW_INDEX_AUTOMATIC, + .FilterExpression = $"DOCTYPE_ID = {pProfileId}" + }) + + If oAutomaticIndexes.OK = False Then + LogAndThrow(oAutomaticIndexes.ErrorMessage) + End If + + Dim oIndexes As New List(Of AutomaticIndex) + + For Each oRow As DataRow In oAutomaticIndexes.Table.Rows + Dim oAutomaticIndex As New AutomaticIndex With { + .Id = oRow.ItemEx(Of Integer)("GUID"), + .Name = oRow.ItemEx(Of String)("INDEXNAME"), + .ProfileId = oRow.ItemEx(Of Integer)("DOCTYPE_ID"), + .SQLCommand = oRow.ItemEx(Of String)("SQL_RESULT"), + .SQLConnectionId = oRow.ItemEx(Of Integer)("CONNECTION_ID"), + .Sequence = oRow.ItemEx(Of String)("SEQUENCE"), + .Value = oRow.ItemEx(Of String)("VALUE") + } + + oIndexes.Add(oAutomaticIndex) + Next + + Logger.Info("Automatic indexes loaded: [{0}]", oIndexes.Count) + + Return oIndexes + Catch ex As Exception + LogAndThrow(ex, "Error while automatic loading indexes!") + Return Nothing + End Try + End Function + + ''' + ''' Load manual indexes for this Import + ''' + Public Function LoadManualIndexes(pProfileId As Integer) As List(Of ManualIndex) + Logger.Debug("Start of Method [LoadManualIndexes]") + + Try + ' Load manual Indexes for this Import + Dim oManualIndexes = GetDatatable.Run( + New GetDatatableFromCacheRequest With { + .DataTable = VIEW_INDEX_MANUAL, + .FilterExpression = $"DOK_ID = {pProfileId}" + }) + + If oManualIndexes.OK = False Then + LogAndThrow(oManualIndexes.ErrorMessage) + End If + + Dim oIndexes As New List(Of ManualIndex) + + For Each oRow As DataRow In oManualIndexes.Table.Rows + Dim oManualIndex As New ManualIndex With { + .Id = oRow.ItemEx(Of Integer)("GUID"), + .Name = oRow.ItemEx(Of String)("INDEXNAME"), + .ProfileId = oRow.ItemEx(Of Integer)("DOCTYPE_ID"), + .IsOptional = oRow.ItemEx(Of Boolean)("OPTIONAL"), + .IsMultiselect = oRow.ItemEx(Of String)("MULTISELECT"), + .SQLCommand = oRow.ItemEx(Of String)("SQL_RESULT"), + .SQLConnectionId = oRow.ItemEx(Of Integer)("CONNECTION_ID"), + .DefaultValue = oRow.ItemEx(Of String)("DEFAULT_VALUE"), + .DataType = oRow.ItemEx(Of String)("DATA_TYPE") + } + + oIndexes.Add(oManualIndex) + Next + + Logger.Info("Manual indexes loaded: [{0}]", oIndexes.Count) + + Return oIndexes + + Catch ex As Exception + LogAndThrow(ex, "Error while loading indexes!") + Return Nothing + End Try + End Function + + Public Function LoadPostProcessingSteps(pManualIndexes As List(Of ManualIndex)) As DataTable + Logger.Debug("Start of Method [LoadPostProcessingSteps]") + + Try + ' Generate a string containing all index ids joined into a string + Dim oIndexIdList As List(Of Integer) = pManualIndexes. + Select(Function(index) index.Id). + ToList() + Dim oIndexIds As String = String.Join(",", oIndexIdList) + + If oIndexIdList.Count = 0 Then + Logger.Debug("No Postprocessing steps found for this profile. Exiting.") + Return Nothing + End If + + ' Load all relevant postprocessing steps + Dim oPostProcessingSteps = GetDatatable.Run( + New GetDatatableFromCacheRequest With { + .DataTable = TABLE_POST_PROCESSING, + .FilterExpression = $"IDXMAN_ID IN ({oIndexIds})" + }) + + If oPostProcessingSteps.OK = False Then + LogAndThrow(oPostProcessingSteps.ErrorMessage) + End If + + Return oPostProcessingSteps.Table + Catch ex As Exception + LogAndThrow(ex, "Error while loading post processing steps!") + Return Nothing + End Try + End Function + End Class + +End Namespace + diff --git a/Service.EDMIService/Methods/GlobalIndexer/Profile.vb b/Service.EDMIService/Methods/GlobalIndexer/Profile.vb new file mode 100644 index 00000000..99cd03c2 --- /dev/null +++ b/Service.EDMIService/Methods/GlobalIndexer/Profile.vb @@ -0,0 +1,11 @@ +Namespace Methods.GlobalIndexer + Public Class Profile + Public Property Id As Integer + Public Property Name As String + Public Property ShortName As String + Public Property DynamicPath As String + Public Property IsActive As Boolean + Public Property NameConvention As String + Public Property ObjectStore As String + End Class +End Namespace \ No newline at end of file diff --git a/Service.EDMIService/Scheduler/Scheduler.vb b/Service.EDMIService/Scheduler/Scheduler.vb index 22b6f05b..e968b224 100644 --- a/Service.EDMIService/Scheduler/Scheduler.vb +++ b/Service.EDMIService/Scheduler/Scheduler.vb @@ -106,8 +106,6 @@ Public Class Scheduler _Logger.Warn("Unexpected Error while setting up scheduler: " & ex.Message) End Try End Sub - - Public Async Function GetCronJobs() As Task(Of DataTable) Try Dim oSQL As String = "SELECT * FROM TBAPPSERV_CRON_JOB WHERE ACTIVE = 1"