AppStd/Modules
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

EDMIService: WIP

Browse Source
This commit is contained in:
Jonathan Jenne
2021-12-07 16:37:23 +01:00
parent 6be8b1bdb5
commit 2a6fd3555b
20 changed files with 498 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileMethod.vb
View File

@@ -12,9 +12,6 @@ Namespace Methods.GlobalIndexer.ImportFile
Private ReadOnly GetDatatable As GetDatatableFromCacheMethod
Private Profile As DataRow
Private ManualIndexes As List(Of ManualIndex)
Private AutomaticIndexes As List(Of AutomaticIndex)
Private ManualIndexesPostProcessing As DataTable
Private Const VIEW_PROFILE = "VWGI_DOCTYPE_IDB"
Private Const VIEW_INDEX_MANUAL = "VWDDINDEX_MAN"
@@ -41,7 +38,9 @@ Namespace Methods.GlobalIndexer.ImportFile
' TODO: Add missing user properties in UserState from TBDD_USER
'pData.User = ResolveUserFromUserName(pData.User.UserName)
LoadIndexes(pData.ProfileId)
Dim oManualIndexes = LoadManualIndexes(pData.ProfileId)
Dim oAutomaticIndexes = LoadAutomaticIndexes(pData.ProfileId)
Dim oPostProcessingSteps As DataTable = LoadPostProcessingSteps(oManualIndexes)
LoadProfile(pData.ProfileId)
@@ -49,13 +48,11 @@ Namespace Methods.GlobalIndexer.ImportFile
Dim oFileName As String = GetFilenameByNameconvention(pData.File.FileName, Profile.Item("NAMENKONVENTION"))
' Apply post processing
Dim oPostProcessing = New Steps.PostProcessing(LogConfig, ManualIndexesPostProcessing)
Dim oPostProcessing = New Steps.PostProcessing(LogConfig, oPostProcessingSteps)
oFinalAttributes = oPostProcessing.ApplyManualPostprocessing(oFinalAttributes)
' Apply automatic attributes
Dim oAutomaticIndexing = New Steps.AutomaticIndexing(LogConfig, Database, AutomaticIndexes, GlobalState)
Dim oAutomaticIndexing = New Steps.AutomaticIndexing(LogConfig, Database, oAutomaticIndexes, GlobalState)
oFinalAttributes = oAutomaticIndexing.ApplyAutomaticeAttributes(oFinalAttributes, pData.File.FileInfoRaw, pData.User)
' Import the file
@@ -69,6 +66,7 @@ Namespace Methods.GlobalIndexer.ImportFile
})
If oResponse.OK Then
Logger.Info("Import of file [{0}] under ObjectId [{1}] successful!", pData.File.FileName, oResponse.ObjectId)
Return New ImportFileResponse(oResponse.ObjectId)
Else
Throw New ApplicationException(oResponse.ErrorMessage)
@@ -87,14 +85,6 @@ Namespace Methods.GlobalIndexer.ImportFile
Return pFileName
End Function
Private Sub LoadIndexes(pProfileId As Integer)
Logger.Debug("Start of Method [LoadIndexes]")
LoadManualIndexes(pProfileId)
LoadAutomaticIndexes(pProfileId)
LoadPostProcessingSteps()
End Sub
''' <summary>
''' Load Profiles for this Import
''' </summary>
@@ -118,15 +108,18 @@ Namespace Methods.GlobalIndexer.ImportFile
End Try
End Sub
Private Sub LoadAutomaticIndexes(pProfileId As Integer)
''' <summary>
''' Load automatic indexes for this Import
''' </summary>
Private Function LoadAutomaticIndexes(pProfileId As Integer) As List(Of AutomaticIndex)
Logger.Debug("Start of Method [LoadAutomaticIndexes]")
Try
' Load automatic Indexes for this Import
Dim oAutomaticIndexes = GetDatatable.Run(
New GetDatatableFromCacheRequest With {
.DataTable = VIEW_INDEX_MANUAL,
.FilterExpression = $"DOK_ID = {pProfileId}"
.DataTable = VIEW_INDEX_AUTOMATIC,
.FilterExpression = $"DOCTYPE_ID = {pProfileId}"
})
If oAutomaticIndexes.OK = False Then
@@ -145,15 +138,22 @@ Namespace Methods.GlobalIndexer.ImportFile
.Sequence = oRow.ItemEx(Of String)("SEQUENCE"),
.Value = oRow.ItemEx(Of String)("VALUE")
}
oIndexes.Add(oAutomaticIndex)
Next
AutomaticIndexes = oIndexes
Logger.Info("[{0}] automatic indexes loaded.", oIndexes)
Return oIndexes
Catch ex As Exception
LogAndThrow(ex, "Error while automatic loading indexes!")
End Try
End Sub
End Function
Private Sub LoadManualIndexes(pProfileId As Integer)
''' <summary>
''' Load manual indexes for this Import
''' </summary>
Private Function LoadManualIndexes(pProfileId As Integer) As List(Of ManualIndex)
Logger.Debug("Start of Method [LoadManualIndexes]")
Try
@@ -182,23 +182,32 @@ Namespace Methods.GlobalIndexer.ImportFile
.DefaultValue = oRow.ItemEx(Of String)("DEFAULT_VALUE"),
.DataType = oRow.ItemEx(Of String)("DATA_TYPE")
}
oIndexes.Add(oManualIndex)
Next
ManualIndexes = oIndexes
Return oIndexes
Catch ex As Exception
LogAndThrow(ex, "Error while loading indexes!")
End Try
End Sub
End Function
Private Sub LoadPostProcessingSteps()
Private Function LoadPostProcessingSteps(pManualIndexes As List(Of ManualIndex)) As DataTable
Logger.Debug("Start of Method [LoadPostProcessingSteps]")
Try
' Generate a string containing all index ids joined into a string
Dim oIndexIdList As List(Of Integer) = ManualIndexes.Select(Function(index) index.Id).ToList()
Dim oIndexIdList As List(Of Integer) = pManualIndexes.
Select(Function(index) index.Id).
ToList()
Dim oIndexIds As String = String.Join(",", oIndexIdList)
If oIndexIdList.Count = 0 Then
Logger.Debug("No Postprocessing steps found for this profile. Exiting.")
Return Nothing
End If
' Load all relevant postprocessing steps
Dim oPostProcessingSteps = GetDatatable.Run(
New GetDatatableFromCacheRequest With {
@@ -210,11 +219,11 @@ Namespace Methods.GlobalIndexer.ImportFile
LogAndThrow(oPostProcessingSteps.ErrorMessage)
End If
ManualIndexesPostProcessing = oPostProcessingSteps.Table
Return oPostProcessingSteps.Table
Catch ex As Exception
LogAndThrow(ex, "Error while loading post processing steps!")
End Try
End Sub
End Function
End Class
End Namespace

2
Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileRequest.vb
View File

@@ -40,7 +40,7 @@ Namespace Methods.GlobalIndexer.ImportFile
''' Attribute Name/Attribute Value/ControlName
''' </summary>
<DataMember>
Public Property AttributeValues As List(Of UserAttributeValue)
Public Property AttributeValues As New List(Of UserAttributeValue)
''' <summary>
''' User Importing the file

91
Service.EDMIService/Methods/GlobalIndexer/ImportFile/Steps/AutomaticIndexing.vb
View File

@@ -20,17 +20,30 @@ Namespace Methods.GlobalIndexer.ImportFile.Steps
GlobalState = pGlobalState
AutomaticIndexes = pAutomaticIndexes
Patterns = New Patterns2(pLogConfig)
Logger.Info("Starting Automatic Indexing")
End Sub
Public Function ApplyAutomaticeAttributes(pManualAttributes As List(Of UserAttributeValue), pFileInfo As FileInfo, pUserState As UserState) As List(Of UserAttributeValue)
Logger.Debug("Start of Method [ApplyAutomaticeAttributes]")
Dim oAttributes = pManualAttributes
Public Function ApplyAutomaticeAttributes(pUserAttributes As List(Of UserAttributeValue), pFileInfo As FileInfo, pUserState As UserState) As List(Of UserAttributeValue)
Logger.Debug("Start of Method [ApplyAutomaticAttributes]")
If AutomaticIndexes Is Nothing OrElse AutomaticIndexes.Count = 0 Then
Logger.Warn("No Automatix Indexes supplied. Exiting.")
Return pUserAttributes
End If
Logger.Info("Processing [{0}] automatic indexes", AutomaticIndexes.Count)
Dim oAttributes As List(Of UserAttributeValue) = pUserAttributes
For Each oAutomaticIndex In AutomaticIndexes
' We add oAttributes from the previous run into the current run so it is in theory possible to reference
' automatic attributes which have been set just before.
Dim oAttribute = ApplyAutomaticIndex(oAutomaticIndex, pFileInfo, pUserState, oAttributes)
oAttributes.Add(oAttribute)
If oAttribute IsNot Nothing Then
Logger.Info("Adding Attribute [{0}]", oAttribute)
oAttributes.Add(oAttribute)
End If
Next
Return oAttributes
@@ -38,41 +51,65 @@ Namespace Methods.GlobalIndexer.ImportFile.Steps
Private Function ApplyAutomaticIndex(pAutomaticIndex As AutomaticIndex, pFileInfo As FileInfo, pUserState As UserState, pAttributes As List(Of UserAttributeValue)) As UserAttributeValue
Dim oAttributeDict = pAttributes.ToDictionary(
Function(attr) attr.AttributeName,
Function(attr) attr.AttributeValues)
Try
Dim oAttributeDict = pAttributes.ToDictionary(
Function(attr) attr.AttributeName,
Function(attr) attr.AttributeValues)
' If there is no SQL command, we use the Value property and replace all placeholders in it.
If pAutomaticIndex.HasSqlCommand = False Then
Dim oResult As String = GetPlaceholderValue(pAutomaticIndex.Value, pFileInfo, pUserState, oAttributeDict)
Logger.Info("Applying Automatic Index [{0}]", pAutomaticIndex.Name)
Dim oHasSqlCommand As Boolean = pAutomaticIndex.HasSqlCommand
Logger.Debug("Index has SQLCommand: [{0}]", oHasSqlCommand)
' If there is no SQL command, we use the Value property and replace all placeholders in it.
If oHasSqlCommand = False Then
Dim oResult As String = GetPlaceholderValue(pAutomaticIndex.Value, pFileInfo, pUserState, oAttributeDict)
Return New UserAttributeValue With {
.AttributeValues = New List(Of String) From {oResult},
.AttributeName = pAutomaticIndex.Name,
.AttributeId = pAutomaticIndex.Id
}
End If
' Otherwise we will replace placeholders in the SQL command and then execute it
Dim oConnectionString As String = GlobalState.GetConnectionString(pAutomaticIndex.SQLConnectionId)
Dim oFinalSQLCommand = pAutomaticIndex.SQLCommand
Logger.Debug("SQL Command is: [{0}]", oFinalSQLCommand)
oFinalSQLCommand = GetPlaceholderValue(oFinalSQLCommand, pFileInfo, pUserState, oAttributeDict)
' Now we have a SQL command which only contains vector placeholders
' Next, we execute the command to get our result
Dim oValue = Database.GetScalarValueWithConnection(oFinalSQLCommand, oConnectionString)
If oValue Is Nothing Then
Logger.Warn("SQL for Automatic Index [{0}] returned an error. Exiting.")
Return Nothing
End If
Logger.Info("Value for Automatic Index [{0}] is [{1}]", pAutomaticIndex.Name, oValue.ToString)
' TODO: Return multiple values
Return New UserAttributeValue With {
.AttributeValues = New List(Of String) From {oResult},
.AttributeValues = New List(Of String) From {oValue},
.AttributeName = pAutomaticIndex.Name,
.AttributeId = pAutomaticIndex.Id
}
End If
Catch ex As Exception
Logger.Warn("Automatic Indexing for index failed.")
Logger.Error(ex)
' Otherwise we will replace placeholders in the SQL command and then execute it
Dim oConnectionString As String = GlobalState.GetConnectionString(pAutomaticIndex.SQLConnectionId)
Dim oFinalSQLCommand = pAutomaticIndex.SQLCommand
oFinalSQLCommand = GetPlaceholderValue(oFinalSQLCommand, pFileInfo, pUserState, oAttributeDict)
' Now we have a SQL command which only contains vector placeholders
' Next, we execute the command to get our result
Dim oValue = Database.GetScalarValueWithConnection(oFinalSQLCommand, oConnectionString)
' TODO: Return multiple values
Return New UserAttributeValue With {
.AttributeValues = New List(Of String) From {oValue},
.AttributeName = pAutomaticIndex.Name,
.AttributeId = pAutomaticIndex.Id
}
Return Nothing
End Try
End Function
Private Function GetPlaceholderValue(pValue As String, pFileInfo As FileInfo, pUserState As UserState, pAttributes As Dictionary(Of String, List(Of String))) As String
Dim oResult As String = pValue
oResult = Patterns.ReplaceInternalValues(oResult)
oResult = Patterns.ReplaceFileValues(oResult, pFileInfo)
oResult = Patterns.ReplaceUserValues(oResult, pUserState)

55
Service.EDMIService/Methods/GlobalIndexer/ImportFile/Steps/PostProcessing.vb
View File

@@ -14,33 +14,53 @@ Namespace Methods.GlobalIndexer.ImportFile.Steps
Public Sub New(pLogConfig As LogConfig, pPostProcessingSteps As DataTable)
MyBase.New(pLogConfig)
PostprocessingSteps = pPostProcessingSteps
Logger.Info("Starting Postprocessing of Manual Indexes")
End Sub
Public Function ApplyManualPostprocessing(pManualAttributes As List(Of UserAttributeValue)) As List(Of UserAttributeValue)
Logger.Debug("Start of Method [ApplyManualPostprocessing]")
Dim oAttributes = pManualAttributes
For Each oProcessingRow As DataRow In PostprocessingSteps.Rows
Try
Logger.Debug("Start of Method [ApplyManualPostprocessing]")
Dim oIndexId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")
Dim oIndex As UserAttributeValue = pManualAttributes.
Where(Function(attr) attr.AttributeId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")).
FirstOrDefault()
If PostprocessingSteps Is Nothing Then
Logger.Debug("No Postprocessing steps found. Exiting.")
Return oAttributes
End If
Dim oValue = GetPostprocessingValue(oIndex.AttributeValues, oProcessingRow)
For Each oProcessingRow As DataRow In PostprocessingSteps.Rows
Dim oIndexId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")
Dim oIndex As UserAttributeValue = pManualAttributes.
Where(Function(attr) attr.AttributeId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")).
FirstOrDefault()
Dim oIndexPosition = pManualAttributes.IndexOf(oIndex)
oAttributes.Add(New UserAttributeValue With {
.AttributeId = oIndexId,
.AttributeName = oIndex.AttributeName,
.AttributeValues = oIndex.AttributeValues,
.ControlName = oIndex.ControlName
})
Next
Logger.Info("Postprocessing Index [{0}]", oIndex.AttributeName)
Return oAttributes
Dim oValues = GetPostprocessingValue(oIndex.AttributeValues, oProcessingRow)
Logger.Info("New Value for Index [{0}] is [{1}]", oIndex.AttributeName, String.Join(",", oValues))
' Replace the old AttributeValue with the new one
oAttributes.Item(oIndexPosition) = New UserAttributeValue With {
.AttributeId = oIndexId,
.AttributeName = oIndex.AttributeName,
.AttributeValues = oValues,
.ControlName = oIndex.ControlName
}
Next
Return oAttributes
Catch ex As Exception
Logger.Warn("Postprocessing failed. Returning incomplete Attributes.")
Logger.Error(ex)
Return oAttributes
End Try
End Function
Public Function GetPostprocessingValue(pValues As List(Of String), pRow As DataRow)
Public Function GetPostprocessingValue(pValues As List(Of String), pRow As DataRow) As List(Of String)
Logger.Debug("Start of Method [GetPostprocessingValue]")
Dim oType = pRow.Item("TYPE")
@@ -88,7 +108,8 @@ Namespace Methods.GlobalIndexer.ImportFile.Steps
Next
Case Else
LogAndThrow($"Postprocessing type [{oType}] is not supported!")
Logger.Warn("Postprocessing type [{0}] is not supported!", oType)
End Select
Return oResult

4
Service.EDMIService/Methods/UserAttributeValue.vb
View File

@@ -5,6 +5,10 @@
Public Property AttributeName As String
Public Property AttributeValues As List(Of String)
Public Property ControlName As String
Public Overrides Function ToString() As String
Return AttributeName
End Function
End Class
End Namespace

2
Service.EDMIService/Scheduler/Scheduler.vb
View File

@@ -22,7 +22,7 @@ Public Class Scheduler
Public Sub New(LogConfig As LogConfig, MSSQL_ECM As MSSQLServer, TableStore As DataSet)
_LogConfig = LogConfig
_Logger = LogConfig.GetLoggerFor("Scheduler")
_Logger = LogConfig.GetLogger()
_Factory = New StdSchedulerFactory(_Props)
_MSSQL = MSSQL_ECM

4
Service.EDMIService/WindowsService.vb
View File

@@ -12,6 +12,7 @@ Public Class WindowsService
Private _ServiceHost As ServiceHost(Of EDMIService)
Private _LogConfig As LogConfig
Private _LogConfigScheduler As LogConfig
Private _Logger As Logger
Private _Firebird As Firebird
@@ -39,6 +40,7 @@ Public Class WindowsService
Dim oServicePath As String = AppDomain.CurrentDomain.BaseDirectory
_LogConfig = New LogConfig(LogConfig.PathType.CustomPath, IO.Path.Combine(oServicePath, "Log"), FileKeepRangeInDays:=3)
_LogConfigScheduler = New LogConfig(LogConfig.PathType.CustomPath, IO.Path.Combine(oServicePath, "Log"), Suffix:="Scheduler", FileKeepRangeInDays:=3)
_Logger = _LogConfig.GetLogger()
_Logger.Info("Service {0} is starting...", SERVICE_DISPLAY_NAME)
@@ -71,7 +73,7 @@ Public Class WindowsService
_Archive = New EDMI.File.Archive(_LogConfig)
_Filesystem = New Filesystem.File(_LogConfig)
_Global = New GlobalState(_LogConfig, _MSSQL_IDB, _MSSQL_ECM)
_Scheduler = New Scheduler(_LogConfig, _MSSQL_ECM, _Global.TableStore)
_Scheduler = New Scheduler(_LogConfigScheduler, _MSSQL_ECM, _Global.TableStore)
_Logger.Debug("Loading Global Data")
_Global.LoadObjectStores()