AppStd/Modules
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

EDMIService: Automatic indexing, more consolidation of globix logic

Browse Source
This commit is contained in:
Jonathan Jenne
2021-12-06 15:01:14 +01:00
parent 34517ce209
commit 785b138c57
35 changed files with 1259 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Service.EDMIService/Methods/FileProperties.vb
View File

@@ -37,6 +37,12 @@ Namespace Methods
''' </summary>
<DataMember>
Public Property FileChecksum As String
''' <summary>
''' The Raw FileInfo Object
''' </summary>
<DataMember>
Public Property FileInfoRaw As IO.FileInfo
End Class
End Namespace

8
Service.EDMIService/Methods/GlobalIndexer/AutomaticIndex.vb Normal file
View File

@@ -0,0 +1,8 @@
Namespace Methods.GlobalIndexer
Public Class AutomaticIndex
Inherits BaseIndex
Public Value As String
End Class
End Namespace

21
Service.EDMIService/Methods/GlobalIndexer/BaseIndex.vb Normal file
View File

@@ -0,0 +1,21 @@
Namespace Methods.GlobalIndexer
Public MustInherit Class BaseIndex
Public Id As Integer
Public Name As String
Public ProfileId As Integer
Public SQLCommand As String
Public SQLConnectionId As Integer
Public Sequence As Integer = 0
Public Function HasSqlCommand() As Boolean
Return Not (
SQLCommand Is Nothing OrElse
SQLCommand = String.Empty OrElse
SQLConnectionId < 0
)
End Function
End Class
End Namespace

155
Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileMethod.vb
View File

@@ -12,8 +12,8 @@ Namespace Methods.GlobalIndexer.ImportFile
Private ReadOnly GetDatatable As GetDatatableFromCacheMethod
Private Profile As DataRow
Private ManualIndexes As DataTable
Private AutomaticIndexes As DataTable
Private ManualIndexes As List(Of ManualIndex)
Private AutomaticIndexes As List(Of AutomaticIndex)
Private ManualIndexesPostProcessing As DataTable
Private Const VIEW_PROFILE = "VWGI_DOCTYPE_IDB"
@@ -21,10 +21,6 @@ Namespace Methods.GlobalIndexer.ImportFile
Private Const VIEW_INDEX_AUTOMATIC = "VWDDINDEX_AUTOM"
Private Const TABLE_POST_PROCESSING = "TBDD_INDEX_MAN_POSTPROCESSING"
Private Const TYPE_VBSPLIT = "VBSPLIT"
Private Const TYPE_VBREPLACE = "VBREPLACE"
Private Const TYPE_REGEXPRESSION = "REG. EXPRESSION"
Public Sub New(pLogConfig As LogConfig, pMSSQLServer As MSSQLServer, pGlobalState As GlobalState)
MyBase.New(pLogConfig, pMSSQLServer, pGlobalState)
@@ -42,17 +38,25 @@ Namespace Methods.GlobalIndexer.ImportFile
''' </remarks>
Public Function Run(pData As ImportFileRequest)
Try
' TODO: Add missing user properties in UserState from TBDD_USER
'pData.User = ResolveUserFromUserName(pData.User.UserName)
LoadIndexes(pData.ProfileId)
LoadProfile(pData.ProfileId)
Dim oFinalAttributes = pData.AttributeValues
Dim oFileName As String = GetFilenameByNameconvention(pData.File.FileName, Profile.Item("NAMENKONVENTION"))
' apply the post processing
oFinalAttributes = ApplyManualPostprocessing(oFinalAttributes, ManualIndexesPostProcessing)
' Apply post processing
Dim oPostProcessing = New Steps.PostProcessing(LogConfig, ManualIndexesPostProcessing)
oFinalAttributes = oPostProcessing.ApplyManualPostprocessing(oFinalAttributes)
' TODO: apply the manual attributes
oFinalAttributes = ApplyAutomaticeAttributes(oFinalAttributes)
' Apply automatic attributes
Dim oAutomaticIndexing = New Steps.AutomaticIndexing(LogConfig, Database, AutomaticIndexes, GlobalState)
oFinalAttributes = oAutomaticIndexing.ApplyAutomaticeAttributes(oFinalAttributes, pData.File.FileInfoRaw, pData.User)
' Import the file
Dim oNewFile As New NewFileMethod(LogConfig, Database, GlobalState)
@@ -60,9 +64,8 @@ Namespace Methods.GlobalIndexer.ImportFile
.File = pData.File,
.BusinessEntity = pData.BusinessEntity,
.KindType = pData.KindType,
.Language = pData.Language,
.Who = pData.Who,
.StoreName = pData.StoreName
.StoreName = pData.StoreName,
.User = pData.User
})
If oResponse.OK Then
@@ -76,95 +79,12 @@ Namespace Methods.GlobalIndexer.ImportFile
End Try
End Function
Private Function ApplyManualPostprocessing(pManualAttributes As List(Of UserAttributeValue), pPostprocessingSteps As DataTable) As List(Of UserAttributeValue)
Logger.Debug("Start of Method [ApplyManualPostprocessing]")
Dim oAttributes = pManualAttributes
For Each oProcessingRow As DataRow In pPostprocessingSteps.Rows
Dim oIndexId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")
Dim oIndexRow As DataRow = ManualIndexes.Select($"GUID = {oIndexId}").FirstOrDefault()
Dim oIndex As UserAttributeValue = pManualAttributes.
Where(Function(attr) attr.AttributeId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")).
FirstOrDefault()
Dim oValue = GetPostprocessingValue(oIndex.AttributeValues, oIndexRow)
oAttributes.Add(New UserAttributeValue With {
.AttributeId = oIndexId,
.AttributeName = oIndex.AttributeName,
.AttributeValues = oIndex.AttributeValues,
.ControlName = oIndex.ControlName
})
Next
Return oAttributes
End Function
Private Function ApplyAutomaticeAttributes(pManualAttributes As List(Of UserAttributeValue)) As List(Of UserAttributeValue)
Logger.Debug("Start of Method [ApplyAutomaticeAttributes]")
Return pManualAttributes
End Function
Private Function GetPostprocessingValue(pValues As List(Of String), pRow As DataRow)
Logger.Debug("Start of Method [GetPostprocessingValue]")
Dim oType = pRow.Item("TYPE")
Dim oResult As New List(Of String)
Logger.Debug("Type of Postprocessing is [{0}]", oType)
Select Case oType
Case TYPE_VBREPLACE
Dim oFindString = pRow.Item("TEXT1")
Dim oReplaceString = pRow.Item("TEXT2")
Logger.Debug("Replacing [{0}] with [{1}]", oFindString, oReplaceString)
For Each oIndexValue In pValues
Dim oReplaceResult = oIndexValue.Replace(oFindString, oReplaceString)
If oReplaceResult.Equals(oIndexValue) Then
Logger.Debug("Replace did not succeed, ReplaceString was not found.")
Else
Logger.Debug("Replace successful for [{0}].", oIndexValue)
End If
oResult.Add(oReplaceResult)
Next
Case TYPE_VBSPLIT
Dim oSeparator As String = pRow.Item("TEXT1")
Dim oSplitIndex As Integer = 0
Integer.TryParse(pRow.Item("TEXT2"), oSplitIndex)
Logger.Debug("Splitting String at Separator [{0}] and Index [{1}]", oSeparator, oSplitIndex)
For Each oIndexValue In pValues
Dim oSplitted As List(Of String) = oIndexValue.Split(oSeparator).ToList()
Logger.Debug("Split succeeded, resulting list has [{0}] items.", oSplitted.Count)
If oSplitIndex < oSplitted.Count Then
Dim oValue = oSplitted.Item(oSplitIndex)
Logger.Debug("Saving value [{0}] from Index [{1}]", oValue, oSplitIndex)
oResult.Add(oValue)
Else
Logger.Debug("SplitIndex(TEXT2) was out of array bounds. Skipping.")
End If
Next
Case Else
LogAndThrow($"Postprocessing type [{oType}] is not supported!")
End Select
Return oResult
End Function
Private Function GetFilenameByNameconvention(pFileName As String, pNameconvention As String) As String
Private Function GetFilenameByNameconvention(pFileName As String, pNameconvention As String) As String
Return pFileName
End Function
Private Sub LoadIndexes(pProfileId As Integer)
@@ -213,7 +133,21 @@ Namespace Methods.GlobalIndexer.ImportFile
LogAndThrow(oAutomaticIndexes.ErrorMessage)
End If
AutomaticIndexes = oAutomaticIndexes.Table
Dim oIndexes As New List(Of AutomaticIndex)
For Each oRow As DataRow In oAutomaticIndexes.Table.Rows
Dim oAutomaticIndex As New AutomaticIndex With {
.Id = oRow.ItemEx(Of Integer)("GUID"),
.Name = oRow.ItemEx(Of String)("INDEXNAME"),
.ProfileId = oRow.ItemEx(Of Integer)("DOCTYPE_ID"),
.SQLCommand = oRow.ItemEx(Of String)("SQL_RESULT"),
.SQLConnectionId = oRow.ItemEx(Of Integer)("CONNECTION_ID"),
.Sequence = oRow.ItemEx(Of String)("SEQUENCE"),
.Value = oRow.ItemEx(Of String)("VALUE")
}
Next
AutomaticIndexes = oIndexes
Catch ex As Exception
LogAndThrow(ex, "Error while automatic loading indexes!")
End Try
@@ -234,7 +168,23 @@ Namespace Methods.GlobalIndexer.ImportFile
LogAndThrow(oManualIndexes.ErrorMessage)
End If
ManualIndexes = oManualIndexes.Table
Dim oIndexes As New List(Of ManualIndex)
For Each oRow As DataRow In oManualIndexes.Table.Rows
Dim oManualIndex As New ManualIndex With {
.Id = oRow.ItemEx(Of Integer)("GUID"),
.Name = oRow.ItemEx(Of String)("INDEXNAME"),
.ProfileId = oRow.ItemEx(Of Integer)("DOCTYPE_ID"),
.IsOptional = oRow.ItemEx(Of Boolean)("OPTIONAL"),
.IsMultiselect = oRow.ItemEx(Of String)("MULTISELECT"),
.SQLCommand = oRow.ItemEx(Of String)("SQL_RESULT"),
.SQLConnectionId = oRow.ItemEx(Of Integer)("CONNECTION_ID"),
.DefaultValue = oRow.ItemEx(Of String)("DEFAULT_VALUE"),
.DataType = oRow.ItemEx(Of String)("DATA_TYPE")
}
Next
ManualIndexes = oIndexes
Catch ex As Exception
LogAndThrow(ex, "Error while loading indexes!")
@@ -246,10 +196,7 @@ Namespace Methods.GlobalIndexer.ImportFile
Try
' Generate a string containing all index ids joined into a string
Dim oIndexIdList As New List(Of Integer)
For Each oRow As DataRow In ManualIndexes.Rows
oIndexIdList.Add(oRow.ItemEx(Of Integer)("GUID"))
Next
Dim oIndexIdList As List(Of Integer) = ManualIndexes.Select(Function(index) index.Id).ToList()
Dim oIndexIds As String = String.Join(",", oIndexIdList)
' Load all relevant postprocessing steps

14
Service.EDMIService/Methods/GlobalIndexer/ImportFile/ImportFileRequest.vb
View File

@@ -1,4 +1,5 @@
Imports System.Runtime.Serialization
Imports DigitalData.Modules.ZooFlow.State
Namespace Methods.GlobalIndexer.ImportFile
<Serializable>
@@ -13,6 +14,7 @@ Namespace Methods.GlobalIndexer.ImportFile
''' <summary>
''' The ProfileId the file will be imported with
''' </summary>
<DataMember>
Public Property ProfileId As Integer
''' <summary>
@@ -37,18 +39,14 @@ Namespace Methods.GlobalIndexer.ImportFile
''' The attribute values given by the user in the form of
''' Attribute Name/Attribute Value/ControlName
''' </summary>
<DataMember>
Public Property AttributeValues As List(Of UserAttributeValue)
''' <summary>
''' The name of the user importing the file, ex. JenneJ
''' User Importing the file
''' </summary>
''' <returns></returns>
<DataMember>
Public Property Who As String
''' <summary>
''' The language of the user, ex. de-DE
''' </summary>
<DataMember>
Public Property Language As String
Public Property User As UserState
End Class
End Namespace

85
Service.EDMIService/Methods/GlobalIndexer/ImportFile/Steps/AutomaticIndexing.vb Normal file
View File

@@ -0,0 +1,85 @@
Imports System.IO
Imports DigitalData.Modules.Database
Imports DigitalData.Modules.Logging
Imports DigitalData.Modules.Patterns
Imports DigitalData.Modules.ZooFlow.State
Imports DigitalData.Services.EDMIService.GlobalState
Namespace Methods.GlobalIndexer.ImportFile.Steps
Public Class AutomaticIndexing
Inherits BaseClass
Private ReadOnly GlobalState As GlobalState
Private ReadOnly AutomaticIndexes As List(Of AutomaticIndex)
Private ReadOnly Patterns As Patterns2
Private ReadOnly Database As MSSQLServer
Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pAutomaticIndexes As List(Of AutomaticIndex), pGlobalState As GlobalState)
MyBase.New(pLogConfig)
Database = pDatabase
GlobalState = pGlobalState
AutomaticIndexes = pAutomaticIndexes
Patterns = New Patterns2(pLogConfig)
End Sub
Public Function ApplyAutomaticeAttributes(pManualAttributes As List(Of UserAttributeValue), pFileInfo As FileInfo, pUserState As UserState) As List(Of UserAttributeValue)
Logger.Debug("Start of Method [ApplyAutomaticeAttributes]")
Dim oAttributes = pManualAttributes
For Each oAutomaticIndex In AutomaticIndexes
' We add oAttributes from the previous run into the current run so it is in theory possible to reference
' automatic attributes which have been set just before.
Dim oAttribute = ApplyAutomaticIndex(oAutomaticIndex, pFileInfo, pUserState, oAttributes)
oAttributes.Add(oAttribute)
Next
Return oAttributes
End Function
Private Function ApplyAutomaticIndex(pAutomaticIndex As AutomaticIndex, pFileInfo As FileInfo, pUserState As UserState, pAttributes As List(Of UserAttributeValue)) As UserAttributeValue
Dim oAttributeDict = pAttributes.ToDictionary(
Function(attr) attr.AttributeName,
Function(attr) attr.AttributeValues)
' If there is no SQL command, we use the Value property and replace all placeholders in it.
If pAutomaticIndex.HasSqlCommand = False Then
Dim oResult As String = GetPlaceholderValue(pAutomaticIndex.Value, pFileInfo, pUserState, oAttributeDict)
Return New UserAttributeValue With {
.AttributeValues = New List(Of String) From {oResult},
.AttributeName = pAutomaticIndex.Name,
.AttributeId = pAutomaticIndex.Id
}
End If
' Otherwise we will replace placeholders in the SQL command and then execute it
Dim oConnectionString As String = GlobalState.GetConnectionString(pAutomaticIndex.SQLConnectionId)
Dim oFinalSQLCommand = pAutomaticIndex.SQLCommand
oFinalSQLCommand = GetPlaceholderValue(oFinalSQLCommand, pFileInfo, pUserState, oAttributeDict)
' Now we have a SQL command which only contains vector placeholders
' Next, we execute the command to get our result
Dim oValue = Database.GetScalarValueWithConnection(oFinalSQLCommand, oConnectionString)
' TODO: Return multiple values
Return New UserAttributeValue With {
.AttributeValues = New List(Of String) From {oValue},
.AttributeName = pAutomaticIndex.Name,
.AttributeId = pAutomaticIndex.Id
}
End Function
Private Function GetPlaceholderValue(pValue As String, pFileInfo As FileInfo, pUserState As UserState, pAttributes As Dictionary(Of String, List(Of String))) As String
Dim oResult As String = pValue
oResult = Patterns.ReplaceInternalValues(oResult)
oResult = Patterns.ReplaceFileValues(oResult, pFileInfo)
oResult = Patterns.ReplaceUserValues(oResult, pUserState)
oResult = Patterns.ReplaceGlobixValues(oResult, pAttributes)
Return oResult
End Function
End Class
End Namespace

98
Service.EDMIService/Methods/GlobalIndexer/ImportFile/Steps/PostProcessing.vb Normal file
View File

@@ -0,0 +1,98 @@
Imports DigitalData.Modules.Logging
Imports DigitalData.Modules.Language
Namespace Methods.GlobalIndexer.ImportFile.Steps
Public Class PostProcessing
Inherits BaseClass
Public Const TYPE_VBSPLIT = "VBSPLIT"
Public Const TYPE_VBREPLACE = "VBREPLACE"
Public Const TYPE_REGEXPRESSION = "REG. EXPRESSION"
Private PostprocessingSteps As DataTable
Public Sub New(pLogConfig As LogConfig, pPostProcessingSteps As DataTable)
MyBase.New(pLogConfig)
PostprocessingSteps = pPostProcessingSteps
End Sub
Public Function ApplyManualPostprocessing(pManualAttributes As List(Of UserAttributeValue)) As List(Of UserAttributeValue)
Logger.Debug("Start of Method [ApplyManualPostprocessing]")
Dim oAttributes = pManualAttributes
For Each oProcessingRow As DataRow In PostprocessingSteps.Rows
Dim oIndexId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")
Dim oIndex As UserAttributeValue = pManualAttributes.
Where(Function(attr) attr.AttributeId = oProcessingRow.ItemEx(Of Integer)("IDXMAN_ID")).
FirstOrDefault()
Dim oValue = GetPostprocessingValue(oIndex.AttributeValues, oProcessingRow)
oAttributes.Add(New UserAttributeValue With {
.AttributeId = oIndexId,
.AttributeName = oIndex.AttributeName,
.AttributeValues = oIndex.AttributeValues,
.ControlName = oIndex.ControlName
})
Next
Return oAttributes
End Function
Public Function GetPostprocessingValue(pValues As List(Of String), pRow As DataRow)
Logger.Debug("Start of Method [GetPostprocessingValue]")
Dim oType = pRow.Item("TYPE")
Dim oResult As New List(Of String)
Logger.Debug("Type of Postprocessing is [{0}]", oType)
Select Case oType
Case TYPE_VBREPLACE
Dim oFindString = pRow.Item("TEXT1")
Dim oReplaceString = pRow.Item("TEXT2")
Logger.Debug("Replacing [{0}] with [{1}]", oFindString, oReplaceString)
For Each oIndexValue In pValues
Dim oReplaceResult = oIndexValue.Replace(oFindString, oReplaceString)
If oReplaceResult.Equals(oIndexValue) Then
Logger.Debug("Replace did not succeed, ReplaceString was not found.")
Else
Logger.Debug("Replace successful for [{0}].", oIndexValue)
End If
oResult.Add(oReplaceResult)
Next
Case TYPE_VBSPLIT
Dim oSeparator As String = pRow.Item("TEXT1")
Dim oSplitIndex As Integer = 0
Integer.TryParse(pRow.Item("TEXT2"), oSplitIndex)
Logger.Debug("Splitting String at Separator [{0}] and Index [{1}]", oSeparator, oSplitIndex)
For Each oIndexValue In pValues
Dim oSplitted As List(Of String) = oIndexValue.Split(oSeparator).ToList()
Logger.Debug("Split succeeded, resulting list has [{0}] items.", oSplitted.Count)
If oSplitIndex < oSplitted.Count Then
Dim oValue = oSplitted.Item(oSplitIndex)
Logger.Debug("Saving value [{0}] from Index [{1}]", oValue, oSplitIndex)
oResult.Add(oValue)
Else
Logger.Debug("SplitIndex(TEXT2) was out of array bounds. Skipping.")
End If
Next
Case Else
LogAndThrow($"Postprocessing type [{oType}] is not supported!")
End Select
Return oResult
End Function
End Class
End Namespace

12
Service.EDMIService/Methods/GlobalIndexer/ManualIndex.vb Normal file
View File

@@ -0,0 +1,12 @@
Namespace Methods.GlobalIndexer
Public Class ManualIndex
Inherits BaseIndex
Public DataType As String
Public IsOptional As Boolean
Public IsMultiselect As Boolean
Public DefaultValue As String
End Class
End Namespace

15
Service.EDMIService/Methods/NewFile/NewFileMethod.vb
View File

@@ -28,7 +28,7 @@ Public Class NewFileMethod
Try
Dim oObjectId = NewObjectId(pData.KindType, pData.BusinessEntity, pData.Who)
Dim oObjectId = NewObjectId(pData.KindType, pData.BusinessEntity, pData.User.UserName)
If oObjectId = 0 Then
LogAndThrow("Could not create new ObjectId!")
End If
@@ -107,7 +107,16 @@ Public Class NewFileMethod
Logger.Info("Creating IDB FileObject for ObjectId [{0}].", oObjectId)
' Insert into DB
Dim oSQL As String = $"EXEC PRIDB_NEW_IDBFO '{oFinalPath}', '{oFileObjectName}', '{oFileObjectExtension}',{oFileObjectSize},'{pData.File.FileChecksum}' ,'{pData.Who}','{oObjectId}',{oStore.Id}"
Dim oSQL As String = $"EXEC PRIDB_NEW_IDBFO
'{oFinalPath}',
'{oFileObjectName}',
'{oFileObjectExtension}',
{oFileObjectSize},
'{pData.File.FileChecksum}' ,
'{pData.User.UserName}',
'{oObjectId}',
{oStore.Id}"
Dim oResult As Boolean = Database.ExecuteNonQueryWithConnectionObject(oSQL, Connection, ExternalTransaction, Transaction)
If oResult = False Then
@@ -131,7 +140,7 @@ Public Class NewFileMethod
Continue For
End If
Dim oSuccess = Helpers.SetAttributeValue(Connection, Transaction, oObjectId, oAttribute.Key, oAttribute.Value, "de-DE", pData.Who)
Dim oSuccess = Helpers.SetAttributeValue(Connection, Transaction, oObjectId, oAttribute.Key, oAttribute.Value, pData.User.Language, pData.User.UserName)
If oSuccess Then
Logger.Debug("Default Attribute [{0}] written with value [{1}]", oAttribute.Key, oAttribute.Value)
Else

11
Service.EDMIService/Methods/NewFile/NewFileRequest.vb
View File

@@ -1,4 +1,5 @@
Imports System.Runtime.Serialization
Imports DigitalData.Modules.ZooFlow.State
Namespace Methods.NewFile
<Serializable>
@@ -26,17 +27,11 @@ Namespace Methods.NewFile
Public Property KindType As String
''' <summary>
''' The name of the user importing the file, ex. JenneJ
''' </summary>
<DataMember>
Public Property Who As String
''' <summary>
''' The language of the user
''' User Importing the file
''' </summary>
''' <returns></returns>
<DataMember>
Public Property Language As String
Public Property User As UserState
End Class
End Namespace