EDMIService: First working version of ImportFile

This commit is contained in:
Jonathan Jenne 2021-12-08 13:09:25 +01:00
parent d75272a17f
commit 3e5918297c
9 changed files with 180 additions and 118 deletions

View File

@ -39,7 +39,12 @@ Public Class frmtest
Dim oObjectId As Long = Await My.Application.Service.Client.ImportFileAsync(
txtFile2Import.Text,
txtProfileId.Text,
New List(Of EDMIServiceReference.UserAttributeValue),
New List(Of EDMIServiceReference.UserAttributeValue) From {
New EDMIServiceReference.UserAttributeValue With {
.AttributeName = "Attribut String1",
.AttributeValues = New List(Of String) From {"SchreiberM"}.ToArray
}
},
"WORK",
"DOC",
"DEFAULT"

View File

@ -5,8 +5,9 @@ Public Class BaseClass
Friend Logger As Logger
Public Sub New(pLogConfig As LogConfig)
Dim oClassName = [GetType]().Name
LogConfig = pLogConfig
Logger = pLogConfig.GetLogger()
Logger = pLogConfig.GetLogger(oClassName)
End Sub
Public Sub LogAndThrow(pMessage As String)

View File

@ -39,10 +39,10 @@ Public Class GlobalState
.DataSource = oConnection.Server,
.InitialCatalog = oConnection.Database,
.UserID = oConnection.Username,
.Password = MSSQLServer.DecryptConnectionString(oConnection.Password)
.Password = oConnection.Password
}
Return oBuilder.ToString
Return MSSQLServer.DecryptConnectionString(oBuilder.ToString)
End Function

View File

@ -10,7 +10,8 @@ Namespace Methods.GlobalIndexer.ImportFile
Private ReadOnly Patterns As Patterns2
Private ReadOnly GetDatatable As GetDatatableFromCacheMethod
Private ReadOnly Connection As SqlClient.SqlConnection
Private ReadOnly Transaction As SqlClient.SqlTransaction
Private Profile As DataRow
Private Const VIEW_PROFILE = "VWGI_DOCTYPE_IDB"
@ -23,6 +24,8 @@ Namespace Methods.GlobalIndexer.ImportFile
Patterns = New Patterns2(pLogConfig)
GetDatatable = New GetDatatableFromCacheMethod(LogConfig, Database, GlobalState)
Connection = Database.GetConnection()
Transaction = Connection.BeginTransaction()
End Sub
''' <summary>
@ -67,12 +70,52 @@ Namespace Methods.GlobalIndexer.ImportFile
If oResponse.OK Then
Logger.Info("Import of file [{0}] under ObjectId [{1}] successful!", pData.File.FileName, oResponse.ObjectId)
Return New ImportFileResponse(oResponse.ObjectId)
Else
Throw New ApplicationException(oResponse.ErrorMessage)
End If
Logger.Info("Writing Attributes for ObjectId [{0}]", oResponse.ObjectId)
Dim oAttributes As New Dictionary(Of String, Object)
For Each oFinalAttribute In oFinalAttributes
If oFinalAttribute.AttributeValues Is Nothing OrElse oFinalAttribute.AttributeValues.Count = 0 Then
Logger.Warn("Values for Attribute [{0}] are empty. Skipping.", oFinalAttribute.AttributeName)
Continue For
End If
oAttributes.Add(oFinalAttribute.AttributeName, oFinalAttribute.AttributeValues.First)
Next
For Each oAttribute As KeyValuePair(Of String, Object) In oAttributes
Try
' Dont write empty attributes
If oAttribute.Value Is Nothing Then
Continue For
End If
Dim oSuccess = Helpers.SetAttributeValue(Connection, Transaction, oResponse.ObjectId, oAttribute.Key, oAttribute.Value, pData.User.Language, pData.User.UserName)
If oSuccess Then
Logger.Info("Attribute [{0}] written with value [{1}]", oAttribute.Key, oAttribute.Value)
Else
Logger.Warn("Attribute value could not be written")
End If
Catch ex As Exception
LogAndThrow(ex, $"Attribute [{oAttribute.Key}] could not be written!")
End Try
Next
' Finally, commit the transaction
Transaction?.Commit()
Return New ImportFileResponse(oResponse.ObjectId)
Catch ex As Exception
Logger.Warn("Error occurred while importing file!")
Logger.Error(ex)
Logger.Info("Rolling back transaction.")
Transaction?.Rollback()
Return New ImportFileResponse(ex)
End Try
End Function
@ -142,7 +185,7 @@ Namespace Methods.GlobalIndexer.ImportFile
oIndexes.Add(oAutomaticIndex)
Next
Logger.Info("[{0}] automatic indexes loaded.", oIndexes)
Logger.Info("Automatic indexes loaded: [{0}]", oIndexes.Count)
Return oIndexes
Catch ex As Exception
@ -186,6 +229,8 @@ Namespace Methods.GlobalIndexer.ImportFile
oIndexes.Add(oManualIndex)
Next
Logger.Info("Manual indexes loaded: [{0}]", oIndexes.Count)
Return oIndexes
Catch ex As Exception

View File

@ -33,24 +33,29 @@ Namespace Methods.GlobalIndexer.ImportFile.Steps
End If
Logger.Info("Processing [{0}] automatic indexes", AutomaticIndexes.Count)
Dim oAttributes As List(Of UserAttributeValue) = pUserAttributes
Dim oUserAttributes As List(Of UserAttributeValue) = pUserAttributes
Dim oAutoAttributes As New List(Of UserAttributeValue)
For Each oAutomaticIndex In AutomaticIndexes
' We add oAttributes from the previous run into the current run so it is in theory possible to reference
' automatic attributes which have been set just before.
Dim oAttribute = ApplyAutomaticIndex(oAutomaticIndex, pFileInfo, pUserState, oAttributes)
Dim oAttribute = ApplyAutomaticIndex(oAutomaticIndex, pFileInfo, pUserState, oUserAttributes, oAutoAttributes)
If oAttribute IsNot Nothing Then
Logger.Info("Adding Attribute [{0}]", oAttribute)
oAttributes.Add(oAttribute)
Logger.Debug("Adding Attribute [{0}]", oAttribute)
oAutoAttributes.Add(oAttribute)
End If
Next
Return oAttributes
oUserAttributes.AddRange(oAutoAttributes)
Return oUserAttributes
End Function
Private Function ApplyAutomaticIndex(pAutomaticIndex As AutomaticIndex, pFileInfo As FileInfo, pUserState As UserState, pAttributes As List(Of UserAttributeValue)) As UserAttributeValue
Private Function ApplyAutomaticIndex(pAutomaticIndex As AutomaticIndex,
pFileInfo As FileInfo,
pUserState As UserState,
pAttributes As List(Of UserAttributeValue),
pAutoAttributes As List(Of UserAttributeValue)) As UserAttributeValue
Try
Dim oAttributeDict = pAttributes.ToDictionary(
Function(attr) attr.AttributeName,
@ -77,6 +82,8 @@ Namespace Methods.GlobalIndexer.ImportFile.Steps
Dim oConnectionString As String = GlobalState.GetConnectionString(pAutomaticIndex.SQLConnectionId)
Dim oFinalSQLCommand = pAutomaticIndex.SQLCommand
' TODO: Dont show the unmasked conn string
Logger.Debug("SQL Connection String is: [{0}]", oConnectionString)
Logger.Debug("SQL Command is: [{0}]", oFinalSQLCommand)
oFinalSQLCommand = GetPlaceholderValue(oFinalSQLCommand, pFileInfo, pUserState, oAttributeDict)
@ -109,11 +116,12 @@ Namespace Methods.GlobalIndexer.ImportFile.Steps
Private Function GetPlaceholderValue(pValue As String, pFileInfo As FileInfo, pUserState As UserState, pAttributes As Dictionary(Of String, List(Of String))) As String
Dim oResult As String = pValue
oResult = Patterns.ReplaceInternalValues(oResult)
oResult = Patterns.ReplaceFileValues(oResult, pFileInfo)
oResult = Patterns.ReplaceUserValues(oResult, pUserState)
oResult = Patterns.ReplaceGlobixValues(oResult, pAttributes)
' TODO: Get the automatic indexes in here too
oResult = Patterns.ReplaceGlobixValues(oResult, New Dictionary(Of String, List(Of String)), pAttributes)
Return oResult
End Function

View File

@ -127,13 +127,13 @@ Public Class NewFileMethod
'TODO: File dates in try catch
Dim oDefaultAttributes As New Dictionary(Of String, Object) From {
Dim oSystemAttributes As New Dictionary(Of String, Object) From {
{"OriginFileName", pData.File.FileName},
{"OriginCreationDatetime", pData.File.FileCreatedAt},
{"OriginChangedDatetime", pData.File.FileChangedAt}
}
For Each oAttribute As KeyValuePair(Of String, Object) In oDefaultAttributes
For Each oAttribute As KeyValuePair(Of String, Object) In oSystemAttributes
Try
' Dont write empty attributes
If oAttribute.Value Is Nothing Then
@ -142,9 +142,9 @@ Public Class NewFileMethod
Dim oSuccess = Helpers.SetAttributeValue(Connection, Transaction, oObjectId, oAttribute.Key, oAttribute.Value, pData.User.Language, pData.User.UserName)
If oSuccess Then
Logger.Debug("Default Attribute [{0}] written with value [{1}]", oAttribute.Key, oAttribute.Value)
Logger.Debug("System Attribute [{0}] written with value [{1}]", oAttribute.Key, oAttribute.Value)
Else
Logger.Warn("Default attribute value could not be written")
Logger.Warn("System attribute value could not be written")
End If
Catch ex As Exception
LogAndThrow(ex, $"System attribute [{oAttribute.Key}] could not be written!")
@ -158,7 +158,7 @@ Public Class NewFileMethod
Return New NewFile.NewFileResponse(oObjectId)
Catch ex As Exception
Logger.Warn("Error occurred while importing file!")
Logger.Warn("Error occurred while creating file!")
Logger.Error(ex)
Logger.Info("Cleaning up files.")

View File

@ -18,7 +18,7 @@ Public Class DatatableJob
Dim oLogConfig As LogConfig = oJobData.Item("LogConfig")
Dim oCronJobId As Integer = oJobData.Item("CronJobId")
Dim oCronJobTitle As String = oJobData.Item("CronJobTitle")
Dim oLogger As Logger = oLogConfig.GetLoggerFor("Scheduler")
Dim oLogger As Logger = oLogConfig.GetLogger()
Dim oResult As New JobResult()

View File

@ -21,7 +21,7 @@ Public Class JobListener
MyBase.New()
_LogConfig = LogConfig
_Logger = LogConfig.GetLoggerFor("Scheduler")
_Logger = LogConfig.GetLogger()
_MSSQL = MSSQL
Dataset = ResultDataSet
End Sub

View File

@ -10,22 +10,21 @@ Imports System.ServiceModel.Channels
Public Class WindowsService
Inherits ServiceBase
Private _ServiceHost As ServiceHost(Of EDMIService)
Private _LogConfig As LogConfig
Private _LogConfigScheduler As LogConfig
Private _Logger As Logger
Private ServiceHost As ServiceHost(Of EDMIService)
Private LogConfig As LogConfig
Private LogConfigScheduler As LogConfig
Private Logger As Logger
Private _Firebird As Firebird
Private _MSSQL_ECM As MSSQLServer
Private _MSSQL_IDB As MSSQLServer
Private Firebird As Firebird
Private MSSQL_ECM As MSSQLServer
Private MSSQL_IDB As MSSQLServer
Private _ConfigManager As ConfigManager(Of Config)
Private _Config As Config
Private _Path As EDMI.File.Path
Private _Archive As EDMI.File.Archive
Private _Filesystem As Filesystem.File
Private _Global As GlobalState
Private _Scheduler As Scheduler
Private ConfigManager As ConfigManager(Of Config)
Private Config As Config
Private Archive As EDMI.File.Archive
Private Filesystem As Filesystem.File
Private GlobalState As GlobalState
Private Scheduler As Scheduler
Public Sub New()
ServiceName = SERVICE_NAME
@ -39,123 +38,127 @@ Public Class WindowsService
Try
Dim oServicePath As String = AppDomain.CurrentDomain.BaseDirectory
_LogConfig = New LogConfig(LogConfig.PathType.CustomPath, IO.Path.Combine(oServicePath, "Log"), FileKeepRangeInDays:=3)
_LogConfigScheduler = New LogConfig(LogConfig.PathType.CustomPath, IO.Path.Combine(oServicePath, "Log"), Suffix:="Scheduler", FileKeepRangeInDays:=3)
_Logger = _LogConfig.GetLogger()
LogConfig = New LogConfig(LogConfig.PathType.CustomPath, IO.Path.Combine(oServicePath, "Log"), FileKeepRangeInDays:=3)
LogConfigScheduler = New LogConfig(LogConfig.PathType.CustomPath, IO.Path.Combine(oServicePath, "Log"), Suffix:="Scheduler", FileKeepRangeInDays:=3)
Logger = LogConfig.GetLogger()
_Logger.Info("Service {0} is starting...", SERVICE_DISPLAY_NAME)
_Logger.Info("ServiceDirectory: {0}", oServicePath)
Logger.Info("Service {0} is starting...", SERVICE_DISPLAY_NAME)
Logger.Info("ServiceDirectory: {0}", oServicePath)
_Logger.Info("Loading Config")
_ConfigManager = New ConfigManager(Of Config)(_LogConfig, oServicePath)
_Config = _ConfigManager.Config
_LogConfig.Debug = _ConfigManager.Config.Debug
Logger.Info("Loading Config")
ConfigManager = New ConfigManager(Of Config)(LogConfig, oServicePath)
Config = ConfigManager.Config
LogConfig.Debug = ConfigManager.Config.Debug
Dim oTimer As New Timers.Timer(60000)
AddHandler oTimer.Elapsed, Sub()
_Logger.Debug("Reloading config..")
_ConfigManager.Reload()
_Config = _ConfigManager.Config
_LogConfig.Debug = _ConfigManager.Config.Debug
'UpdateTraceLogging()
End Sub
AddHandler oTimer.Elapsed, AddressOf ReloadTimer_Tick
oTimer.Start()
_Logger.Debug("Connecting to Databases")
Logger.Debug("Connecting to Databases")
_Firebird = StartFirebird()
_MSSQL_ECM = StartMSSQL_ECM()
_MSSQL_IDB = StartMSSQL_IDB()
Firebird = StartFirebird()
MSSQL_ECM = GetMSSQL_ECM(LogConfig)
MSSQL_IDB = GetMSSQL_IDB(LogConfig)
_Logger.Debug("Initializing EDMI Functions")
Logger.Debug("Initializing EDMI Functions")
_Archive = New EDMI.File.Archive(_LogConfig)
_Filesystem = New Filesystem.File(_LogConfig)
_Global = New GlobalState(_LogConfig, _MSSQL_IDB, _MSSQL_ECM)
_Scheduler = New Scheduler(_LogConfigScheduler, _MSSQL_ECM, _Global.TableStore)
Archive = New EDMI.File.Archive(LogConfig)
Filesystem = New Filesystem.File(LogConfig)
GlobalState = New GlobalState(LogConfig, MSSQL_IDB, MSSQL_ECM)
_Logger.Debug("Loading Global Data")
_Global.LoadObjectStores()
_Global.LoadConnections()
Dim oMSSQLServer = GetMSSQL_ECM(LogConfigScheduler)
Scheduler = New Scheduler(LogConfigScheduler, oMSSQLServer, GlobalState.TableStore)
_Logger.Debug("Starting Scheduler")
_Scheduler.Start()
Logger.Debug("Loading Global Data")
GlobalState.LoadObjectStores()
GlobalState.LoadConnections()
_Logger.Debug("Preparing WCF ServiceHost")
EDMIService.MSSQL_ECM = _MSSQL_ECM
EDMIService.MSSQL_IDB = _MSSQL_IDB
EDMIService.Firebird = _Firebird
EDMIService.LogConfig = _LogConfig
EDMIService.AppConfig = _Config
EDMIService.EDMIArchive = _Archive
EDMIService.Filesystem = _Filesystem
EDMIService.GlobalState = _Global
EDMIService.Scheduler = _Scheduler
Logger.Debug("Starting Scheduler")
Scheduler.Start()
_Logger.Debug("Starting WCF ServiceHost")
Logger.Debug("Preparing WCF ServiceHost")
EDMIService.MSSQL_ECM = MSSQL_ECM
EDMIService.MSSQL_IDB = MSSQL_IDB
EDMIService.Firebird = Firebird
EDMIService.LogConfig = LogConfig
EDMIService.AppConfig = Config
EDMIService.EDMIArchive = Archive
EDMIService.Filesystem = Filesystem
EDMIService.GlobalState = GlobalState
EDMIService.Scheduler = Scheduler
Logger.Debug("Starting WCF ServiceHost")
Dim oBaseAddresses() As Uri = {New Uri(SERVICE_BASE_ADDRESS)}
_ServiceHost = New ServiceHost(Of EDMIService)(oBaseAddresses)
_ServiceHost.EnableMetadataExchange(False)
ServiceHost = New ServiceHost(Of EDMIService)(oBaseAddresses)
ServiceHost.EnableMetadataExchange(False)
_Logger.Debug("Listing Endpoints:")
For Each oEndpoint In _ServiceHost.Description.Endpoints
_Logger.Debug("Name: {0}", oEndpoint.Name)
_Logger.Debug("Address: {0}", oEndpoint.Address.ToString)
_Logger.Debug("Listen Uri: {0}", oEndpoint.ListenUri.AbsoluteUri)
_Logger.Debug("Binding: {0}", oEndpoint.Binding.Name)
_Logger.Debug("Contract: {0}", oEndpoint.Contract.Name)
Logger.Debug("Listing Endpoints:")
For Each oEndpoint In ServiceHost.Description.Endpoints
Logger.Debug("Name: {0}", oEndpoint.Name)
Logger.Debug("Address: {0}", oEndpoint.Address.ToString)
Logger.Debug("Listen Uri: {0}", oEndpoint.ListenUri.AbsoluteUri)
Logger.Debug("Binding: {0}", oEndpoint.Binding.Name)
Logger.Debug("Contract: {0}", oEndpoint.Contract.Name)
Next
_ServiceHost.Open()
ServiceHost.Open()
_Logger.Info("WCF ServiceHost started")
_Logger.Info("Service {0} successfully started", SERVICE_DISPLAY_NAME)
Logger.Info("WCF ServiceHost started")
Logger.Info("Service {0} successfully started", SERVICE_DISPLAY_NAME)
Catch ex As Exception
_Logger.Warn("Unexpected Error while starting the service: {0}", ex.Message)
_Logger.Error(ex)
Logger.Warn("Unexpected Error while starting the service: {0}", ex.Message)
Logger.Error(ex)
GracefullyStop()
End Try
End Sub
Private Function StartFirebird() As Firebird
_Logger.Debug("Connecting to Firebird")
Private Sub ReloadTimer_Tick()
If ConfigManager.Reload() = False Then
Logger.Warn("Could not reload config, check the service and config file.")
End If
If _Config.Firebird_Datasource = String.Empty Then
_Logger.Info("Firebird database not configured. Skipping.")
Config = ConfigManager.Config
LogConfig.Debug = ConfigManager.Config.Debug
End Sub
Private Function StartFirebird() As Firebird
Logger.Debug("Connecting to Firebird")
If Config.Firebird_Datasource = String.Empty Then
Logger.Info("Firebird database not configured. Skipping.")
Return Nothing
End If
Try
Dim oFirebird = New Firebird(
_LogConfig,
_Config.Firebird_Datasource,
_Config.Firebird_DatabaseName,
_Config.Firebird_DatabaseUser,
_Config.Firebird_DatabasePassword
LogConfig,
Config.Firebird_Datasource,
Config.Firebird_DatabaseName,
Config.Firebird_DatabaseUser,
Config.Firebird_DatabasePassword
)
_Logger.Info("Database connection established.")
Logger.Info("Database connection established.")
Return oFirebird
Catch ex As Exception
_Logger.Warn("StartFirebird: Could not connect to firebird database.")
_Logger.Error(ex)
Logger.Warn("StartFirebird: Could not connect to firebird database.")
Logger.Error(ex)
Return Nothing
End Try
End Function
Private Function StartMSSQL_ECM() As MSSQLServer
_Logger.Debug("Connecting to ECM MSSQL")
Dim oMSSQL = New MSSQLServer(_LogConfig, _Config.ConnectionString_ECM)
_Logger.Info("Database connection to ECM Database established.")
Private Function GetMSSQL_ECM(pLogConfig As LogConfig) As MSSQLServer
Logger.Debug("Connecting to ECM MSSQL")
Dim oMSSQL = New MSSQLServer(pLogConfig, Config.ConnectionString_ECM)
Logger.Info("Database connection to ECM Database established.")
Return oMSSQL
End Function
Private Function StartMSSQL_IDB() As MSSQLServer
_Logger.Debug("Connecting to IDB MSSQL")
Dim oMSSQL = New MSSQLServer(_LogConfig, _Config.ConnectionString_IDB)
_Logger.Info("Database connection to IDB Database established.")
Private Function GetMSSQL_IDB(pLogConfig As LogConfig) As MSSQLServer
Logger.Debug("Connecting to IDB MSSQL")
Dim oMSSQL = New MSSQLServer(pLogConfig, Config.ConnectionString_IDB)
Logger.Info("Database connection to IDB Database established.")
Return oMSSQL
End Function
@ -164,13 +167,13 @@ Public Class WindowsService
End Sub
Private Sub GracefullyStop()
_Logger.Info("Service {0} is stopping!", SERVICE_DISPLAY_NAME)
If _ServiceHost IsNot Nothing Then
_ServiceHost.Close()
_ServiceHost = Nothing
Logger.Info("Service {0} is stopping!", SERVICE_DISPLAY_NAME)
If ServiceHost IsNot Nothing Then
ServiceHost.Close()
ServiceHost = Nothing
End If
_Scheduler.Stop()
Scheduler.Stop()
End Sub
End Class