336 lines
15 KiB
VB.net
336 lines
15 KiB
VB.net
Imports ECM.JobRunner.Windows.Scheduler.Jobs
|
|
Imports Quartz
|
|
Imports System.Text.RegularExpressions
|
|
Imports DigitalData.Modules.Filesystem
|
|
Imports ECM.JobRunner.Common
|
|
|
|
Namespace Scheduler.Jobs
|
|
|
|
Public Class FileImportJob
|
|
Inherits BaseJob
|
|
Implements IJob
|
|
|
|
Private FileEx As File
|
|
|
|
Public Function Execute(context As IJobExecutionContext) As Task Implements IJob.Execute
|
|
Dim oArgs = MyBase.InitializeJob(context)
|
|
|
|
Try
|
|
FileEx = New File(LogConfig)
|
|
|
|
Dim oProfile = State.ProfileDefintions.ImportProfiles.Where(Function(p) p.JobId = Id).SingleOrDefault()
|
|
|
|
Logger.Info("Running File Import [{0}]", Name)
|
|
Logger.Info("Source directory: [{0}]", oProfile.SourceFolder)
|
|
Logger.Info("Target directory: [{0}]", oProfile.TargetFolder)
|
|
Logger.Info("Backup directory: [{0}]", oProfile.BackupFolder)
|
|
|
|
Dim oObjectType = State.ObjectTypes.Where(Function(o) o.Name = oProfile.ObjectTypeName).SingleOrDefault()
|
|
|
|
If IO.Directory.Exists(oProfile.SourceFolder) = False Then
|
|
LogError("Source directory [{0}] does not exist!", oProfile.SourceFolder)
|
|
Return Task.FromResult(False)
|
|
End If
|
|
|
|
Dim oRecursive As Boolean = oProfile.IncludeSubfolders
|
|
Dim oFileNames = GetFiles(oProfile.SourceFolder, oRecursive)
|
|
Dim oRegexList As List(Of Regex) = GetRegexList(oProfile.FileExcludeRegex)
|
|
|
|
Logger.Debug("[{0}] Regexes loaded", oRegexList.Count)
|
|
|
|
If oFileNames.Count = 0 Then
|
|
Logger.Info("No Files for Profile [{0}]", Name)
|
|
Return CompleteJob("No files for profile")
|
|
End If
|
|
|
|
LogInfo("{0} files found in source directory {1}", oFileNames.Count, oProfile.SourceFolder)
|
|
|
|
' - [ ] Process Rules, build list of files and indexes
|
|
' - [x] Process Regex to filter out files
|
|
' - [x] Check time to filter out files
|
|
' - [ ] (Check if files can be accessed)
|
|
' - [ ] Check if backup is needed and backup files
|
|
' - [ ] Import into windream
|
|
' - [ ] Create original subfolder structure
|
|
' - [ ] Create DateTime Subfolders
|
|
' - [x] Check if file exists and version
|
|
' - [x] Do import
|
|
' - [ ] Check for filesize 0
|
|
' - [ ] Write indexes (using data from getimportfile)
|
|
' - [ ] Check if orig file should be deleted
|
|
' - [ ] (delete subdirectories in source path)
|
|
|
|
Dim oFiles = oFileNames.
|
|
Select(Function(f) New ImportFile(f)).
|
|
ToList()
|
|
Dim oFilteredFiles = oFiles
|
|
|
|
' Check time to filter out files
|
|
Dim oDateFilteredFiles = oFilteredFiles.Where(Function(f) FileIsOlderThan(f, pMinutes:=1)).ToList()
|
|
Dim oDateFilteredCount = oFilteredFiles.Except(oDateFilteredFiles).Count()
|
|
LogDebug("{0} Files filtered out for being too new.", oDateFilteredCount)
|
|
oFilteredFiles = oDateFilteredFiles
|
|
|
|
' Process Regex to filter out files
|
|
Dim oRegexFilteredFiles = oFilteredFiles.Where(Function(f) Not FileMatchesRegex(f, oRegexList))
|
|
Dim oRegexFilteredCount = oFilteredFiles.Except(oRegexFilteredFiles).Count()
|
|
LogDebug("{0} Files filtered out for matching exclusion Regex.", oRegexFilteredCount)
|
|
oFilteredFiles = oDateFilteredFiles
|
|
|
|
'-------------------------------------------------------------------------------------------------
|
|
' After this point, files are treated as processed and are being deleted before finishing the job
|
|
'-------------------------------------------------------------------------------------------------
|
|
|
|
If oFilteredFiles.Count = 0 Then
|
|
Return CompleteJob($"No files to process.")
|
|
End If
|
|
|
|
Logger.Info("Importing files..")
|
|
Dim oImportedFiles As New List(Of ImportFile)
|
|
For Each oFile In oFilteredFiles
|
|
|
|
Dim oImportedPath = ImportFile(oFile, oProfile)
|
|
If oImportedPath Is Nothing Then
|
|
Logger.Warn("File [{0}] could not be imported!", oFile.FilePath)
|
|
Continue For
|
|
End If
|
|
oFile.FilePathWindream = oImportedPath
|
|
|
|
oImportedFiles.Add(oFile)
|
|
Next
|
|
|
|
LogInfo("{0} files successfully Imported!", oImportedFiles.Count)
|
|
|
|
Dim oIndexedFiles As New List(Of ImportFile)
|
|
For Each oFile In oImportedFiles
|
|
Logger.Info("Indexing file [{0}]", oFile.FilePathWindream)
|
|
Dim oIndexResult = IndexFile(oFile, oProfile)
|
|
If oIndexResult = True Then
|
|
oIndexedFiles.Add(oFile)
|
|
End If
|
|
Logger.Info("Indexing of file [{0}] done!", oFile.FilePathWindream)
|
|
Next
|
|
|
|
LogInfo("{0} files successfully Indexed!", oIndexedFiles.Count)
|
|
|
|
If oProfile.DeleteFiles = True Then
|
|
Logger.Debug("Deleting [{0}] files before finishing job.", oFiles.Count)
|
|
|
|
For Each oFile In oFilteredFiles
|
|
Try
|
|
IO.File.Delete(oFile.FilePathOriginal)
|
|
Catch ex As Exception
|
|
Logger.Warn("Could not clean up processed files before finishing Job!")
|
|
Logger.Error(ex)
|
|
End Try
|
|
Next
|
|
End If
|
|
|
|
Return CompleteJob($"{oImportedFiles.Count} files successfully Processed!")
|
|
Catch ex As Exception
|
|
Logger.Error(ex)
|
|
LogError("Unexpected Error: [{0}]", ex.Message)
|
|
|
|
Return CompleteJobWithError(ex)
|
|
End Try
|
|
End Function
|
|
|
|
Private Function FileIsOlderThan(pFile As ImportFile, pMinutes As Integer)
|
|
Dim oCreationTime = pFile.FileInfo.CreationTime
|
|
Dim oTimeSpan = New TimeSpan(0, pMinutes, 0)
|
|
|
|
If (Now - oCreationTime) > oTimeSpan Then
|
|
Return True
|
|
Else
|
|
Return False
|
|
End If
|
|
End Function
|
|
|
|
Private Function FileMatchesRegex(pFile As ImportFile, pRegexList As List(Of Regex))
|
|
Return pRegexList.Any(Function(r) r.IsMatch(pFile.FilePath))
|
|
End Function
|
|
|
|
Private Function GetRegexList(pRegexString As String) As List(Of Regex)
|
|
Return pRegexString.
|
|
Split(vbNewLine).
|
|
ToList().
|
|
Where(Function(r) String.IsNullOrWhiteSpace(r) = False).
|
|
Select(Function(s)
|
|
Logger.Debug("Regex loaded: [{0}]", s)
|
|
Return New Regex(s)
|
|
End Function).ToList()
|
|
|
|
End Function
|
|
|
|
Private Function ImportFile(pFile As ImportFile, pProfile As ImportProfile) As String
|
|
'Check if target folder exists
|
|
If Windream.TestFolderExists(pProfile.TargetFolder) = False Then
|
|
If Windream.NewFolder(pProfile.TargetFolder) = False Then
|
|
Logger.Warn("Folder [{0}] could not be created!", pProfile.TargetFolder)
|
|
Return Nothing
|
|
End If
|
|
End If
|
|
|
|
' Generate new filepath and stream file
|
|
Dim oFileName = IO.Path.GetFileName(pFile.FilePath)
|
|
Dim oNewFilePath As String = IO.Path.Combine(pProfile.TargetFolder, oFileName)
|
|
Dim oVersionedFilePath = GetVersionedWindreamPath(oNewFilePath)
|
|
|
|
If Windream.NewFileStream(pFile.FilePathOriginal, oVersionedFilePath, pProfile.ObjectTypeName) = False Then
|
|
Logger.Warn("File [{0}] could not be streamed to path [{1}]!", pFile.FilePathOriginal, oNewFilePath)
|
|
Return Nothing
|
|
End If
|
|
|
|
Return oVersionedFilePath
|
|
End Function
|
|
|
|
Private Function GetVersionedWindreamPath(pWindreamFilePath As String) As String
|
|
Dim oAbsolutePath = Windream.GetAbsolutePath(pWindreamFilePath)
|
|
|
|
' This versions the filename but does not rely on access though the filesystem.
|
|
' Instead the check for file existence is made through the windream sdk.
|
|
Dim oVersionedPath = FileEx.GetVersionedFilenameWithFilecheck(oAbsolutePath, Function(pPath As String) Windream.TestFileExists(pPath))
|
|
Dim oVersionedAndNormalizedPath = Windream.GetNormalizedPath(oVersionedPath, False)
|
|
|
|
Return oVersionedAndNormalizedPath
|
|
End Function
|
|
|
|
Private Function GetImportFile(pFilename As String, pProfile As ImportProfile) As ImportFile
|
|
Dim oImportFile = New ImportFile(pFilename)
|
|
|
|
Return oImportFile
|
|
End Function
|
|
|
|
Private Function ProcessSteps(pFile As ImportFile, pProfile As ImportProfile) As List(Of ImportFile.IndexItem)
|
|
Dim oIndexItems = New List(Of ImportFile.IndexItem)
|
|
|
|
' Process Steps on Filename
|
|
For Each oStep In pProfile.Steps
|
|
Dim oValue As String
|
|
|
|
' Get the base value by checking scope
|
|
Select Case oStep.Scope
|
|
Case Common.Constants.SCOPE_FILE
|
|
oValue = pFile.FileInfo.Name
|
|
Case Common.Constants.SCOPE_FOLDER
|
|
oValue = pFile.FileInfo.DirectoryName
|
|
Case Else
|
|
oValue = pFile.FilePath
|
|
End Select
|
|
|
|
Logger.Info("Running Method [{0}] on Index [{1}]", oStep.Method, oStep.IndexName)
|
|
Logger.Debug("Value is [{0}]", oValue)
|
|
Logger.Debug("Scope is [{0}]", oStep.Scope)
|
|
|
|
Select Case oStep.Method
|
|
Case Common.Constants.METHOD_SUBSTRING
|
|
Try
|
|
Logger.Debug("Parsing Value [{0}] for Parameter 'StartIndex'", oStep.Argument1)
|
|
Dim oStartIndex = 0
|
|
If Integer.TryParse(oStep.Argument1, oStartIndex) = False Then
|
|
Throw New ArgumentException("StartIndex")
|
|
End If
|
|
|
|
Logger.Debug("Parsing Value [{0}] for Parameter 'Length'", oStep.Argument2)
|
|
Dim oLength = 0
|
|
If Integer.TryParse(oStep.Argument2, oLength) = False Then
|
|
Throw New ArgumentException("Length")
|
|
End If
|
|
|
|
oValue = oValue.Substring(oStartIndex, oLength)
|
|
Catch ex As Exception
|
|
Dim oMessage = "Method SUBSTRING could not be applied to Index '{0}' and Parameters StartIndex [{1}], Length [{2}]. Error: '{3}'"
|
|
LogError(oMessage, oStep.IndexName, ex.Message, oStep.Argument1, oStep.Argument2)
|
|
End Try
|
|
|
|
Case Common.Constants.METHOD_SPLIT
|
|
Try
|
|
If String.IsNullOrEmpty(oStep.Argument1) Then
|
|
Throw New ArgumentException("Separator")
|
|
End If
|
|
Dim oSeparator = oStep.Argument1.Substring(0, 1)
|
|
|
|
Dim oIndex = 0
|
|
If Integer.TryParse(oStep.Argument2, oIndex) = False Then
|
|
Throw New ArgumentException("Index")
|
|
End If
|
|
|
|
Dim oSplit = oValue.Split(oSeparator)
|
|
oValue = oSplit(oIndex)
|
|
Catch ex As Exception
|
|
LogError("Method SPLIT could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
|
|
End Try
|
|
|
|
Case Common.Constants.METHOD_REGEX
|
|
Try
|
|
Dim oRegex = New Regex(oStep.Argument1)
|
|
Dim oTrueValue = oStep.Argument2
|
|
Dim oFalseValue = oStep.Argument3
|
|
If oRegex.IsMatch(oValue) Then
|
|
oValue = oTrueValue
|
|
Else
|
|
oValue = oFalseValue
|
|
End If
|
|
Catch ex As Exception
|
|
LogError("Method REGEX could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
|
|
End Try
|
|
|
|
Case Common.Constants.METHOD_VALUE
|
|
oValue = oStep.Argument1
|
|
|
|
Case Common.Constants.METHOD_ALL
|
|
'noop
|
|
Case Else
|
|
'noop
|
|
End Select
|
|
|
|
oIndexItems.Add(New ImportFile.IndexItem With {
|
|
.IndexName = oStep.IndexName,
|
|
.Value = oValue
|
|
})
|
|
Next
|
|
|
|
Return oIndexItems
|
|
End Function
|
|
|
|
Private Function IndexFile(pFile As ImportFile, pProfile As ImportProfile) As Boolean
|
|
Logger.Debug("Writing [{0}] indexes for File [{1}]", pFile.IndexValues.Count, pFile.FilePathWindream)
|
|
|
|
Dim oIndexItems = ProcessSteps(pFile, pProfile)
|
|
|
|
Dim oResults = oIndexItems.
|
|
Select(Function(v)
|
|
Dim oIndexResult = False
|
|
Logger.Info("Writing Index [{0}] with value [{1}]", v.IndexName, v.Value)
|
|
|
|
If Windream.TestIndexNameIsVectorIndex(v.IndexName) Then
|
|
oIndexResult = Windream.SetFileIndex(pFile.FilePathWindream, v.IndexName, New List(Of String) From {v.Value}, pProfile.ObjectTypeName)
|
|
Else
|
|
oIndexResult = Windream.SetFileIndex(pFile.FilePathWindream, v.IndexName, v.Value, pProfile.ObjectTypeName)
|
|
End If
|
|
|
|
Return oIndexResult
|
|
End Function).
|
|
ToList()
|
|
|
|
' Return True if all Indexes were set correctly
|
|
Return oResults.All(Function(r) r = True)
|
|
End Function
|
|
|
|
Private Function GetFiles(pDirectory As String, pRecursive As Boolean) As String()
|
|
Dim oFiles As String()
|
|
|
|
If pRecursive Then
|
|
oFiles = IO.Directory.GetFiles(pDirectory, "*.*", IO.SearchOption.AllDirectories)
|
|
Logger.Info("Found [{0}] files in Folder [{1}] (and subdirectories)", oFiles.Count, pDirectory)
|
|
Else
|
|
oFiles = IO.Directory.GetFiles(pDirectory, "*.*", IO.SearchOption.TopDirectoryOnly)
|
|
Logger.Info("Found [{0}] files in Folder [{1}]", oFiles.Count, pDirectory)
|
|
End If
|
|
|
|
Return oFiles
|
|
End Function
|
|
End Class
|
|
|
|
End Namespace
|