15-12-2022
This commit is contained in:
Jonathan Jenne
@@ -120,6 +120,14 @@ Namespace Scheduler.Jobs
|
||||
LogStep(HistoryItem.StepLevel.Debug, "{0} Files filtered out for matching exclusion Regex.", oRegexFilteredCount)
|
||||
oFilteredFiles = oDateFilteredFiles
|
||||
|
||||
'-------------------------------------------------------------------------------------------------
|
||||
' After this point, files are treated as processed and are being deleted before finishing the job
|
||||
'-------------------------------------------------------------------------------------------------
|
||||
|
||||
If oFilteredFiles.Count = 0 Then
|
||||
Return CompleteJob($"No files to process.")
|
||||
End If
|
||||
|
||||
Logger.Info("Importing files..")
|
||||
Dim oImportedFiles As New List(Of ImportFile)
|
||||
For Each oFile In oFilteredFiles
|
||||
@@ -150,6 +158,19 @@ Namespace Scheduler.Jobs
|
||||
Logger.Info("[{0}] files successfully Indexed!", oIndexedFiles.Count)
|
||||
LogStep(HistoryItem.StepLevel.Info, "{0} files successfully Indexed!", oIndexedFiles.Count)
|
||||
|
||||
If oProfile.DeleteFiles = True Then
|
||||
Logger.Debug("Deleting [{0}] files before finishing job.", oFiles.Count)
|
||||
|
||||
For Each oFile In oFilteredFiles
|
||||
Try
|
||||
IO.File.Delete(oFile.FilePathOriginal)
|
||||
Catch ex As Exception
|
||||
Logger.Warn("Could not clean up processed files before finishing Job!")
|
||||
Logger.Error(ex)
|
||||
End Try
|
||||
Next
|
||||
End If
|
||||
|
||||
Return CompleteJob($"{oImportedFiles.Count} files successfully Processed!")
|
||||
Catch ex As Exception
|
||||
Logger.Error(ex)
|
||||
@@ -161,7 +182,14 @@ Namespace Scheduler.Jobs
|
||||
End Function
|
||||
|
||||
Private Function FileIsOlderThan(pFile As ImportFile, pMinutes As Integer)
|
||||
Return pFile.FileInfo.CreationTime.AddMinutes(pMinutes) < Now
|
||||
Dim oCreationTime = pFile.FileInfo.CreationTime
|
||||
Dim oTimeSpan = New TimeSpan(0, pMinutes, 0)
|
||||
|
||||
If (Now - oCreationTime) > oTimeSpan Then
|
||||
Return True
|
||||
Else
|
||||
Return False
|
||||
End If
|
||||
End Function
|
||||
|
||||
Private Function FileMatchesRegex(pFile As ImportFile, pRegexList As List(Of Regex))
|
||||
@@ -228,31 +256,52 @@ Namespace Scheduler.Jobs
|
||||
|
||||
' Get the base value by checking scope
|
||||
Select Case oStep.Scope
|
||||
Case ImportProfileStep.StepScope.FILE
|
||||
Case Common.Constants.SCOPE_FILE
|
||||
oValue = pFile.FileInfo.Name
|
||||
Case ImportProfileStep.StepScope.FOLDER
|
||||
Case Common.Constants.SCOPE_FOLDER
|
||||
oValue = pFile.FileInfo.DirectoryName
|
||||
Case Else
|
||||
oValue = pFile.FilePath
|
||||
End Select
|
||||
|
||||
Logger.Info("Running Method [{0}] on Index [{1}]", oStep.Method, oStep.IndexName)
|
||||
Logger.Debug("Value is [{0}]", oValue)
|
||||
Logger.Debug("Scope is [{0}]", oStep.Scope)
|
||||
|
||||
' TODO: Error handling!
|
||||
Select Case oStep.Method
|
||||
Case ImportProfileStep.StepMethod.SUBSTRING
|
||||
Case Common.Constants.METHOD_SUBSTRING
|
||||
Try
|
||||
Dim oIndex1 = Integer.Parse(oStep.Argument1)
|
||||
Dim oLength = Integer.Parse(oStep.Argument2)
|
||||
oValue = oValue.Substring(oIndex1, oLength)
|
||||
Logger.Debug("Parsing Value [{0}] for Parameter 'StartIndex'", oStep.Argument1)
|
||||
Dim oStartIndex = 0
|
||||
If Integer.TryParse(oStep.Argument1, oStartIndex) = False Then
|
||||
Throw New ArgumentException("StartIndex")
|
||||
End If
|
||||
|
||||
Logger.Debug("Parsing Value [{0}] for Parameter 'Length'", oStep.Argument2)
|
||||
Dim oLength = 0
|
||||
If Integer.TryParse(oStep.Argument2, oLength) = False Then
|
||||
Throw New ArgumentException("Length")
|
||||
End If
|
||||
|
||||
oValue = oValue.Substring(oStartIndex, oLength)
|
||||
Catch ex As Exception
|
||||
LogStep(HistoryItem.StepLevel.Error, "Method SUBSTRING could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
|
||||
Dim oMessage = "Method SUBSTRING could not be applied to Index '{0}' and Parameters StartIndex [{1}], Length [{2}]. Error: '{3}'"
|
||||
LogStep(HistoryItem.StepLevel.Error, oMessage, oStep.IndexName, ex.Message, oStep.Argument1, oStep.Argument2)
|
||||
Logger.Error(ex)
|
||||
End Try
|
||||
|
||||
Case ImportProfileStep.StepMethod.SPLIT
|
||||
Case Common.Constants.METHOD_SPLIT
|
||||
Try
|
||||
If String.IsNullOrEmpty(oStep.Argument1) Then
|
||||
Throw New ArgumentException("Separator")
|
||||
End If
|
||||
Dim oSeparator = oStep.Argument1.Substring(0, 1)
|
||||
Dim oIndex = Integer.Parse(oStep.Argument2)
|
||||
|
||||
Dim oIndex = 0
|
||||
If Integer.TryParse(oStep.Argument2, oIndex) = False Then
|
||||
Throw New ArgumentException("Index")
|
||||
End If
|
||||
|
||||
Dim oSplit = oValue.Split(oSeparator)
|
||||
oValue = oSplit(oIndex)
|
||||
Catch ex As Exception
|
||||
@@ -260,7 +309,7 @@ Namespace Scheduler.Jobs
|
||||
Logger.Error(ex)
|
||||
End Try
|
||||
|
||||
Case ImportProfileStep.StepMethod.REGEX
|
||||
Case Common.Constants.METHOD_REGEX
|
||||
Try
|
||||
Dim oRegex = New Regex(oStep.Argument1)
|
||||
Dim oTrueValue = oStep.Argument2
|
||||
@@ -275,10 +324,10 @@ Namespace Scheduler.Jobs
|
||||
Logger.Error(ex)
|
||||
End Try
|
||||
|
||||
Case ImportProfileStep.StepMethod.VALUE
|
||||
Case Common.Constants.METHOD_VALUE
|
||||
oValue = oStep.Argument1
|
||||
|
||||
Case ImportProfileStep.StepMethod.ALL
|
||||
Case Common.Constants.METHOD_ALL
|
||||
'noop
|
||||
Case Else
|
||||
'noop
|
||||
|
||||
Reference in New Issue
Block a user