Imports ECM.JobRunner.Windows.Scheduler.Jobs
Imports Quartz
Imports System.Text.RegularExpressions
Imports DigitalData.Modules.Filesystem
Imports ECM.JobRunner.Common

Namespace Scheduler.Jobs

    Public Class FileImportJob
        Inherits BaseJob
        Implements IJob

        Private FileEx As File

        Public Function Execute(context As IJobExecutionContext) As Task Implements IJob.Execute
            Dim oArgs = MyBase.InitializeJob(context)

            Try
                FileEx = New File(LogConfig)

                Dim oProfile = State.ProfileDefintions.ImportProfiles.Where(Function(p) p.JobId = Id).SingleOrDefault()

                Logger.Info("Running File Import [{0}]", Name)
                Logger.Info("Source directory: [{0}]", oProfile.SourceFolder)
                Logger.Info("Target directory: [{0}]", oProfile.TargetFolder)
                Logger.Info("Backup directory: [{0}]", oProfile.BackupFolder)

                Dim oObjectType = State.ObjectTypes.Where(Function(o) o.Name = oProfile.ObjectTypeName).SingleOrDefault()

                If IO.Directory.Exists(oProfile.SourceFolder) = False Then
                    LogError("Source directory [{0}] does not exist!", oProfile.SourceFolder)
                    Return Task.FromResult(False)
                End If

                Dim oRecursive As Boolean = oProfile.IncludeSubfolders
                Dim oFileNames = GetFiles(oProfile.SourceFolder, oRecursive)
                Dim oRegexList As List(Of Regex) = GetRegexList(oProfile.FileExcludeRegex)

                Logger.Debug("[{0}] Regexes loaded", oRegexList.Count)

                If oFileNames.Count = 0 Then
                    Logger.Info("No Files for Profile [{0}]", Name)
                    Return CompleteJob("No files for profile")
                End If

                LogInfo("{0} files found in source directory {1}", oFileNames.Count, oProfile.SourceFolder)

                ' - [ ] Process Rules, build list of files and indexes
                ' - [x] Process Regex to filter out files
                ' - [x] Check time to filter out files 
                ' - [ ] (Check if files can be accessed)
                ' - [ ] Check if backup is needed and backup files
                ' - [ ] Import into windream
                '   - [ ] Create original subfolder structure
                '   - [ ] Create DateTime Subfolders
                '   - [x] Check if file exists and version 
                '   - [x] Do import
                '   - [ ] Check for filesize 0
                '   - [ ] Write indexes (using data from getimportfile)
                ' - [ ] Check if orig file should be deleted
                ' - [ ] (delete subdirectories in source path)

                Dim oFiles = oFileNames.
                    Select(Function(f) New ImportFile(f)).
                    ToList()
                Dim oFilteredFiles = oFiles

                ' Check time to filter out files 
                Dim oDateFilteredFiles = oFilteredFiles.Where(Function(f) FileIsOlderThan(f, pMinutes:=1)).ToList()
                Dim oDateFilteredCount = oFilteredFiles.Except(oDateFilteredFiles).Count()
                LogDebug("{0} Files filtered out for being too new.", oDateFilteredCount)
                oFilteredFiles = oDateFilteredFiles

                ' Process Regex to filter out files
                Dim oRegexFilteredFiles = oFilteredFiles.Where(Function(f) Not FileMatchesRegex(f, oRegexList))
                Dim oRegexFilteredCount = oFilteredFiles.Except(oRegexFilteredFiles).Count()
                LogDebug("{0} Files filtered out for matching exclusion Regex.", oRegexFilteredCount)
                oFilteredFiles = oDateFilteredFiles

                '-------------------------------------------------------------------------------------------------
                ' After this point, files are treated as processed and are being deleted before finishing the job
                '-------------------------------------------------------------------------------------------------

                If oFilteredFiles.Count = 0 Then
                    Return CompleteJob($"No files to process.")
                End If

                Logger.Info("Importing files..")
                Dim oImportedFiles As New List(Of ImportFile)
                For Each oFile In oFilteredFiles

                    Dim oImportedPath = ImportFile(oFile, oProfile)
                    If oImportedPath Is Nothing Then
                        Logger.Warn("File [{0}] could not be imported!", oFile.FilePath)
                        Continue For
                    End If
                    oFile.FilePathWindream = oImportedPath

                    oImportedFiles.Add(oFile)
                Next

                LogInfo("{0} files successfully Imported!", oImportedFiles.Count)

                Dim oIndexedFiles As New List(Of ImportFile)
                For Each oFile In oImportedFiles
                    Logger.Info("Indexing file [{0}]", oFile.FilePathWindream)
                    Dim oIndexResult = IndexFile(oFile, oProfile)
                    If oIndexResult = True Then
                        oIndexedFiles.Add(oFile)
                    End If
                    Logger.Info("Indexing of file [{0}] done!", oFile.FilePathWindream)
                Next

                LogInfo("{0} files successfully Indexed!", oIndexedFiles.Count)

                If oProfile.DeleteFiles = True Then
                    Logger.Debug("Deleting [{0}] files before finishing job.", oFiles.Count)

                    For Each oFile In oFilteredFiles
                        Try
                            IO.File.Delete(oFile.FilePathOriginal)
                        Catch ex As Exception
                            Logger.Warn("Could not clean up processed files before finishing Job!")
                            Logger.Error(ex)
                        End Try
                    Next
                End If

                Return CompleteJob($"{oImportedFiles.Count} files successfully Processed!")
            Catch ex As Exception
                Logger.Error(ex)
                LogError("Unexpected Error: [{0}]", ex.Message)

                Return CompleteJobWithError(ex)
            End Try
        End Function

        Private Function FileIsOlderThan(pFile As ImportFile, pMinutes As Integer)
            Dim oCreationTime = pFile.FileInfo.CreationTime
            Dim oTimeSpan = New TimeSpan(0, pMinutes, 0)

            If (Now - oCreationTime) > oTimeSpan Then
                Return True
            Else
                Return False
            End If
        End Function

        Private Function FileMatchesRegex(pFile As ImportFile, pRegexList As List(Of Regex))
            Return pRegexList.Any(Function(r) r.IsMatch(pFile.FilePath))
        End Function

        Private Function GetRegexList(pRegexString As String) As List(Of Regex)
            Return pRegexString.
                Split(vbNewLine).
                ToList().
                Where(Function(r) String.IsNullOrWhiteSpace(r) = False).
                Select(Function(s)
                           Logger.Debug("Regex loaded: [{0}]", s)
                           Return New Regex(s)
                       End Function).ToList()

        End Function

        Private Function ImportFile(pFile As ImportFile, pProfile As ImportProfile) As String
            'Check if target folder exists
            If Windream.TestFolderExists(pProfile.TargetFolder) = False Then
                If Windream.NewFolder(pProfile.TargetFolder) = False Then
                    Logger.Warn("Folder [{0}] could not be created!", pProfile.TargetFolder)
                    Return Nothing
                End If
            End If

            ' Generate new filepath and stream file
            Dim oFileName = IO.Path.GetFileName(pFile.FilePath)
            Dim oNewFilePath As String = IO.Path.Combine(pProfile.TargetFolder, oFileName)
            Dim oVersionedFilePath = GetVersionedWindreamPath(oNewFilePath)

            If Windream.NewFileStream(pFile.FilePathOriginal, oVersionedFilePath, pProfile.ObjectTypeName) = False Then
                Logger.Warn("File [{0}] could not be streamed to path [{1}]!", pFile.FilePathOriginal, oNewFilePath)
                Return Nothing
            End If

            Return oVersionedFilePath
        End Function

        Private Function GetVersionedWindreamPath(pWindreamFilePath As String) As String
            Dim oAbsolutePath = Windream.GetAbsolutePath(pWindreamFilePath)

            ' This versions the filename but does not rely on access though the filesystem.
            ' Instead the check for file existence is made through the windream sdk.
            Dim oVersionedPath = FileEx.GetVersionedFilenameWithFilecheck(oAbsolutePath, Function(pPath As String) Windream.TestFileExists(pPath))
            Dim oVersionedAndNormalizedPath = Windream.GetNormalizedPath(oVersionedPath, False)

            Return oVersionedAndNormalizedPath
        End Function

        Private Function GetImportFile(pFilename As String, pProfile As ImportProfile) As ImportFile
            Dim oImportFile = New ImportFile(pFilename)

            Return oImportFile
        End Function

        Private Function ProcessSteps(pFile As ImportFile, pProfile As ImportProfile) As List(Of ImportFile.IndexItem)
            Dim oIndexItems = New List(Of ImportFile.IndexItem)

            ' Process Steps on Filename
            For Each oStep In pProfile.Steps
                Dim oValue As String

                ' Get the base value by checking scope
                Select Case oStep.Scope
                    Case Common.Constants.SCOPE_FILE
                        oValue = pFile.FileInfo.Name
                    Case Common.Constants.SCOPE_FOLDER
                        oValue = pFile.FileInfo.DirectoryName
                    Case Else
                        oValue = pFile.FilePath
                End Select

                Logger.Info("Running Method [{0}] on Index [{1}]", oStep.Method, oStep.IndexName)
                Logger.Debug("Value is [{0}]", oValue)
                Logger.Debug("Scope is [{0}]", oStep.Scope)

                Select Case oStep.Method
                    Case Common.Constants.METHOD_SUBSTRING
                        Try
                            Logger.Debug("Parsing Value [{0}] for Parameter 'StartIndex'", oStep.Argument1)
                            Dim oStartIndex = 0
                            If Integer.TryParse(oStep.Argument1, oStartIndex) = False Then
                                Throw New ArgumentException("StartIndex")
                            End If

                            Logger.Debug("Parsing Value [{0}] for Parameter 'Length'", oStep.Argument2)
                            Dim oLength = 0
                            If Integer.TryParse(oStep.Argument2, oLength) = False Then
                                Throw New ArgumentException("Length")
                            End If

                            oValue = oValue.Substring(oStartIndex, oLength)
                        Catch ex As Exception
                            Dim oMessage = "Method SUBSTRING could not be applied to Index '{0}' and Parameters StartIndex [{1}], Length [{2}]. Error: '{3}'"
                            LogError(oMessage, oStep.IndexName, ex.Message, oStep.Argument1, oStep.Argument2)
                        End Try

                    Case Common.Constants.METHOD_SPLIT
                        Try
                            If String.IsNullOrEmpty(oStep.Argument1) Then
                                Throw New ArgumentException("Separator")
                            End If
                            Dim oSeparator = oStep.Argument1.Substring(0, 1)

                            Dim oIndex = 0
                            If Integer.TryParse(oStep.Argument2, oIndex) = False Then
                                Throw New ArgumentException("Index")
                            End If

                            Dim oSplit = oValue.Split(oSeparator)
                            oValue = oSplit(oIndex)
                        Catch ex As Exception
                            LogError("Method SPLIT could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
                        End Try

                    Case Common.Constants.METHOD_REGEX
                        Try
                            Dim oRegex = New Regex(oStep.Argument1)
                            Dim oTrueValue = oStep.Argument2
                            Dim oFalseValue = oStep.Argument3
                            If oRegex.IsMatch(oValue) Then
                                oValue = oTrueValue
                            Else
                                oValue = oFalseValue
                            End If
                        Catch ex As Exception
                            LogError("Method REGEX could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
                        End Try

                    Case Common.Constants.METHOD_VALUE
                        oValue = oStep.Argument1

                    Case Common.Constants.METHOD_ALL
                        'noop
                    Case Else
                        'noop
                End Select

                oIndexItems.Add(New ImportFile.IndexItem With {
                    .IndexName = oStep.IndexName,
                    .Value = oValue
                })
            Next

            Return oIndexItems
        End Function

        Private Function IndexFile(pFile As ImportFile, pProfile As ImportProfile) As Boolean
            Logger.Debug("Writing [{0}] indexes for File [{1}]", pFile.IndexValues.Count, pFile.FilePathWindream)

            Dim oIndexItems = ProcessSteps(pFile, pProfile)

            Dim oResults = oIndexItems.
                Select(Function(v)
                           Dim oIndexResult = False
                           Logger.Info("Writing Index [{0}] with value [{1}]", v.IndexName, v.Value)

                           If Windream.TestIndexNameIsVectorIndex(v.IndexName) Then
                               oIndexResult = Windream.SetFileIndex(pFile.FilePathWindream, v.IndexName, New List(Of String) From {v.Value}, pProfile.ObjectTypeName)
                           Else
                               oIndexResult = Windream.SetFileIndex(pFile.FilePathWindream, v.IndexName, v.Value, pProfile.ObjectTypeName)
                           End If

                           Return oIndexResult
                       End Function).
                ToList()

            ' Return True if all Indexes were set correctly
            Return oResults.All(Function(r) r = True)
        End Function

        Private Function GetFiles(pDirectory As String, pRecursive As Boolean) As String()
            Dim oFiles As String()

            If pRecursive Then
                oFiles = IO.Directory.GetFiles(pDirectory, "*.*", IO.SearchOption.AllDirectories)
                Logger.Info("Found [{0}] files in Folder [{1}] (and subdirectories)", oFiles.Count, pDirectory)
            Else
                oFiles = IO.Directory.GetFiles(pDirectory, "*.*", IO.SearchOption.TopDirectoryOnly)
                Logger.Info("Found [{0}] files in Folder [{1}]", oFiles.Count, pDirectory)
            End If

            Return oFiles
        End Function
    End Class

End Namespace