09-12-2022

ECM.JobRunner.Windows/Constants.vb

@@ -7,6 +7,7 @@
         Public Const JOB_CONFIG_ARGUMENTS = "__Arguments"
         Public Const JOB_CONFIG_DATABASE = "__Database"
         Public Const JOB_CONFIG_STATE = "__State"
+        Public Const JOB_CONFIG_WINDREAM = "__Windream"
     End Class
 
     Public Class Jobs

ECM.JobRunner.Windows/ECM.JobRunner.Windows.vbproj

@@ -57,6 +57,10 @@
     <Reference Include="DigitalData.Modules.Database">
       <HintPath>..\..\DDModules\Database\bin\Debug\DigitalData.Modules.Database.dll</HintPath>
     </Reference>
+    <Reference Include="DigitalData.Modules.Filesystem, Version=1.4.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\DDModules\Filesystem\bin\Debug\DigitalData.Modules.Filesystem.dll</HintPath>
+    </Reference>
     <Reference Include="DigitalData.Modules.Language, Version=1.7.0.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\DDModules\Language\bin\Debug\DigitalData.Modules.Language.dll</HintPath>
@@ -123,6 +127,7 @@
     <Compile Include="Models\Jobs\HistoryItem.vb" />
     <Compile Include="Models\Jobs\StatusItem.vb" />
     <Compile Include="Models\Profiles\BaseProfile.vb" />
+    <Compile Include="Models\Profiles\ImportFile.vb" />
     <Compile Include="Models\Profiles\ImportProfile.vb" />
     <Compile Include="Models\Profiles\ImportProfileStep.vb" />
     <Compile Include="Models\Windream\ObjectType.vb" />
@@ -168,6 +173,7 @@
     <Compile Include="WCF\JobRunner.vb" />
     <Compile Include="WCF\Methods\Base.vb" />
     <Compile Include="WCF\Methods\GetJobConfig.vb" />
+    <Compile Include="WCF\Methods\RunJob.vb" />
     <Compile Include="WCF\Methods\UpdateJob.vb" />
     <Compile Include="WCF\Methods\UpdateProfile.vb" />
     <Compile Include="WCF\Methods\GetJobStatus.vb" />

ECM.JobRunner.Windows/Models/Jobs/HistoryItem.vb

@@ -4,4 +4,19 @@
     Public Property Successful As Boolean
     Public Property ErrorMessage As String
     Public Property Message As String
+    Public Property Steps As List(Of HistoryStep)
+
+    Public Class HistoryStep
+        Public Property Created As Date = Now
+        Public Property Message As String
+        Public Property Level As StepLevel
+    End Class
+
+    Public Enum StepLevel
+        Debug
+        Info
+        Warning
+        [Error]
+    End Enum
+
 End Class

ECM.JobRunner.Windows/Models/Profiles/ImportFile.vb

@@ -0,0 +1,19 @@
+Public Class ImportFile
+    Public Property FilePath As String
+    Public Property FilePathOriginal As String
+    Public Property FilePathWindream As String
+    Public Property FileInfo As IO.FileInfo
+
+    Public Property IndexValues As New List(Of IndexItem)
+
+    Public Class IndexItem
+        Public Value As String
+        Public IndexName As String
+    End Class
+
+    Public Sub New(pFilePath As String)
+        _FilePathOriginal = pFilePath
+        _FilePath = pFilePath
+        _FileInfo = New IO.FileInfo(pFilePath)
+    End Sub
+End Class

ECM.JobRunner.Windows/Scheduler/JobHistory.vb

@@ -12,23 +12,25 @@ Namespace Scheduler
             MyBase.New(pLogConfig)
         End Sub
 
-        Public Sub AddSuccess(pName As String, pMessage As String)
+        Public Sub AddSuccess(pName As String, pMessage As String, pSteps As List(Of HistoryItem.HistoryStep))
             Entries.Add(New HistoryItem With {
                 .CreatedAt = Now,
                 .JobName = pName,
                 .Successful = True,
                 .ErrorMessage = Nothing,
-                .Message = pMessage
+                .Message = pMessage,
+                .Steps = pSteps
             })
         End Sub
 
-        Public Sub AddError(pName As String, ErrorMessage As String)
+        Public Sub AddError(pName As String, ErrorMessage As String, pSteps As List(Of HistoryItem.HistoryStep))
             Entries.Add(New HistoryItem With {
                 .CreatedAt = Now,
                 .JobName = pName,
                 .Successful = False,
                 .ErrorMessage = ErrorMessage,
-                .Message = ""
+                .Message = "",
+                .Steps = pSteps
             })
         End Sub
     End Class

ECM.JobRunner.Windows/Scheduler/JobListener.vb

@@ -25,13 +25,13 @@ Namespace Scheduler
             If jobException Is Nothing Then
                 If TypeOf context.Result Is JobResult Then
                     Dim oResult As JobResult = context.Result
-                    History.AddSuccess(context.JobDetail.Key.Name, oResult.Description)
+                    History.AddSuccess(context.JobDetail.Key.Name, oResult.Description, oResult.Steps)
                 Else
 
-                    History.AddSuccess(context.JobDetail.Key.Name, "Job Successful!")
+                    History.AddSuccess(context.JobDetail.Key.Name, "Job Successful!", New List(Of HistoryItem.HistoryStep))
                 End If
             Else
-                History.AddError(context.JobDetail.Key.Name, jobException.Message)
+                History.AddError(context.JobDetail.Key.Name, jobException.Message, New List(Of HistoryItem.HistoryStep))
             End If
 
             Return MyBase.JobWasExecuted(context, jobException, cancellationToken)

ECM.JobRunner.Windows/Scheduler/JobResult.vb

@@ -1,3 +1,5 @@
 Public Class JobResult
+    Public Property Successful As Boolean
     Public Property Description As String
+    Public Property Steps As List(Of HistoryItem.HistoryStep)
 End Class

ECM.JobRunner.Windows/Scheduler/JobScheduler.vb

@@ -1,8 +1,10 @@
 Imports System.Collections.Specialized
 Imports DigitalData.Modules.Database
 Imports DigitalData.Modules.Logging
+Imports DigitalData.Modules.Windream
 Imports ECM.JobRunner.Common
 Imports Quartz
+Imports Quartz.Logging.OperationName
 
 Namespace Scheduler
     Public Class JobScheduler
@@ -16,20 +18,32 @@ Namespace Scheduler
         Private ReadOnly Database As MSSQLServer
         Private ReadOnly Factory As Quartz.Impl.StdSchedulerFactory
         Private ReadOnly State As State
+        Private ReadOnly Windream As Windream
 
         Private Scheduler As IScheduler
 
         Private Const JOB_TYPE_IMPORT As Integer = 1
         Private Const JOB_TYPE_INDEX As Integer = 2
 
-        Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pState As State)
+        Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pState As State, pWindream As Windream)
             LogConfig = pLogConfig
             Logger = pLogConfig.GetLogger()
             Factory = New Impl.StdSchedulerFactory(Settings)
             Database = pDatabase
+            Windream = pWindream
             State = pState
         End Sub
 
+        Private Function BuildJobData(pJobConfig As JobConfig) As JobDataMap
+            Return New JobDataMap From {
+                {Constants.Scheduler.JOB_CONFIG_LOGCONFIG, LogConfig},
+                {Constants.Scheduler.JOB_CONFIG_ARGUMENTS, pJobConfig.Arguments},
+                {Constants.Scheduler.JOB_CONFIG_DATABASE, Database},
+                {Constants.Scheduler.JOB_CONFIG_STATE, State},
+                {Constants.Scheduler.JOB_CONFIG_WINDREAM, Windream}
+            }
+        End Function
+
         Public Async Function Start() As Task(Of Boolean)
             Try
                 ' Log all quartz events into our standard log files
@@ -69,26 +83,39 @@ Namespace Scheduler
             Return Await Scheduler.GetCurrentlyExecutingJobs()
         End Function
 
+        Public Async Function ScheduleJob(pJobId As Integer) As Task
+            Dim oJob = State.JobDefinitions.Where(Function(j) j.Id = pJobId).SingleOrDefault()
+
+            If oJob IsNot Nothing Then
+                Logger.Info("Scheduling Job [{0}] manually!", oJob.Name)
+                Await PrepareScheduleJob(oJob)
+            End If
+        End Function
+
+        Private Async Function PrepareScheduleJob(pJob As JobDefinition) As Task
+            Logger.Debug("Loading Job Definition [{0}]", pJob.Name)
+            Logger.Debug("Job Type is [{0}]", pJob.Type.Name)
+
+            Select Case pJob.TypeId
+                Case JOB_TYPE_IMPORT
+                    Dim oJobConfig = BuildJobConfig(Of Jobs.FileImportJob)(pJob)
+                    Await ScheduleJob(Of Jobs.FileImportJob)(oJobConfig)
+
+                Case JOB_TYPE_INDEX
+                    Dim oJobConfig = BuildJobConfig(Of Jobs.FileIndexJob)(pJob)
+                    Await ScheduleJob(Of Jobs.FileIndexJob)(oJobConfig)
+
+                Case Else
+                    Logger.Warn("Job for TypeId [{0}] is not implemented!", pJob.TypeId)
+            End Select
+        End Function
+
         Private Async Sub ScheduleJobs()
 
             Logger.Info("Loading [{0}] Job Definitions..", State.JobDefinitions.Count)
 
             For Each oJob In State.JobDefinitions
-                Logger.Debug("Loading Job Definition [{0}]", oJob.Name)
-                Logger.Debug("Job Type is [{0}]", oJob.Type.Name)
-
-                Select Case oJob.TypeId
-                    Case JOB_TYPE_IMPORT
-                        Dim oJobConfig = BuildJobConfig(Of Jobs.FileImportJob)(oJob)
-                        Await ScheduleJob(Of Jobs.FileImportJob)(oJobConfig)
-
-                    Case JOB_TYPE_INDEX
-                        Dim oJobConfig = BuildJobConfig(Of Jobs.FileIndexJob)(oJob)
-                        Await ScheduleJob(Of Jobs.FileIndexJob)(oJobConfig)
-
-                    Case Else
-                        Logger.Warn("Job for TypeId [{0}] is not implemented!", oJob.TypeId)
-                End Select
+                Await PrepareScheduleJob(oJob)
             Next
         End Sub
 
@@ -155,15 +182,6 @@ Namespace Scheduler
             End If
         End Function
 
-        Private Function BuildJobData(pJobConfig As JobConfig) As JobDataMap
-            Return New JobDataMap From {
-                {Constants.Scheduler.JOB_CONFIG_LOGCONFIG, LogConfig},
-                {Constants.Scheduler.JOB_CONFIG_ARGUMENTS, pJobConfig.Arguments},
-                {Constants.Scheduler.JOB_CONFIG_DATABASE, Database},
-                {Constants.Scheduler.JOB_CONFIG_STATE, State}
-            }
-        End Function
-
         Private Function BuildJob(Of T As IJob)(pJobConfig As JobConfig, pJobData As JobDataMap) As IJobDetail
             Dim oJobName = GetJobName(pJobConfig)
             Return JobBuilder.Create(Of T)().

ECM.JobRunner.Windows/Scheduler/Jobs/BaseJob.vb

@@ -1,5 +1,6 @@
 Imports DigitalData.Modules.Database
 Imports DigitalData.Modules.Logging
+Imports DigitalData.Modules.Windream
 Imports ECM.JobRunner.Common
 Imports Quartz
 
@@ -8,11 +9,14 @@ Namespace Scheduler.Jobs
         Friend LogConfig As LogConfig
         Friend Logger As Logger
         Friend Database As MSSQLServer
+        Friend Windream As Windream
         Friend State As State
 
         Friend Id As Integer
         Friend Name As String
 
+        Friend ResultItems As New List(Of HistoryItem.HistoryStep)
+
         Private ctx As IJobExecutionContext
 
         Public Function InitializeJob(context As IJobExecutionContext) As Dictionary(Of String, String)
@@ -23,6 +27,7 @@ Namespace Scheduler.Jobs
             LogConfig = oJobData.Item(Constants.Scheduler.JOB_CONFIG_LOGCONFIG)
             Database = oJobData.Item(Constants.Scheduler.JOB_CONFIG_DATABASE)
             State = oJobData.Item(Constants.Scheduler.JOB_CONFIG_STATE)
+            Windream = oJobData.Item(Constants.Scheduler.JOB_CONFIG_WINDREAM)
             Logger = LogConfig.GetLogger()
 
             Id = Integer.Parse(oArgs.Item("Id"))
@@ -36,9 +41,45 @@ Namespace Scheduler.Jobs
             State.JobStatus.Update(ctx, pCurrentValue, pTotalValue)
         End Sub
 
-        Public Sub CompleteJob()
-            State.JobStatus.Complete(ctx)
+        Public Sub LogStep(pLevel As HistoryItem.StepLevel, pMessage As String, ParamArray pArgs As String())
+            ResultItems.Add(New HistoryItem.HistoryStep With {
+                .Message = String.Format(pMessage, pArgs),
+                .Level = pLevel
+            })
         End Sub
+
+        Public Function CompleteJob() As Task(Of Boolean)
+            ctx.Result = New JobResult With {
+                .Successful = True,
+                .Steps = ResultItems,
+                .Description = "Job completed."
+            }
+            State.JobStatus.Complete(ctx)
+
+            Return Task.FromResult(True)
+        End Function
+
+        Public Function CompleteJob(pCompletionMessage As String) As Task(Of Boolean)
+            ctx.Result = New JobResult With {
+                .Successful = True,
+                .Steps = ResultItems,
+                .Description = pCompletionMessage
+            }
+            State.JobStatus.Complete(ctx)
+
+            Return Task.FromResult(True)
+        End Function
+
+        Public Function CompleteJob(pException As Exception) As Task(Of Boolean)
+            ctx.Result = New JobResult With {
+                .Successful = False,
+                .Steps = ResultItems,
+                .Description = pException.Message
+            }
+            State.JobStatus.Complete(ctx)
+
+            Return Task.FromResult(False)
+        End Function
     End Class
 
 End Namespace

ECM.JobRunner.Windows/Scheduler/Jobs/DebugJob.vb

@@ -18,7 +18,7 @@ Namespace Scheduler.Jobs
 
             context.Result = oResult
 
-            CompleteJob()
+            CompleteJob("Done!")
             Return Task.FromResult(True)
         End Function
     End Class

ECM.JobRunner.Windows/Scheduler/Jobs/FileImportJob.vb

@@ -1,7 +1,7 @@
 Imports ECM.JobRunner.Windows.Scheduler.Jobs
 Imports Quartz
-
-
+Imports System.Text.RegularExpressions
+Imports DigitalData.Modules.Filesystem
 
 Namespace Scheduler.Jobs
 
@@ -50,47 +50,271 @@ Namespace Scheduler.Jobs
         Inherits BaseJob
         Implements IJob
 
+        Private FileEx As File
+
         Public Function Execute(context As IJobExecutionContext) As Task Implements IJob.Execute
             Dim oArgs = MyBase.InitializeJob(context)
 
             Try
-                Logger.Info("Running File Import [{0}]", Name)
+                FileEx = New File(LogConfig)
 
                 Dim oProfile = State.ProfileDefintions.ImportProfiles.Where(Function(p) p.JobId = Id).SingleOrDefault()
 
-                Dim oSourceDirectory As String = oProfile.SourceFolder
-                Dim oRecursive As Boolean = oProfile.IncludeSubfolders
-                Dim oFiles = GetFiles(oSourceDirectory, oRecursive)
+                Logger.Info("Running File Import [{0}]", Name)
+                Logger.Info("Source directory: [{0}]", oProfile.SourceFolder)
+                Logger.Info("Target directory: [{0}]", oProfile.TargetFolder)
+                Logger.Info("Backup directory: [{0}]", oProfile.BackupFolder)
 
-                If oFiles.Count = 0 Then
-                    Logger.Info("No Files for Profile [{0}]", Name)
-                    Return Task.FromResult(True)
+                Dim oObjectType = State.ObjectTypes.Where(Function(o) o.Name = oProfile.ObjectTypeName).SingleOrDefault()
+
+                If IO.Directory.Exists(oProfile.SourceFolder) = False Then
+                    LogStep(HistoryItem.StepLevel.Error, "Source directory [{0}] does not exist!", oProfile.SourceFolder)
+                    Return Task.FromResult(False)
                 End If
 
-                Logger.Info("[0] files found in source directory [{1}]", oFiles.Count, oSourceDirectory)
+                Dim oRecursive As Boolean = oProfile.IncludeSubfolders
+                Dim oFileNames = GetFiles(oProfile.SourceFolder, oRecursive)
+                Dim oRegexList As List(Of Regex) = GetRegexList(oProfile.FileExcludeRegex)
 
-                'Dim oMax = 100
-                'For index = 1 To oMax
-                '    UpdateProgress(index, oMax)
-                '    Threading.Thread.Sleep(100)
-                'Next
+                Logger.Debug("[{0}] Regexes loaded", oRegexList.Count)
 
-                Dim oResult = New JobResult() With {
-                    .Description = $"File Import Job [{Name}] completed!"
-                }
+                If oFileNames.Count = 0 Then
+                    Logger.Info("No Files for Profile [{0}]", Name)
+                    Return CompleteJob("No files for profile")
+                End If
 
-                context.Result = oResult
+                Logger.Info("[{0}] files found in source directory [{1}]", oFileNames.Count, oProfile.SourceFolder)
+                LogStep(HistoryItem.StepLevel.Info, "{0} files found in source directory {1}", oFileNames.Count, oProfile.SourceFolder)
 
-                Return Task.FromResult(True)
+                ' - [ ] Process Rules, build list of files and indexes
+                ' - [x] Process Regex to filter out files
+                ' - [x] Check time to filter out files 
+                ' - [ ] (Check if files can be accessed)
+                ' - [ ] Check if backup is needed and backup files
+                ' - [ ] Import into windream
+                '   - [ ] Create original subfolder structure
+                '   - [ ] Create DateTime Subfolders
+                '   - [x] Check if file exists and version 
+                '   - [x] Do import
+                '   - [ ] Check for filesize 0
+                '   - [ ] Write indexes (using data from getimportfile)
+                ' - [ ] Check if orig file should be deleted
+                ' - [ ] (delete subdirectories in source path)
+
+                Dim oFiles = oFileNames.
+                    Select(Function(f) New ImportFile(f)).
+                    ToList()
+                Dim oFilteredFiles = oFiles
+
+                ' Check time to filter out files 
+                Dim oDateFilteredFiles = oFilteredFiles.Where(Function(f) FileIsOlderThan(f, pMinutes:=1)).ToList()
+                Dim oDateFilteredCount = oFilteredFiles.Except(oDateFilteredFiles).Count()
+                Logger.Debug("[{0}] Files filtered out for being too new.", oDateFilteredCount)
+                LogStep(HistoryItem.StepLevel.Debug, "{0} Files filtered out for being too new.", oDateFilteredCount)
+                oFilteredFiles = oDateFilteredFiles
+
+                ' Process Regex to filter out files
+                Dim oRegexFilteredFiles = oFilteredFiles.Where(Function(f) Not FileMatchesRegex(f, oRegexList))
+                Dim oRegexFilteredCount = oFilteredFiles.Except(oRegexFilteredFiles).Count()
+                Logger.Debug("[{0}] Files filtered out for matching exclusion Regex.", oRegexFilteredCount)
+                LogStep(HistoryItem.StepLevel.Debug, "{0} Files filtered out for matching exclusion Regex.", oRegexFilteredCount)
+                oFilteredFiles = oDateFilteredFiles
+
+                Logger.Info("Importing files..")
+                Dim oImportedFiles As New List(Of ImportFile)
+                For Each oFile In oFilteredFiles
+
+                    Dim oImportedPath = ImportFile(oFile, oProfile)
+                    If oImportedPath Is Nothing Then
+                        Logger.Warn("File [{0}] could not be imported!", oFile.FilePath)
+                        Continue For
+                    End If
+                    oFile.FilePathWindream = oImportedPath
+
+                    oImportedFiles.Add(oFile)
+                Next
+
+                Logger.Info("[{0}] files successfully Imported!", oImportedFiles.Count)
+                LogStep(HistoryItem.StepLevel.Info, "{0} files successfully Imported!", oImportedFiles.Count)
+
+                Dim oIndexedFiles As New List(Of ImportFile)
+                For Each oFile In oImportedFiles
+                    Logger.Info("Indexing file [{0}]", oFile.FilePathWindream)
+                    Dim oIndexResult = IndexFile(oFile, oProfile)
+                    If oIndexResult = True Then
+                        oIndexedFiles.Add(oFile)
+                    End If
+                    Logger.Info("Indexing of file [{0}] done!", oFile.FilePathWindream)
+                Next
+
+                Logger.Info("[{0}] files successfully Indexed!", oIndexedFiles.Count)
+                LogStep(HistoryItem.StepLevel.Info, "{0} files successfully Indexed!", oIndexedFiles.Count)
+
+                Return CompleteJob($"{oImportedFiles.Count} files successfully Processed!")
             Catch ex As Exception
                 Logger.Error(ex)
 
+                LogStep(HistoryItem.StepLevel.Error, "Unexpected Error: [{0}]", ex.Message)
 
-                Return Task.FromResult(False)
-            Finally
-                CompleteJob()
+                Return CompleteJob(ex)
             End Try
+        End Function
 
+        Private Function FileIsOlderThan(pFile As ImportFile, pMinutes As Integer)
+            Return pFile.FileInfo.CreationTime.AddMinutes(pMinutes) < Now
+        End Function
+
+        Private Function FileMatchesRegex(pFile As ImportFile, pRegexList As List(Of Regex))
+            Return pRegexList.Any(Function(r) r.IsMatch(pFile.FilePath))
+        End Function
+
+        Private Function GetRegexList(pRegexString As String) As List(Of Regex)
+            Return pRegexString.
+                Split(vbNewLine).
+                ToList().
+                Where(Function(r) String.IsNullOrWhiteSpace(r) = False).
+                Select(Function(s)
+                           Logger.Debug("Regex loaded: [{0}]", s)
+                           Return New Regex(s)
+                       End Function).ToList()
+
+        End Function
+
+        Private Function ImportFile(pFile As ImportFile, pProfile As ImportProfile) As String
+            'Check if target folder exists
+            If Windream.TestFolderExists(pProfile.TargetFolder) = False Then
+                If Windream.NewFolder(pProfile.TargetFolder) = False Then
+                    Logger.Warn("Folder [{0}] could not be created!", pProfile.TargetFolder)
+                    Return Nothing
+                End If
+            End If
+
+            ' Generate new filepath and stream file
+            Dim oFileName = IO.Path.GetFileName(pFile.FilePath)
+            Dim oNewFilePath As String = IO.Path.Combine(pProfile.TargetFolder, oFileName)
+            Dim oVersionedFilePath = GetVersionedWindreamPath(oNewFilePath)
+
+            If Windream.NewFileStream(pFile.FilePathOriginal, oVersionedFilePath, pProfile.ObjectTypeName) = False Then
+                Logger.Warn("File [{0}] could not be streamed to path [{1}]!", pFile.FilePathOriginal, oNewFilePath)
+                Return Nothing
+            End If
+
+            Return oVersionedFilePath
+        End Function
+
+        Private Function GetVersionedWindreamPath(pWindreamFilePath As String) As String
+            Dim oAbsolutePath = Windream.GetAbsolutePath(pWindreamFilePath)
+
+            ' This versions the filename but does not rely on access though the filesystem.
+            ' Instead the check for file existence is made through the windream sdk.
+            Dim oVersionedPath = FileEx.GetVersionedFilenameWithFilecheck(oAbsolutePath, Function(pPath As String) Windream.TestFileExists(pPath))
+            Dim oVersionedAndNormalizedPath = Windream.GetNormalizedPath(oVersionedPath, False)
+
+            Return oVersionedAndNormalizedPath
+        End Function
+
+        Private Function GetImportFile(pFilename As String, pProfile As ImportProfile) As ImportFile
+            Dim oImportFile = New ImportFile(pFilename)
+
+            Return oImportFile
+        End Function
+
+        Private Function ProcessSteps(pFile As ImportFile, pProfile As ImportProfile) As List(Of ImportFile.IndexItem)
+            Dim oIndexItems = New List(Of ImportFile.IndexItem)
+
+            ' Process Steps on Filename
+            For Each oStep In pProfile.Steps
+                Dim oValue As String
+
+                ' Get the base value by checking scope
+                Select Case oStep.Scope
+                    Case ImportProfileStep.StepScope.FILE
+                        oValue = pFile.FileInfo.Name
+                    Case ImportProfileStep.StepScope.FOLDER
+                        oValue = pFile.FileInfo.DirectoryName
+                    Case Else
+                        oValue = pFile.FilePath
+                End Select
+
+
+                ' TODO: Error handling!
+                Select Case oStep.Method
+                    Case ImportProfileStep.StepMethod.SUBSTRING
+                        Try
+                            Dim oIndex1 = Integer.Parse(oStep.Argument1)
+                            Dim oLength = Integer.Parse(oStep.Argument2)
+                            oValue = oValue.Substring(oIndex1, oLength)
+                        Catch ex As Exception
+                            LogStep(HistoryItem.StepLevel.Error, "Method SUBSTRING could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
+                            Logger.Error(ex)
+                        End Try
+
+                    Case ImportProfileStep.StepMethod.SPLIT
+                        Try
+                            Dim oSeparator = oStep.Argument1.Substring(0, 1)
+                            Dim oIndex = Integer.Parse(oStep.Argument2)
+                            Dim oSplit = oValue.Split(oSeparator)
+                            oValue = oSplit(oIndex)
+                        Catch ex As Exception
+                            LogStep(HistoryItem.StepLevel.Error, "Method SPLIT could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
+                            Logger.Error(ex)
+                        End Try
+
+                    Case ImportProfileStep.StepMethod.REGEX
+                        Try
+                            Dim oRegex = New Regex(oStep.Argument1)
+                            Dim oTrueValue = oStep.Argument2
+                            Dim oFalseValue = oStep.Argument3
+                            If oRegex.IsMatch(oValue) Then
+                                oValue = oTrueValue
+                            Else
+                                oValue = oFalseValue
+                            End If
+                        Catch ex As Exception
+                            LogStep(HistoryItem.StepLevel.Error, "Method REGEX could not be applied to Index '{0}'. Error: '{1}'", oStep.IndexName, ex.Message)
+                            Logger.Error(ex)
+                        End Try
+
+                    Case ImportProfileStep.StepMethod.VALUE
+                        oValue = oStep.Argument1
+
+                    Case ImportProfileStep.StepMethod.ALL
+                        'noop
+                    Case Else
+                        'noop
+                End Select
+
+                oIndexItems.Add(New ImportFile.IndexItem With {
+                    .IndexName = oStep.IndexName,
+                    .Value = oValue
+                })
+            Next
+
+            Return oIndexItems
+        End Function
+
+        Private Function IndexFile(pFile As ImportFile, pProfile As ImportProfile) As Boolean
+            Logger.Debug("Writing [{0}] indexes for File [{1}]", pFile.IndexValues.Count, pFile.FilePathWindream)
+
+            Dim oIndexItems = ProcessSteps(pFile, pProfile)
+
+            Dim oResults = oIndexItems.
+                Select(Function(v)
+                           Dim oIndexResult = False
+                           Logger.Info("Writing Index [{0}] with value [{1}]", v.IndexName, v.Value)
+
+                           If Windream.TestIndexNameIsVectorIndex(v.IndexName) Then
+                               oIndexResult = Windream.SetFileIndex(pFile.FilePathWindream, v.IndexName, New List(Of String) From {v.Value}, pProfile.ObjectTypeName)
+                           Else
+                               oIndexResult = Windream.SetFileIndex(pFile.FilePathWindream, v.IndexName, v.Value, pProfile.ObjectTypeName)
+                           End If
+
+                           Return oIndexResult
+                       End Function).
+                ToList()
+
+            ' Return True if all Indexes were set correctly
+            Return oResults.All(Function(r) r = True)
         End Function
 
         Private Function GetFiles(pDirectory As String, pRecursive As Boolean) As String()

ECM.JobRunner.Windows/Scheduler/Jobs/FileIndexJob.vb

@@ -17,7 +17,7 @@ Namespace Scheduler.Jobs
 
             context.Result = oResult
 
-            CompleteJob()
+            CompleteJob("Done!")
             Return Task.FromResult(True)
         End Function
     End Class

ECM.JobRunner.Windows/Service.vb

@@ -41,14 +41,14 @@ Public Class Service
 
             ' Initialize Windream
             Dim oWindream = Config.Windream
-            Windream = New Windream(LogConfig, True, oWindream.DriveLetter, "/", True,
+            Windream = New Windream(LogConfig, True, oWindream.DriveLetter, "\\windream\objects\", True,
                                     oWindream.Server, oWindream.Username, oWindream.Password, oWindream.Domain)
 
             ' initialize global state
             State = New State(LogConfig, Database, Windream)
 
             ' start the scheduler 
-            Scheduler = New Scheduler.JobScheduler(LogConfig, Database, State)
+            Scheduler = New Scheduler.JobScheduler(LogConfig, Database, State, Windream)
             If Await Scheduler.Start() Then
                 Logger.Info("Scheduler started!")
             Else

ECM.JobRunner.Windows/WCF/IJobRunner.vb

@@ -4,6 +4,7 @@ Imports ECM.JobRunner.Windows.UpdateJob
 Imports ECM.JobRunner.Windows.UpdateProfile
 Imports ECM.JobRunner.Windows.GetJobStatus
 Imports ECM.JobRunner.Windows.GetJobConfig
+Imports ECM.JobRunner.Windows.RunJob
 
 Namespace WCF
     <ServiceContract(Name:="IEDMIService", [Namespace]:="http://DigitalData.Services.EDMIService")>
@@ -15,6 +16,9 @@ Namespace WCF
         <OperationContract>
         Function GetJobStatus() As GetJobStatusResponse
 
+        'TODO: implement
+        Function RunJob() As RunJobResponse
+
         <OperationContract>
         Function UpdateJob(pData As UpdateJobRequest) As UpdateJobResponse
 

ECM.JobRunner.Windows/WCF/JobRunner.vb

@@ -4,6 +4,7 @@ Imports DigitalData.Modules.Database
 Imports DigitalData.Modules.Logging
 Imports DigitalData.Modules.Messaging.WCF
 Imports ECM.JobRunner.Windows.Scheduler
+Imports FxResources.System
 
 Namespace WCF
 
@@ -55,6 +56,11 @@ Namespace WCF
             Dim oMethod As New UpdateProfile.UpdateProfileMethod(LogConfig, Database, State, Scheduler)
             Return oMethod.Run(pData)
         End Function
+
+        Public Function RunJob() As RunJob.RunJobResponse Implements IJobRunner.RunJob
+            'Dim oMethod As New RunJob.RunJobMethod(LogConfig, Database, State, Scheduler)
+            'Return oMethod.Run(pData)
+        End Function
     End Class
 
 End Namespace

ECM.JobRunner.Windows/WCF/Methods/RunJob.vb

@@ -0,0 +1,31 @@
+Imports DigitalData.Modules.Database
+Imports DigitalData.Modules.Logging
+Imports ECM.JobRunner.Windows.Scheduler
+Imports System.Runtime.Serialization
+
+Public Class RunJob
+    Public Class RunJobMethod
+        Inherits Base.BaseMethod
+
+        Private ReadOnly Scheduler As JobScheduler
+
+        Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer, pState As State, pScheduler As JobScheduler)
+            MyBase.New(pLogConfig, pDatabase, pState)
+            Scheduler = pScheduler
+        End Sub
+
+        Public Async Function Run(pData As RunJobRequest) As Task(Of RunJobResponse)
+            Await Scheduler.ScheduleJob(pData.JobId)
+            Return New RunJobResponse()
+        End Function
+
+    End Class
+
+    Public Class RunJobRequest
+        Public Property JobId As Integer
+    End Class
+
+    Public Class RunJobResponse
+        Inherits Base.BaseResponse
+    End Class
+End Class