From f39b79e13ccf3f85eb6d654ef1e4b5e9b6e6d016 Mon Sep 17 00:00:00 2001 From: Jonathan Jenne Date: Mon, 3 May 2021 10:30:38 +0200 Subject: [PATCH] Check dropped files by hash instead of filename #GI-12 --- Global_Indexer/ClassFilehandle.vb | 17 +++- Global_Indexer/ClassIndexFunctions.vb | 37 ++++++-- Global_Indexer/ClassInit.vb | 3 + Global_Indexer/ModuleCURRENT.vb | 3 + Global_Indexer/frmIndex.vb | 128 ++++++++++++++------------ Global_Indexer/frmStart.vb | 40 +++++--- 6 files changed, 145 insertions(+), 83 deletions(-) diff --git a/Global_Indexer/ClassFilehandle.vb b/Global_Indexer/ClassFilehandle.vb index 84c657b..ad2b896 100644 --- a/Global_Indexer/ClassFilehandle.vb +++ b/Global_Indexer/ClassFilehandle.vb @@ -144,8 +144,21 @@ Public Class ClassFilehandle Try filename = filename.Replace("'", "''") + Dim oHash As String + + If File.Exists(filename) Then + Try + oHash = FILESYSTEM.GetChecksum(filename) + Catch ex As Exception + LOGGER.Error(ex) + oHash = String.Empty + End Try + Else + oHash = String.Empty + End If + Dim filename_only As String = Path.GetFileName(filename) - Dim ins As String = "INSERT INTO TBGI_FILES_USER (FILENAME2WORK, USER@WORK,HANDLE_TYPE,FILENAME_ONLY) VALUES ('" & filename & "','" & Environment.UserName & "','" & handleType & "','" & filename_only & "')" + Dim ins As String = $"INSERT INTO TBGI_FILES_USER (FILENAME2WORK, USER@WORK, HANDLE_TYPE, FILENAME_ONLY, FILE_HASH) VALUES ('{filename}','{Environment.UserName}','{handleType}','{filename_only}', '{oHash}')" Return ClassDatabase.Execute_non_Query(ins, True) Catch ex As Exception @@ -157,7 +170,7 @@ Public Class ClassFilehandle ' Prüft, ob die angegeben Datei aktuell durch eine ' andere Anwendung in Benutzung ist Dim ff As Integer = FreeFile() - If System.IO.File.Exists(fullFilePath) Then + If File.Exists(fullFilePath) Then Try ' Versuchen, die Datei mit *exklusiven* Lese- und ' Schreibrechten zu öffnen diff --git a/Global_Indexer/ClassIndexFunctions.vb b/Global_Indexer/ClassIndexFunctions.vb index 8fe3f4a..56d07d8 100644 --- a/Global_Indexer/ClassIndexFunctions.vb +++ b/Global_Indexer/ClassIndexFunctions.vb @@ -1,19 +1,38 @@ - - -Public Class ClassIndexFunctions +Public Class ClassIndexFunctions Public Shared Function FileExistsinDropTable(Filename As String) - Dim check As String + Dim oSQL As String + Dim oHash As String + Try If Filename.Contains("'") Then Filename = Filename.Replace("'", "''") End If - check = "SELECT COUNT(*) FROM TBGI_FILES_USER WHERE UPPER(FILENAME2WORK) = UPPER('" & Filename & "') AND WORKED = 0" - Dim result = ClassDatabase.Execute_Scalar(check, MyConnectionString, True) - Return result + + Try + oHash = FILESYSTEM.GetChecksum(Filename) + Catch ex As Exception + LOGGER.Error(ex) + oHash = "" + End Try + + oSQL = "SELECT COUNT(*) FROM TBGI_FILES_USER WHERE UPPER(FILE_HASH) = UPPER('" & oHash & "') AND WORKED = 0" + Dim oResult = ClassDatabase.Execute_Scalar(oSQL, MyConnectionString, True) + + If oResult = 0 Then + oSQL = "SELECT COUNT(*) FROM TBGI_HISTORY WHERE UPPER(FILE_HASH) = UPPER('" & oHash & "')" + oResult = ClassDatabase.Execute_Scalar(oSQL, MyConnectionString, True) + + If oResult = 0 Then + Return False + Else + Return True + End If + Else + Return True + End If Catch ex As Exception - MsgBox("Error in FileExistsinDropTable - Error-Message:" & vbNewLine & ex.Message & vbNewLine & "SQL-Command:" & vbNewLine & check, MsgBoxStyle.Critical) + MsgBox("Error in FileExistsinDropTable - Error-Message:" & vbNewLine & ex.Message & vbNewLine & "SQL-Command:" & vbNewLine & oSQL, MsgBoxStyle.Critical) Return Nothing End Try - End Function End Class diff --git a/Global_Indexer/ClassInit.vb b/Global_Indexer/ClassInit.vb index b3fd73b..b6e4952 100644 --- a/Global_Indexer/ClassInit.vb +++ b/Global_Indexer/ClassInit.vb @@ -3,6 +3,7 @@ Imports DLLLicenseManager Imports DigitalData.Modules.Logging Imports DigitalData.Modules.Config Imports DigitalData.Modules.Windream +Imports DigitalData.Modules.Filesystem Public Class ClassInit Public _lizenzManager As ClassLicenseManager @@ -65,6 +66,8 @@ Public Class ClassInit Dim configResult As Boolean configResult = Load_BasicConfig() + FILESYSTEM = New File(LOGCONFIG) + If configResult = False Then If USER_LANGUAGE = "de-DE" Then Throw New Exception("Unerwarteter Fehler beim Initialisieren der Basis-Einstellungen. Weitere Informationen finden Sie in der Logdatei.") diff --git a/Global_Indexer/ModuleCURRENT.vb b/Global_Indexer/ModuleCURRENT.vb index fe06e88..ccbf736 100644 --- a/Global_Indexer/ModuleCURRENT.vb +++ b/Global_Indexer/ModuleCURRENT.vb @@ -1,5 +1,6 @@ Imports DigitalData.Modules.Config Imports DigitalData.Modules.Logging +Imports DigitalData.Modules.Filesystem Module ModuleCURRENT Public ERROR_STATE As String @@ -7,12 +8,14 @@ Module ModuleCURRENT Public CURRENT_FILENAME As String Public CURRENT_NEWFILENAME As String Public CURRENT_WORKFILE_GUID As Integer + Public CURRENT_WORKFILE_HASH As String Public CURRENT_WORKFILE As String Public CURR_WORKFILE_EXTENSION As String Public CONFIG As ConfigManager(Of ClassConfig) Public LOGCONFIG As LogConfig Public LOGGER As Logger + Public FILESYSTEM As File Public CURRENT_DOKART_ID As Integer Public CURRENT_DOKART_DUPLICATE_HANDLING As String = "Default" diff --git a/Global_Indexer/frmIndex.vb b/Global_Indexer/frmIndex.vb index 26dece8..18a6a47 100644 --- a/Global_Indexer/frmIndex.vb +++ b/Global_Indexer/frmIndex.vb @@ -550,7 +550,7 @@ Public Class frmIndex Dim oMatchelements As System.Text.RegularExpressions.MatchCollection = regulärerAusdruck.Matches(oNamenkonvention) '#### If oMatchelements.Count = 0 Then - _Logger.Debug("No RegularExpression Fileds on Nameconvention!") + _Logger.Debug("No RegularExpression Files on Nameconvention!") End If ' alle Vorkommen innerhalbd er Namenkonvention durchlaufen For Each oElement As System.Text.RegularExpressions.Match In oMatchelements @@ -1301,7 +1301,9 @@ Public Class frmIndex Try Dim tempCur_WF = CURRENT_WORKFILE.Replace("'", "''") Dim tempCur_New_FN = CURRENT_NEWFILENAME.Replace("'", "''") - Insert_String = sql_history_INSERT_INTO & ",ADDED_WHO,ADDED_WHERE) VALUES ('" & tempCur_WF & "','" & tempCur_New_FN & "'" & sql_history_Index_Values & ",'" & Environment.UserDomainName & "\" & Environment.UserName & "','" & Environment.MachineName & "')" + Dim oUser As String = $"{Environment.UserDomainName}\{Environment.UserName}" + + Insert_String = sql_history_INSERT_INTO & $",ADDED_WHO, ADDED_WHERE, FILE_HASH) VALUES ('{tempCur_WF}','{tempCur_New_FN}'{sql_history_Index_Values},'{oUser}','{Environment.MachineName}','{CURRENT_WORKFILE_HASH}')" If ClassDatabase.Execute_non_Query(Insert_String) = True Then If CURRENT_MESSAGEID <> "" Then Dim max As String = "SELECT MAX(GUID) FROM TBGI_HISTORY" @@ -2230,66 +2232,67 @@ Public Class frmIndex End Try End Function - Private Sub OK_Button_Click(sender As Object, e As EventArgs) - ClearError() - ClearNotice() - - 'lblhinweis.Visible = False - 'lblerror.Visible = False - - Me.Cursor = Cursors.WaitCursor - ClassHelper.Refresh_RegexTable() - For Each rowregex As DataRow In CURRENT_DT_REGEX.Rows - If rowregex.Item("FUNCTION_NAME") = "CLEAN_FILENAME" Then - REGEX_CLEAN_FILENAME = rowregex.Item("REGEX") - End If - Next - If checkMultiindex.Visible = True And checkMultiindex.Checked = True Then - 'Die erste Datei indexieren - If WORK_FILE() = True Then - 'Und nun die folgenden - Dim DTFiles2Work As DataTable = ClassDatabase.Return_Datatable("SELECT * FROM TBGI_FILES_USER WHERE WORKED = 0 AND GUID <> " & CURRENT_WORKFILE_GUID & " AND UPPER(USER@WORK) = UPPER('" & Environment.UserName & "')") - If Not DTFiles2Work Is Nothing Then - Dim err = False - For Each filerow As DataRow In DTFiles2Work.Rows - CURRENT_WORKFILE_GUID = filerow.Item("GUID") - CURRENT_WORKFILE = filerow.Item("FILENAME2WORK") - DropType = filerow.Item("HANDLE_TYPE") - - If WORK_FILE() = False Then - err = True - Exit For - End If - Next - Me.Cursor = Cursors.Default - If err = False Then - If USER_LANGUAGE = LANG_DE Then - MsgBox("Alle Dateien wurden mit Multiindexing erfolgreich verarbeitet!", MsgBoxStyle.Information, "Erfolgsmeldung:") - Else - MsgBox("All files were successfully processed through Multiindexing", MsgBoxStyle.Information, "Success") - End If - - Me.Close() - End If - End If - End If - Else - If WORK_FILE() = True Then - Me.Cursor = Cursors.Default + 'Private Sub OK_Button_Click(sender As Object, e As EventArgs) + ' ClearError() + ' ClearNotice() - If CONFIG.Config.ShowIndexResult = True Then - If USER_LANGUAGE = LANG_DE Then - MsgBox("Die Datei wurde erfolgreich verarbeitet!" & vbNewLine & "Ablagepfad:" & vbNewLine & CURRENT_NEWFILENAME, MsgBoxStyle.Information, "Erfolgsmeldung") - Else - MsgBox("File sucessfully processed!" & vbNewLine & "Path:" & vbNewLine & CURRENT_NEWFILENAME, MsgBoxStyle.Information, "Success") - End If - End If + ' 'lblhinweis.Visible = False + ' 'lblerror.Visible = False - Me.Close() - End If - End If - Me.Cursor = Cursors.Default - End Sub + ' Me.Cursor = Cursors.WaitCursor + ' ClassHelper.Refresh_RegexTable() + ' For Each rowregex As DataRow In CURRENT_DT_REGEX.Rows + ' If rowregex.Item("FUNCTION_NAME") = "CLEAN_FILENAME" Then + ' REGEX_CLEAN_FILENAME = rowregex.Item("REGEX") + ' End If + ' Next + ' If checkMultiindex.Visible = True And checkMultiindex.Checked = True Then + ' 'Die erste Datei indexieren + ' If WORK_FILE() = True Then + ' 'Und nun die folgenden + ' Dim DTFiles2Work As DataTable = ClassDatabase.Return_Datatable("SELECT * FROM TBGI_FILES_USER WHERE WORKED = 0 AND GUID <> " & CURRENT_WORKFILE_GUID & " AND UPPER(USER@WORK) = UPPER('" & Environment.UserName & "')") + ' If Not DTFiles2Work Is Nothing Then + ' Dim err = False + ' For Each filerow As DataRow In DTFiles2Work.Rows + ' CURRENT_WORKFILE_GUID = filerow.Item("GUID") + ' CURRENT_WORKFILE_HASH = NotNull(filerow.Item("FILE_HASH"), "") + ' CURRENT_WORKFILE = filerow.Item("FILENAME2WORK") + ' DropType = filerow.Item("HANDLE_TYPE") + + ' If WORK_FILE() = False Then + ' err = True + ' Exit For + ' End If + ' Next + ' Me.Cursor = Cursors.Default + ' If err = False Then + ' If USER_LANGUAGE = LANG_DE Then + ' MsgBox("Alle Dateien wurden mit Multiindexing erfolgreich verarbeitet!", MsgBoxStyle.Information, "Erfolgsmeldung:") + ' Else + ' MsgBox("All files were successfully processed through Multiindexing", MsgBoxStyle.Information, "Success") + ' End If + + ' Me.Close() + ' End If + ' End If + ' End If + ' Else + ' If WORK_FILE() = True Then + ' Me.Cursor = Cursors.Default + + ' If CONFIG.Config.ShowIndexResult = True Then + ' If USER_LANGUAGE = LANG_DE Then + ' MsgBox("Die Datei wurde erfolgreich verarbeitet!" & vbNewLine & "Ablagepfad:" & vbNewLine & CURRENT_NEWFILENAME, MsgBoxStyle.Information, "Erfolgsmeldung") + ' Else + ' MsgBox("File sucessfully processed!" & vbNewLine & "Path:" & vbNewLine & CURRENT_NEWFILENAME, MsgBoxStyle.Information, "Success") + ' End If + ' End If + + ' Me.Close() + ' End If + ' End If + ' Me.Cursor = Cursors.Default + 'End Sub Private Function Move_File2_Target() Dim oError As Boolean Try @@ -2321,7 +2324,9 @@ Public Class frmIndex Try Dim tempCur_WF = CURRENT_WORKFILE.Replace("'", "''") Dim tempCur_New_FN = CURRENT_NEWFILENAME.Replace("'", "''") - Insert_String = sql_history_INSERT_INTO & ",ADDED_WHO) VALUES ('" & tempCur_WF & "','" & tempCur_New_FN & "'" & sql_history_Index_Values & ",'" & Environment.UserDomainName & "\" & Environment.UserName & "')" + Dim oUser As String = $"{Environment.UserDomainName}\{Environment.UserName}" + + Insert_String = sql_history_INSERT_INTO & $",ADDED_WHO, ADDED_WHERE, FILE_HASH) VALUES ('{tempCur_WF}','{tempCur_New_FN}'{sql_history_Index_Values},'{oUser}','{Environment.MachineName}','{CURRENT_WORKFILE_HASH}')" ClassDatabase.Execute_Scalar(Insert_String, MyConnectionString, True) If DropType.Contains("MSG") Or DropType = "|ATTMNTEXTRACTED|" Or DropType = "|OUTLOOK_ATTACHMENT|" Then If CURRENT_MESSAGEID <> "" Then @@ -2641,6 +2646,7 @@ Public Class frmIndex For Each filerow As DataRow In DTFiles2Work.Rows CURRENT_WORKFILE_GUID = filerow.Item("GUID") CURRENT_WORKFILE = filerow.Item("FILENAME2WORK") + CURRENT_WORKFILE_HASH = NotNull(filerow.Item("FILE_HASH"), "") DropType = filerow.Item("HANDLE_TYPE") If WORK_FILE() = False Then diff --git a/Global_Indexer/frmStart.vb b/Global_Indexer/frmStart.vb index 098f889..5ffe6ad 100644 --- a/Global_Indexer/frmStart.vb +++ b/Global_Indexer/frmStart.vb @@ -160,19 +160,35 @@ Public Class frmStart Sub Check_Dropped_Files() Try ClassDatabase.Execute_non_Query("DELETE FROM TBGI_FILES_USER WHERE WORKED = 1 AND UPPER(USER@WORK) = UPPER('" & Environment.UserName & "')") - Dim i As Integer - For Each Str As Object In ClassFileDrop.files_dropped - If Not Str Is Nothing Then - LOGGER.Info(">> Check Drop-File: " & Str.ToString) - Dim handleType As String = Str.Substring(0, Str.LastIndexOf("|") + 1) - Dim filename As String = Str.Substring(Str.LastIndexOf("|") + 1) - If ClassIndexFunctions.FileExistsinDropTable(filename) = False Then - ClassFilehandle.Decide_FileHandle(filename, handleType) - i += 1 + Dim oIndex As Integer + + For Each oFiledropString As String In ClassFileDrop.files_dropped + If oFiledropString IsNot Nothing Then + LOGGER.Info(">> Check Drop-File: " & oFiledropString.ToString) + Dim oLastPipe = oFiledropString.LastIndexOf("|") + Dim oHandleType As String = oFiledropString.Substring(0, oLastPipe + 1) + Dim oFilename As String = oFiledropString.Substring(oLastPipe + 1) + Dim oFileExists As Boolean = ClassIndexFunctions.FileExistsinDropTable(oFilename) + + If oFileExists = False Then + ClassFilehandle.Decide_FileHandle(oFilename, oHandleType) + oIndex += 1 + ElseIf oFileExists Then + Dim oResult As DialogResult + + If USER_LANGUAGE = "de-DE" Then + oResult = MsgBox("Die Datei wurde bereits verarbeitet. Wollen Sie die gleiche Datei noch einmal verarbeiten?", MsgBoxStyle.Question Or MsgBoxStyle.YesNo, Text) + Else + oResult = MsgBox("This file has already been processed. Do you want to process the same file again?", MsgBoxStyle.Question Or MsgBoxStyle.YesNo, Text) + End If + + If oResult = DialogResult.Yes Then + ClassFilehandle.Decide_FileHandle(oFilename, oHandleType) + oIndex += 1 + End If Else - ' Console.WriteLine("File gibt es bereits") - End If + End If End If Next @@ -200,6 +216,8 @@ Public Class frmStart CURRENT_FILENAME = Filerow.Item("FILENAME2WORK") CURRENT_WORKFILE_GUID = Filerow.Item(0) CURRENT_WORKFILE = Filerow.Item("FILENAME2WORK") + CURRENT_WORKFILE_HASH = Filerow.Item("FILE_HASH") + LOGGER.Info(">> CURRENT_WORKFILE: " & CURRENT_WORKFILE) If File.Exists(CURRENT_WORKFILE) = True And DTACTUAL_FILES.Rows.Count > 0 Then Open_IndexDialog()