Import-FileContentToDB: Beschreibung ausgearbeitet, kleine Code Verbesserungen

current/Import-FileContentToDB/Import-FileContentToDB_History.txt

@@ -1,5 +1,21 @@
 -------------------------------------------------------------------------------
 
+Version 2.0.0.1 - 20.03.2024
+NEW: 	-
+FIX: 	-
+CHG: 	- Code improvement, clean up variables
+REM: 	-
+
+-------------------------------------------------------------------------------
+
+Version 2.0.0.0 - 18.03.2024
+NEW: 	-
+FIX: 	-
+CHG: 	- New working Version with Hash tables instead of arrays
+REM: 	-
+
+-------------------------------------------------------------------------------
+
 Version 1.2.0.0 - 06.12.2023
 NEW: 	- Excel Import Methode implemented
 FIX: 	-

current/Import-FileContentToDB/Import-FileContentToDB_Settings.ini

@@ -13,7 +13,7 @@
 # Example: E:\LogFiles\<ScriptName>                                                                #
 # FailSafe Setting is: <ScriptPath>\Logs                                                           #
 ####################################################################################################
-LogPath = 
+LogPath = E:\LogFiles\Import-FileContentToDB
 
 #################################################################################################### 
 # Numerical value how long (in days) log files will be kept. ValidateRange is 0 - 1000             #
@@ -47,8 +47,7 @@ DBType = MSSQL
 # Example: "Data Source=SERVERNAME\INSTANCE\;Initial Catalog=DD_ECM\;User Id=SQLUSERNAME\;         #
 # Password=SQLPASSWORD\;TrustServerCertificate=true\;"                                             #
 ####################################################################################################
-#DBConnString = "Data Source=S-ZLSDMS-DB1\;Initial Catalog=DD_ECM\;Trusted_Connection=True\;TrustServerCertificate=true\;"
+DBConnString = "Data Source=S-ZLSDMS-DB1\;Initial Catalog=DD_ECM\;User Id=DD_ECM\;Password=dd_ecm\;TrustServerCertificate=true\;"
-DBConnString = "Data Source=SDD-VMP04-SQL17\DD_DEVELOP01\;Initial Catalog=DD_ECM\;Trusted_Connection=True\;TrustServerCertificate=true\;"
 
 #==================================================================================================#
 ########################################## Profile Setup ###########################################
@@ -88,35 +87,57 @@ DBConnString = "Data Source=SDD-VMP04-SQL17\DD_DEVELOP01\;Initial Catalog=DD_ECM
 ####################################################################################################
 # Define Profiles in this config file for the executing Script.                                    #
 # Syntax:                                                                                          #
-# Profile = <ImportMethode> ; <ImportFormat> ; <SourcePath> ; <FirstLine> ; <LastLine> ;           #
+# Profile = <ImportMethode> ; <ImportFormat> ; <SourcePath> ; <SourceFileEncoding> ; <HeaderLine> ;#
-# <ValueSeperator> ; <ArchivPath> ; <ErrorPath>                                                    #
+# <FirstLine> ; <LastLine> ; <ValueSeperator|XSLX-Tab-name> ; <ArchivPath> ; <ErrorPath>           #
-# Example:                                                                                         #
-# Profile = BulkImport ; e:\SourceFile.csv                                                         #
-# Profile = CheckSQLQuery ; SQL Statement (SELECT)                                                 #
-# Profile = InitialSQLQuery ; SQL Statement (SELECT, INSERT, UPDATE)                               #
-# Profile = PreparingSQLQuery ; SQL Statement (SELECT, INSERT, UPDATE)                             #
-# Profile = MainLoopSQLQuery ; SQL Statement (INSERT, UPDATE)                                      #
-# Profile = FinalSQLQuery ; SQL Statement (SELECT, INSERT, UPDATE)                                 #
 #--------------------------------------------------------------------------------------------------#
 # First Parameter: <ImportMethode>                                                                 #
 # Set the way, Script is reading the SourceFile.                                                   #
 # Currently only the BulkImport is available.                                                      #
-# Example: BulkImport                                                                              #
+# Example: ExcelImport                                                                             #
+# Example: CSVImport                                                                               #
 #--------------------------------------------------------------------------------------------------#
-# Second Parameter: <SourceFile>                                                                   #
+# Second Parameter: <ImportFormat>                                                                 #
-# Set the full path and name to the file which includes the list you want to Import.               #
+# For future use....                                                                               #
-# Make sure, file is .csv and value separation is done by ";".                                     #
+# Example: %CellValue%                                                                             #
-# Example: E:\UserList.csv                                                                         #
 #--------------------------------------------------------------------------------------------------#
-# Optional Parameter: %ImportPathAD=<OU=OUName>,<DC=DomainName>,<DC=DomainSuffix>                  #
+# Third Parameter: <SourcePath>                                                                    #
-# Determ the TargetPath in your Active Directory (part of the distinguishedName).                  # 
+# Set the path to the source files.                                                                #
-# If you dont use this Parameter, TargetPath will be the "Users" Folder by default.                #
+# Example: E:\SourceFiles\*.csv                                                                    #
-# Example: OU=Testuser,DC=domainname,DC=local                                                      #
+# Example: E:\SourceFiles\SourceFile.xlsx                                                          #
 #--------------------------------------------------------------------------------------------------#
-# Optional Parameter: %ReportFile=<Path>\<FileName>.csv                                            #
+# Fourth Parameter: <SourceFileEncoding>                                                           #
-# By setting this Parameter, you get a overview file, which includes every created User,           #
+# Set the encoding type, the source files are written with.                                        #
-# with all available data (UserName, PreName, Surname, Password ...)                               #
+# Example: ascii                                                                                   #
-# Example: %ReportFile=E:\ReportFile.csv                                                           #
+# Example: utf7                                                                                    #
+# Example: utf8                                                                                    #
+# Example: unicode                                                                                 #
+#--------------------------------------------------------------------------------------------------#
+# Fifth Parameter: <HeaderLine>                                                                    #
+# Set the line of the heading of the table.                                                        #
+# Its the template for the inserts, only selected header columns will be imported.                 #
+# Example: 1                                                                                       #
+#--------------------------------------------------------------------------------------------------#
+# Sixth Parameter: <FirstLine>                                                                     #
+# Set the first line with data to import.                                                          #
+# Example: 2                                                                                       #
+#--------------------------------------------------------------------------------------------------#
+# Seventh Parameter: <LastLine>                                                                    #
+# Set the last line with data to import.                                                           #
+# You can set a very high number to guarantee everything will be imported.                         #
+# Example: 9999999                                                                                 #
+#--------------------------------------------------------------------------------------------------#
+# Eight Parameter: <ValueSeperator|XSLX-Tab-name>                                                  #
+# Set the CSV value seperator or the XLSX tab name.                                                #
+# Example: \;                                                                                      #
+# Example: XAP__Stammdaten_Adressen                                                                #
+#--------------------------------------------------------------------------------------------------#
+# Ninth Parameter: <ArchivPath>                                                                    #
+# Set the path, where successful processed files should be moved to.                               #
+# Example: E:\Archiv                                                                               #
+#--------------------------------------------------------------------------------------------------#
+# Tenth Parameter: <ErrorPath>                                                                     #
+# Set the path, where unsuccessful processed files should be moved to.                             #
+# Example: E:\Error                                                                                #
 #--------------------------------------------------------------------------------------------------#
 # Argument call:                                                                                   #
 # If you need to run the Script with dynamic Values, you can do so, by calling it with arguments.  #
@@ -125,29 +146,98 @@ DBConnString = "Data Source=SDD-VMP04-SQL17\DD_DEVELOP01\;Initial Catalog=DD_ECM
 #                            This Function is currently not implement!                             #
 ####################################################################################################
 
-Profile = ExcelImport ; %CellValue% ; D:\ScriptFiles\Import-FileContentToDB\Stammdaten.xlsx ; utf7 ; 4 ; 5 ; 999999999 ; XAP__Stammdaten_Adressen(2) ; E:\archiv ; E:\error
+####################################################################################################
+# Define additional special Profiles                                             .                 #
+# Syntax:                                                                                          #
+# PreparingFile = <Command>                                                                        #
+#--------------------------------------------------------------------------------------------------#
+# First Parameter: <Command>                                                                       #
+# Set a replace command if necessary.                                                              #
+# Example: Replace(<OldString>~,~<NewString>)                                                      #
+####################################################################################################
+
+####################################################################################################
+# Define additional special Profiles                                             .                 #
+# Syntax:                                                                                          #
+# CheckSQL = <SQLQuery>                                                                            #
+#--------------------------------------------------------------------------------------------------#
+# First Parameter: <Command>                                                                       #
+# Set the first SQL Query, this query will only be run once per file.                              #
+# Example: DELETE FROM [DD_ECM].[dbo].[TABLE]                                                      #
+####################################################################################################
+
+####################################################################################################
+# Define additional special Profiles                                             .                 #
+# Syntax:                                                                                          #
+# InitialSQL = <SQLQuery>                                                                          #
+#--------------------------------------------------------------------------------------------------#
+# First Parameter: <Command>                                                                       #
+# Set the second SQL Query, this query will only be run once per file.                             #
+# Example: DELETE FROM [DD_ECM].[dbo].[TABLE]                                                      #
+####################################################################################################
+
+####################################################################################################
+# Define additional special Profiles                                             .                 #
+# Syntax:                                                                                          #
+# PreparingSQL = <SQLQuery>                                                                        #
+#--------------------------------------------------------------------------------------------------#
+# First Parameter: <Command>                                                                       #
+# Set the third SQL Query, this query will only be run once per file.                              #
+# Example: DELETE FROM [DD_ECM].[dbo].[TABLE]                                                      #
+####################################################################################################
+
+####################################################################################################
+# Define additional special Profiles                                             .                 #
+# Syntax:                                                                                          #
+# MainLoopSQL = <SQLQuery>                                                                         #
+#--------------------------------------------------------------------------------------------------#
+# First Parameter: <Command>                                                                       #
+# Set the fourth SQL Query, this query will be run for every selected line in source file.         #
+# Use placeholder like '%column1%', for the cell values from the source file.                      #
+# Example: INSERT INTO [DD_ECM].[dbo].[TABLE](column1,column2) SELECT '%column1%','%column2%'      #
+####################################################################################################
+
+####################################################################################################
+# Define additional special Profiles                                             .                 #
+# Syntax:                                                                                          #
+# FinalSQL = <SQLQuery>                                                                            #
+#--------------------------------------------------------------------------------------------------#
+# First Parameter: <Command>                                                                       #
+# Set the fifth SQL Query, this query will only be run once per file.                              #
+# Example: DELETE FROM [DD_ECM].[dbo].[TABLE]                                                      #
+####################################################################################################
+
+Profile = ExcelImport ; %CellValue% ; E:\FileProcessing\Input\File\DocumentKindAssigned\XAP_Adressstammdaten\*.xlsx ; utf7 ; 4 ; 5 ; 999999999 ; XAP__Stammdaten_Adressen(2) ; E:\FileProcessing\Output\File\DocumentKindAssigned\XAP_Adressstammdaten ; E:\FileProcessing\Output\File\DocumentKindAssigned\XAP_Adressstammdaten
 PreparingFile = 
 CheckSQL = 
-InitialSQL =  DELETE FROM [DD_ECM_ZLS].[dbo].[TBCUST_XAP_KUNDENSTAMM]
+InitialSQL =  DELETE FROM [DD_ECM].[dbo].[TBCUST_XAP_KUNDENSTAMM]
 PreparingSQL = 
-MainLoopSQL = INSERT INTO [DD_ECM_ZLS].[dbo].[TBCUST_XAP_KUNDENSTAMM]([Kundennummer],[Objektnummer],[Versorgungsart],[rAdressart],[rAnrede],[rNAME1],[rNAME2],[rStraße],[rHausnummer],[rPostfach],[rPLZ],[rOrt],[rTelefon],[rMail],[vAdressart],[vAnrede],[vNAME1],[vNAME2],[vStraße],[vHausnummer],[vPostfach],[vPLZ],[vOrt],[vTelefon],[vMail],[ADDED_WHO],[ADDED_WHEN]) SELECT '%column1%','%column2%','%column3%','%column4%','%column5%','%column6%','%column7%','%column8%','%column9%','%column10%','%column11%','%column12%','%column13%','%column14%','%column15%','%column16%','%column17%','%column18%','%column19%','%column20%','%column21%','%column22%','%column23%','%column24%','%column25%','Import-FileContentToDB',getdate()
+MainLoopSQL = INSERT INTO [DD_ECM].[dbo].[TBCUST_XAP_KUNDENSTAMM]([Kundennummer],[Objektnummer],[Versorgungsart],[rAdressart],[rAnrede],[rNAME1],[rNAME2],[rStraße],[rHausnummer],[rPostfach],[rPLZ],[rOrt],[rTelefon],[rMail],[vAdressart],[vAnrede],[vNAME1],[vNAME2],[vStraße],[vHausnummer],[vPostfach],[vPLZ],[vOrt],[vTelefon],[vMail],[ADDED_WHO],[ADDED_WHEN]) SELECT '%column1%','%column2%','%column3%','%column4%','%column5%','%column6%','%column7%','%column8%','%column9%','%column10%','%column11%','%column12%','%column13%','%column14%','%column15%','%column16%','%column17%','%column18%','%column19%','%column20%','%column21%','%column22%','%column23%','%column24%','%column25%','Import-FileContentToDB',getdate()
 FinalSQL = 
 
-Profile = CSVImport ; %CellValue% ; D:\ScriptFiles\Import-FileContentToDB\*.csv ; utf7 ; 3 ; 4 ; 999999999 ; \^\,\^|\^ ; E:\archiv ; E:\error
+Profile = CSVImport ; %CellValue% ; \\s-zls22\Somentec\XAP-Mandanten\XAPPROD\Archiv\*.csv ; utf7 ; 3 ; 4 ; 999999999 ; \^\,\^|\^ ; \\s-zls22\Somentec\XAP-Mandanten\XAPPROD\Archiv\DMS-Eingelesen ; \\s-zls22\Somentec\XAP-Mandanten\XAPPROD\Archiv\DMS-Fehler
 PreparingFile = Replace(@FOLDER,FT:^HKA-IMP^~,~HKA-IMP)
 CheckSQL =
 InitialSQL =  
 PreparingSQL = 
-MainLoopSQL = INSERT INTO [DD_ECM_ZLS].[dbo].[TBCUST_XAP_DOC_EXPORT]([XAP_Beleg_ID],[XAP_Kunden_ID],[XAP_Kundenname],[XAP_Objekt_ID],[XAP_Objektbezeichnung],[XAP_Belegdatum],[XAP_Formular_ID],[XAP_Exportpfad],[ADDED_WHO],[ADDED_WHEN]) SELECT '%column1%','%column2%','%column3%','%column4%','%column5%','%column6%','%column7%','%column8%','Import-FileContentToDB',getdate()
+MainLoopSQL = INSERT INTO [DD_ECM].[dbo].[TBCUST_XAP_DOC_EXPORT]([XAP_Beleg_ID],[XAP_Kunden_ID],[XAP_Kundenname],[XAP_Objekt_ID],[XAP_Objektbezeichnung],[XAP_Belegdatum],[XAP_Formular_ID],[XAP_Exportpfad],[ADDED_WHO],[ADDED_WHEN]) SELECT '%column1%','%column2%','%column3%','%column4%','%column5%','%column6%','%column7%','%column8%','Import-FileContentToDB',getdate()
 FinalSQL = 
 
-#DELETE FROM [DD_ECM_ZLS].[dbo].[TBCUST_XAP_DOC_EXPORT]
 #Example:
-#Profile = CSVImport ; %ColumnName%|%CellValue% ; P:\Skriptentwickung\development\Import-CSV2DB\test-dateien\AltesFormat\*.txt ; 1 ; 2 ; . ;%SourceFilePath%\Archiv\%Timestamp5% ; %SourceFilePath%\Error
+#Profile = ExcelImport ; %CellValue% ; E:\FileProcessing\Input\File\DocumentKindAssigned\XAP_Adressstammdaten\*.xlsx ; utf7 ; 4 ; 5 ; 999999999 ; XAP__Stammdaten_Adressen(2) ; E:\FileProcessing\Output\File\DocumentKindAssigned\XAP_Adressstammdaten ; E:\FileProcessing\Output\File\DocumentKindAssigned\XAP_Adressstammdaten
-#Profile = CheckSQLQuery = SELECT 'true'
+#PreparingFile = 
-#Profile = InitialSQLQuery = SELECT (SELECT [u003] FROM [CWLDATEN].[dbo].[t693] WHERE [u000] = 1)+1
+#CheckSQL = 
-#Profile = PreparingSQLQuery = INSERT INTO [CWLDATEN].[dbo].[TBDD_BATCHIMPORT_HEAD] (BELEGKEY, Konto, Laufnummer, Auftragsnummer) VALUES (%InitialSQLResult%, '700131', NULL, NULL) UPDATE [CWLDATEN].[dbo].[t693] SET u003 = %InitialSQLResult%  WHERE [u000] = 1 
+#InitialSQL =  DELETE FROM [DD_ECM].[dbo].[TBCUST_XAP_KUNDENSTAMM]
-#Profile = MainLoopSQLQuery = INSERT INTO [CWLDATEN].[dbo].[t699] (u000, u001, u002, u003, u004, u005, u006, u007, u008, u009, u010, u011, u012, u013, u014, u015, u016, u029, u030) Select ISNULL(MAX(u000),0) + 1, %InitialSQLResult%, '%sourcefile%', '%column3%', '%column1%', '%column2%', '%column3%', '%column4%', '%column5%', '%column6%', '%column7%', '%column8%', '%column9%', '%column10%', '%column11%', '%column12%', '%column13%', getdate(), '%username%'  FROM [CWLDATEN].[dbo].[t699]
+#PreparingSQL = 
-#Profile = FinalSQLQuery = 
+#MainLoopSQL = INSERT INTO [DD_ECM].[dbo].[TBCUST_XAP_KUNDENSTAMM]([Kundennummer],[Objektnummer],[Versorgungsart],[rAdressart],[rAnrede],[rNAME1],[rNAME2],[rStraße],[rHausnummer],[rPostfach],[rPLZ],[rOrt],[rTelefon],[rMail],[vAdressart],[vAnrede],[vNAME1],[vNAME2],[vStraße],[vHausnummer],[vPostfach],[vPLZ],[vOrt],[vTelefon],[vMail],[ADDED_WHO],[ADDED_WHEN]) SELECT '%column1%','%column2%','%column3%','%column4%','%column5%','%column6%','%column7%','%column8%','%column9%','%column10%','%column11%','%column12%','%column13%','%column14%','%column15%','%column16%','%column17%','%column18%','%column19%','%column20%','%column21%','%column22%','%column23%','%column24%','%column25%','Import-FileContentToDB',getdate()
+#FinalSQL = 
+
+#Profile = CSVImport ; %CellValue% ; \\s-zls22\Somentec\XAP-Mandanten\XAPPROD\Archiv\*.csv ; utf7 ; 3 ; 4 ; 999999999 ; \^\,\^|\^ ; \\s-zls22\Somentec\XAP-Mandanten\XAPPROD\Archiv\DMS-Eingelesen ; \\s-zls22\Somentec\XAP-Mandanten\XAPPROD\Archiv\DMS-Fehler
+#PreparingFile = Replace(@FOLDER,FT:^HKA-IMP^~,~HKA-IMP)
+#CheckSQL =
+#InitialSQL =  
+#PreparingSQL = 
+#MainLoopSQL = INSERT INTO [DD_ECM].[dbo].[TBCUST_XAP_DOC_EXPORT]([XAP_Beleg_ID],[XAP_Kunden_ID],[XAP_Kundenname],[XAP_Objekt_ID],[XAP_Objektbezeichnung],[XAP_Belegdatum],[XAP_Formular_ID],[XAP_Exportpfad],[ADDED_WHO],[ADDED_WHEN]) SELECT '%column1%','%column2%','%column3%','%column4%','%column5%','%column6%','%column7%','%column8%','Import-FileContentToDB',getdate()
+#FinalSQL =