From 93a30510276cb64a46da667cdae6b69aa1b91077 Mon Sep 17 00:00:00 2001 From: pitzm Date: Thu, 6 Jun 2024 11:05:05 +0200 Subject: [PATCH 1/2] BULK-Insert im ZUGFeRD Service --- Interfaces/ZUGFeRDInterface/PropertyValues.vb | 4 +- Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb | 152 ++++++++++++++---- Patterns/Patterns.vbproj.bak | 151 ----------------- 3 files changed, 123 insertions(+), 184 deletions(-) delete mode 100644 Patterns/Patterns.vbproj.bak diff --git a/Interfaces/ZUGFeRDInterface/PropertyValues.vb b/Interfaces/ZUGFeRDInterface/PropertyValues.vb index 3180d8ea..63cfb0d7 100644 --- a/Interfaces/ZUGFeRDInterface/PropertyValues.vb +++ b/Interfaces/ZUGFeRDInterface/PropertyValues.vb @@ -104,9 +104,7 @@ Public Class PropertyValues Dim oPropertyValue = oColumn.Value.ElementAtOrDefault(oRowIndex) _logger.Debug("Processing itemSpecification *TableColumn* [{0}].", oTableColumn) - If oTableColumn = "INVOICE_SELLER_EMAIL" Then - Console.WriteLine("INVOICE_SELLER_EMAIL") - End If + If IsNothing(oPropertyValue) OrElse String.IsNullOrEmpty(oPropertyValue) Then If oColumn.Key.IsRequired Then _logger.Warn($"{MessageId} # oPropertyValue for specification [{oTableColumn}] is empty or not found but is required. Continuing with Empty String.") diff --git a/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb b/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb index 251bac2f..edae954d 100644 --- a/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb +++ b/Jobs/ZUGFeRD/ImportZUGFeRDFiles.vb @@ -10,6 +10,8 @@ Imports DigitalData.Modules.Interfaces.Exceptions Imports DigitalData.Modules.Jobs.Exceptions Imports DigitalData.Modules.Logging Imports System.Data.SqlClient +Imports Newtonsoft.Json.Linq +Imports System.Xml.Linq Public Class ImportZUGFeRDFiles Implements IJob @@ -527,22 +529,44 @@ Public Class ImportZUGFeRDFiles DeleteExistingPropertyValues(pMessageId, oConnections) - Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault() - If oFirstProperty IsNot Nothing Then - InsertPropertyValue(pMessageId, oConnections, New PropertyValues.ValidProperty() With { - .MessageId = pMessageId, - .Description = "ZUGFeRDSpezifikation", - .GroupCounter = 0, - .IsRequired = False, - .Value = oDocument.Specification, - .TableName = oFirstProperty.TableName, - .TableColumn = "ZUGFERD_SPECIFICATION" - }) + ' MP 05.06.2024 - Einzel-Inserts durch BULK-Insert abgelöst + 'Dim oFirstProperty = oCheckResult.ValidProperties.FirstOrDefault() + 'If oFirstProperty IsNot Nothing Then + ' InsertPropertyValue(pMessageId, oConnections, New PropertyValues.ValidProperty() With { + ' .MessageId = pMessageId, + ' .Description = "ZUGFeRDSpezifikation", + ' .GroupCounter = 0, + ' .IsRequired = False, + ' .Value = oDocument.Specification, + ' .TableName = oFirstProperty.TableName, + ' .TableColumn = "ZUGFERD_SPECIFICATION" + ' }) + 'End If + + 'For Each oProperty In oCheckResult.ValidProperties + ' InsertPropertyValue(pMessageId, oConnections, oProperty) + 'Next + + ' DataTable vorbereiten + Dim oDataTable As DataTable = FillDataTable(pMessageId, oCheckResult, oDocument.Specification) + + ' ColumnList initialisieren + Dim oColumnNames As List(Of String) = New List(Of String) From { + "REFERENCE_GUID", + "ITEM_DESCRIPTION", + "ITEM_VALUE", + "GROUP_COUNTER", + "SPEC_NAME", + "IS_REQUIRED" + } + + Dim oBulkResult = BulkInsert(oConnections, oDataTable, "TBEDMI_ITEM_VALUE", oColumnNames) + + If oBulkResult = False Then + _logger.Error("Bulk Insert for MessageId [{0}] failed!", pMessageId) End If - For Each oProperty In oCheckResult.ValidProperties - InsertPropertyValue(pMessageId, oConnections, oProperty) - Next + _logger.Info("Bulk Insert finished. [{0}] rows inserted for MessageId [{1}].", oDataTable.Rows.Count, pMessageId) _logger.Debug("File processed.") @@ -554,6 +578,53 @@ Public Class ImportZUGFeRDFiles End Function + Private Function FillDataTable(pMessageId As String, pCheckResult As PropertyValues.CheckPropertyValuesResult, pSpecification As String) As DataTable + + Dim oDataTable As DataTable = New DataTable() + oDataTable.Columns.Add(New DataColumn("REFERENCE_GUID", GetType(String))) + oDataTable.Columns.Add(New DataColumn("ITEM_DESCRIPTION", GetType(String))) + oDataTable.Columns.Add(New DataColumn("ITEM_VALUE", GetType(String))) + oDataTable.Columns.Add(New DataColumn("GROUP_COUNTER", GetType(Int32))) + oDataTable.Columns.Add(New DataColumn("SPEC_NAME", GetType(String))) + oDataTable.Columns.Add(New DataColumn("IS_REQUIRED", GetType(Boolean))) + + ' Erste Zeile enthält die Spezifikation + Dim oFirstRow As DataRow = oDataTable.NewRow() + oFirstRow("REFERENCE_GUID") = pMessageId + oFirstRow("ITEM_DESCRIPTION") = "ZUGFeRDSpezifikation" + oFirstRow("ITEM_VALUE") = pSpecification + oFirstRow("GROUP_COUNTER") = 0 + oFirstRow("SPEC_NAME") = "ZUGFERD_SPECIFICATION" + oFirstRow("IS_REQUIRED") = False + + _logger.Debug("Mapping Property [ZUGFERD_SPECIFICATION] with value [{0}]", pSpecification) + oDataTable.Rows.Add(oFirstRow) + + + For Each oProperty In pCheckResult.ValidProperties + + ' If GroupCounter is -1, it means this is a default property that can only occur once. + ' Set the actual inserted value to 0 + Dim oGroupCounterValue As Integer = oProperty.GroupCounter + If oGroupCounterValue = -1 Then + oGroupCounterValue = 0 + End If + + Dim oNewRow As DataRow = oDataTable.NewRow() + oNewRow("REFERENCE_GUID") = pMessageId + oNewRow("ITEM_DESCRIPTION") = oProperty.Description + oNewRow("ITEM_VALUE") = oProperty.Value.Replace("'", "''") + oNewRow("GROUP_COUNTER") = oGroupCounterValue + oNewRow("SPEC_NAME") = oProperty.TableColumn + oNewRow("IS_REQUIRED") = oProperty.IsRequired + + _logger.Debug("Mapping Property [{0}] with value [{1}]", oProperty.TableColumn, oProperty.Value.Replace("'", "''")) + oDataTable.Rows.Add(oNewRow) + Next + + Return oDataTable + End Function + Private Sub DeleteExistingPropertyValues(pMessageId As String, pConnections As DatabaseConnections) Dim oDelSQL = $"DELETE FROM TBEDMI_ITEM_VALUE where REFERENCE_GUID = '{pMessageId}'" Dim oStep As String @@ -566,25 +637,46 @@ Public Class ImportZUGFeRDFiles End Try End Sub - Private Sub InsertPropertyValue(pMessageId As String, pConnections As DatabaseConnections, pProperty As PropertyValues.ValidProperty) - Dim oGroupCounterValue = pProperty.GroupCounter + ' Alte Insert-Methode + 'Private Sub InsertPropertyValue(pMessageId As String, pConnections As DatabaseConnections, pProperty As PropertyValues.ValidProperty) + ' Dim oGroupCounterValue = pProperty.GroupCounter - ' If GroupCounter is -1, it means this is a default property that can only occur once. - ' Set the actual inserted value to 0 - If oGroupCounterValue = -1 Then - oGroupCounterValue = 0 - End If + ' ' If GroupCounter is -1, it means this is a default property that can only occur once. + ' ' Set the actual inserted value to 0 + ' If oGroupCounterValue = -1 Then + ' oGroupCounterValue = 0 + ' End If - Dim oCommand = $"INSERT INTO {pProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER, SPEC_NAME, IS_REQUIRED) VALUES - ('{pMessageId}', '{pProperty.Description}', '{pProperty.Value.Replace("'", "''")}', {oGroupCounterValue},'{pProperty.TableColumn}','{pProperty.IsRequired}')" - _logger.Debug("Mapping Property [{0}] with value [{1}], Will be inserted into table [{2}]", pProperty.TableColumn, pProperty.Value.Replace("'", "''"), pProperty.TableName) + ' Dim oCommand = $"INSERT INTO {pProperty.TableName} (REFERENCE_GUID, ITEM_DESCRIPTION, ITEM_VALUE, GROUP_COUNTER, SPEC_NAME, IS_REQUIRED) VALUES + ' ('{pMessageId}', '{pProperty.Description}', '{pProperty.Value.Replace("'", "''")}', {oGroupCounterValue},'{pProperty.TableColumn}','{pProperty.IsRequired}')" + ' _logger.Debug("Mapping Property [{0}] with value [{1}], Will be inserted into table [{2}]", pProperty.TableColumn, pProperty.Value.Replace("'", "''"), pProperty.TableName) - ' Insert into SQL Server - Dim oResult = _mssql.ExecuteNonQueryWithConnectionObject(oCommand, pConnections.SQLServerConnection, MSSQLServer.TransactionMode.ExternalTransaction, pConnections.SQLServerTransaction) - If oResult = False Then - _logger.Warn($"SQL Command [{oCommand}] was not successful. Check the log.") - End If - End Sub + ' ' Insert into SQL Server + ' Dim oResult = _mssql.ExecuteNonQueryWithConnectionObject(oCommand, pConnections.SQLServerConnection, MSSQLServer.TransactionMode.ExternalTransaction, pConnections.SQLServerTransaction) + ' If oResult = False Then + ' _logger.Warn($"SQL Command [{oCommand}] was not successful. Check the log.") + ' End If + 'End Sub + + Private Function BulkInsert(pConnections As DatabaseConnections, pTable As DataTable, pDestinationTable As String, pColumns As List(Of String)) As Boolean + + Using oBulkCopy = New SqlBulkCopy(pConnections.SQLServerConnection, SqlBulkCopyOptions.Default, pConnections.SQLServerTransaction) + + oBulkCopy.DestinationTableName = pDestinationTable + For Each oColumn In pColumns + oBulkCopy.ColumnMappings.Add(New SqlBulkCopyColumnMapping(oColumn, oColumn)) + Next + + Try + oBulkCopy.WriteToServer(pTable) + Catch ex As Exception + _logger.Error(ex) + Return False + End Try + End Using + + Return True + End Function Private Sub AddRejectedState(pMessageID As String, pTitle As String, pTitle1 As String, pComment As String, pTransaction As SqlTransaction) Try diff --git a/Patterns/Patterns.vbproj.bak b/Patterns/Patterns.vbproj.bak deleted file mode 100644 index c22610ea..00000000 --- a/Patterns/Patterns.vbproj.bak +++ /dev/null @@ -1,151 +0,0 @@ - - - - - Debug - AnyCPU - {7C3B0C7E-59FE-4E1A-A655-27AE119F9444} - Library - DigitalData.Modules.Patterns - DigitalData.Modules.Patterns - 512 - Windows - v4.6.1 - - - true - full - true - true - bin\Debug\ - DigitalData.Modules.Patterns.xml - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 - - - pdbonly - false - true - true - bin\Release\ - DigitalData.Modules.Patterns.xml - 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022 - - - On - - - Binary - - - Off - - - On - - - - - - - P:\Visual Studio Projekte\Bibliotheken\windream\Interop.WINDREAMLib.dll - True - - - - ..\packages\NLog.4.7.10\lib\net45\NLog.dll - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - True - Application.myapp - - - True - True - Resources.resx - - - True - Settings.settings - True - - - - - - VbMyResourcesResXFileCodeGenerator - Resources.Designer.vb - My.Resources - Designer - - - - - - MyApplicationCodeGenerator - Application.Designer.vb - - - SettingsSingleFileGenerator - My - Settings.Designer.vb - - - - - - {3dcd6d1a-c830-4241-b7e4-27430e7ea483} - LookupControl - - - {903B2D7D-3B80-4BE9-8713-7447B704E1B0} - Logging - - - {81cac44f-3711-4c8f-ae98-e02a7448782a} - ZooFlow - - - - - \ No newline at end of file From c0d2e254371bb0c2c7cb01c28813950983f08657 Mon Sep 17 00:00:00 2001 From: pitzm Date: Thu, 6 Jun 2024 11:06:21 +0200 Subject: [PATCH 2/2] Modules.Jobs: Version 2.5.8.0 --- Jobs/My Project/AssemblyInfo.vb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jobs/My Project/AssemblyInfo.vb b/Jobs/My Project/AssemblyInfo.vb index 414eb306..55a40070 100644 --- a/Jobs/My Project/AssemblyInfo.vb +++ b/Jobs/My Project/AssemblyInfo.vb @@ -13,7 +13,7 @@ Imports System.Runtime.InteropServices - + @@ -30,5 +30,5 @@ Imports System.Runtime.InteropServices ' Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern ' übernehmen, indem Sie "*" eingeben: - - + +