AppStd/Monorepo
8
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

set logEntireMessage to false to prevent outofmemory exception when importing multiple large files

Browse Source
This commit is contained in:
Jonathan Jenne 2020-04-08 13:42:17 +02:00
parent b78949ae46
commit 86c99f0fc6
7 changed files with 124 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
GUIs.Test.EDMIBenchmark/Form1.vb
View File

@ -42,7 +42,7 @@ Public Class Form1
Dim oFileName As String = oItem Dim oFileName As String = oItem
Dim oFileInfo As New FileInfo(oFileName) Dim oFileInfo As New FileInfo(oFileName)
listboxLog.Items.Add($"Importing {oFileInfo.Name}...") listboxLog.Items.Add($"Importing {oFileInfo.Name}... ({FormatBytes(oFileInfo.Length)})")
Dim oContents As Byte() = New Byte(oFileInfo.Length) {} Dim oContents As Byte() = New Byte(oFileInfo.Length) {}
@ -58,7 +58,6 @@ Public Class Form1
End If End If
oSW.Stop() oSW.Stop()
listboxLog.Items.Add($"Import Time: {FormatTime(oSW.ElapsedMilliseconds)}") listboxLog.Items.Add($"Import Time: {FormatTime(oSW.ElapsedMilliseconds)}")
listboxLog.Items.Add($"File Size: {FormatBytes(oFileInfo.Length)}")
listboxLog.Items.Add("") listboxLog.Items.Add("")
Next Next

4
Modules.EDMIAPI/Connected Services/EDMIServiceReference/configuration.svcinfo
View File

@ -2,9 +2,9 @@
<configurationSnapshot xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns="urn:schemas-microsoft-com:xml-wcfconfigurationsnapshot"> <configurationSnapshot xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns="urn:schemas-microsoft-com:xml-wcfconfigurationsnapshot">
<behaviors /> <behaviors />
<bindings> <bindings>
<binding digest="System.ServiceModel.Configuration.NetTcpBindingElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089:&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-16&quot;?&gt;&lt;Data name=&quot;tcpBinding&quot;&gt;&lt;security mode=&quot;None&quot;&gt;&lt;transport sslProtocols=&quot;None&quot; /&gt;&lt;/security&gt;&lt;/Data&gt;" bindingType="netTcpBinding" name="tcpBinding" /> <binding digest="System.ServiceModel.Configuration.NetTcpBindingElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089:&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-16&quot;?&gt;&lt;Data name=&quot;tcpBinding&quot;&gt;&lt;security&gt;&lt;transport sslProtocols=&quot;None&quot; /&gt;&lt;/security&gt;&lt;/Data&gt;" bindingType="netTcpBinding" name="tcpBinding" />
</bindings> </bindings>
<endpoints> <endpoints>
<endpoint normalizedDigest="&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-16&quot;?&gt;&lt;Data address=&quot;net.tcp://localhost:9000/DigitalData/Services/Main&quot; binding=&quot;netTcpBinding&quot; bindingConfiguration=&quot;tcpBinding&quot; contract=&quot;EDMIServiceReference.IEDMIService&quot; name=&quot;tcpBinding&quot;&gt;&lt;identity&gt;&lt;dns value=&quot;localhost&quot; /&gt;&lt;/identity&gt;&lt;/Data&gt;" digest="&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-16&quot;?&gt;&lt;Data address=&quot;net.tcp://localhost:9000/DigitalData/Services/Main&quot; binding=&quot;netTcpBinding&quot; bindingConfiguration=&quot;tcpBinding&quot; contract=&quot;EDMIServiceReference.IEDMIService&quot; name=&quot;tcpBinding&quot;&gt;&lt;identity&gt;&lt;dns value=&quot;localhost&quot; /&gt;&lt;/identity&gt;&lt;/Data&gt;" contractName="EDMIServiceReference.IEDMIService" name="tcpBinding" /> <endpoint normalizedDigest="&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-16&quot;?&gt;&lt;Data address=&quot;net.tcp://localhost:9000/DigitalData/Services/Main&quot; binding=&quot;netTcpBinding&quot; bindingConfiguration=&quot;tcpBinding&quot; contract=&quot;EDMIServiceReference.IEDMIService&quot; name=&quot;tcpBinding&quot;&gt;&lt;identity&gt;&lt;servicePrincipalName value=&quot;host/sDD-VMP03-VM09.dd-san01.dd-gan.local.digitaldata.works&quot; /&gt;&lt;/identity&gt;&lt;/Data&gt;" digest="&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-16&quot;?&gt;&lt;Data address=&quot;net.tcp://localhost:9000/DigitalData/Services/Main&quot; binding=&quot;netTcpBinding&quot; bindingConfiguration=&quot;tcpBinding&quot; contract=&quot;EDMIServiceReference.IEDMIService&quot; name=&quot;tcpBinding&quot;&gt;&lt;identity&gt;&lt;servicePrincipalName value=&quot;host/sDD-VMP03-VM09.dd-san01.dd-gan.local.digitaldata.works&quot; /&gt;&lt;/identity&gt;&lt;/Data&gt;" contractName="EDMIServiceReference.IEDMIService" name="tcpBinding" />
</endpoints> </endpoints>
</configurationSnapshot> </configurationSnapshot>

14
Modules.EDMIAPI/Connected Services/EDMIServiceReference/configuration91.svcinfo
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<SavedWcfConfigurationInformation xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" Version="9.1" CheckSum="iWIOdzq/CtaAM4SQMzp+vG4wC2Y="> <SavedWcfConfigurationInformation xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" Version="9.1" CheckSum="e9iHagUK3+X05AXstziypKiw8Rs=">
<bindingConfigurations> <bindingConfigurations>
<bindingConfiguration bindingType="netTcpBinding" name="tcpBinding"> <bindingConfiguration bindingType="netTcpBinding" name="tcpBinding">
<properties> <properties>
@ -81,8 +81,8 @@
<property path="/security" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.NetTcpSecurityElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/security" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.NetTcpSecurityElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue>System.ServiceModel.Configuration.NetTcpSecurityElement</serializedValue> <serializedValue>System.ServiceModel.Configuration.NetTcpSecurityElement</serializedValue>
</property> </property>
<property path="/security/mode" isComplexType="false" isExplicitlyDefined="true" clrType="System.ServiceModel.SecurityMode, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/security/mode" isComplexType="false" isExplicitlyDefined="false" clrType="System.ServiceModel.SecurityMode, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue>None</serializedValue> <serializedValue>Transport</serializedValue>
</property> </property>
<property path="/security/transport" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.TcpTransportSecurityElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/security/transport" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.TcpTransportSecurityElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue>System.ServiceModel.Configuration.TcpTransportSecurityElement</serializedValue> <serializedValue>System.ServiceModel.Configuration.TcpTransportSecurityElement</serializedValue>
@ -156,14 +156,14 @@
<property path="/identity/servicePrincipalName" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.ServicePrincipalNameElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/identity/servicePrincipalName" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.ServicePrincipalNameElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue>System.ServiceModel.Configuration.ServicePrincipalNameElement</serializedValue> <serializedValue>System.ServiceModel.Configuration.ServicePrincipalNameElement</serializedValue>
</property> </property>
<property path="/identity/servicePrincipalName/value" isComplexType="false" isExplicitlyDefined="false" clrType="System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/identity/servicePrincipalName/value" isComplexType="false" isExplicitlyDefined="true" clrType="System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue /> <serializedValue>host/sDD-VMP03-VM09.dd-san01.dd-gan.local.digitaldata.works</serializedValue>
</property> </property>
<property path="/identity/dns" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.DnsElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/identity/dns" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.DnsElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue>System.ServiceModel.Configuration.DnsElement</serializedValue> <serializedValue>System.ServiceModel.Configuration.DnsElement</serializedValue>
</property> </property>
<property path="/identity/dns/value" isComplexType="false" isExplicitlyDefined="true" clrType="System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/identity/dns/value" isComplexType="false" isExplicitlyDefined="false" clrType="System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue>localhost</serializedValue> <serializedValue />
</property> </property>
<property path="/identity/rsa" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.RsaElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"> <property path="/identity/rsa" isComplexType="true" isExplicitlyDefined="false" clrType="System.ServiceModel.Configuration.RsaElement, System.ServiceModel, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<serializedValue>System.ServiceModel.Configuration.RsaElement</serializedValue> <serializedValue>System.ServiceModel.Configuration.RsaElement</serializedValue>

25
Modules.EDMIAPI/Connected Services/EDMIServiceReference/service.wsdl
View File

@ -5,6 +5,29 @@
<wsp:All> <wsp:All>
<msb:BinaryEncoding xmlns:msb="http://schemas.microsoft.com/ws/06/2004/mspolicy/netbinary1"> <msb:BinaryEncoding xmlns:msb="http://schemas.microsoft.com/ws/06/2004/mspolicy/netbinary1">
</msb:BinaryEncoding> </msb:BinaryEncoding>
<sp:TransportBinding xmlns:sp="http://schemas.xmlsoap.org/ws/2005/07/securitypolicy">
<wsp:Policy>
<sp:TransportToken>
<wsp:Policy>
<msf:WindowsTransportSecurity xmlns:msf="http://schemas.microsoft.com/ws/2006/05/framing/policy">
<msf:ProtectionLevel>EncryptAndSign</msf:ProtectionLevel>
</msf:WindowsTransportSecurity>
</wsp:Policy>
</sp:TransportToken>
<sp:AlgorithmSuite>
<wsp:Policy>
<sp:Basic256>
</sp:Basic256>
</wsp:Policy>
</sp:AlgorithmSuite>
<sp:Layout>
<wsp:Policy>
<sp:Strict>
</sp:Strict>
</wsp:Policy>
</sp:Layout>
</wsp:Policy>
</sp:TransportBinding>
<wsaw:UsingAddressing> <wsaw:UsingAddressing>
</wsaw:UsingAddressing> </wsaw:UsingAddressing>
</wsp:All> </wsp:All>
@ -149,7 +172,7 @@
<wsa10:EndpointReference> <wsa10:EndpointReference>
<wsa10:Address>net.tcp://localhost:9000/DigitalData/Services/Main</wsa10:Address> <wsa10:Address>net.tcp://localhost:9000/DigitalData/Services/Main</wsa10:Address>
<Identity xmlns="http://schemas.xmlsoap.org/ws/2006/02/addressingidentity"> <Identity xmlns="http://schemas.xmlsoap.org/ws/2006/02/addressingidentity">
<Dns>localhost</Dns> <Spn>host/sDD-VMP03-VM09.dd-san01.dd-gan.local.digitaldata.works</Spn>
</Identity> </Identity>
</wsa10:EndpointReference> </wsa10:EndpointReference>
</wsdl:port> </wsdl:port>

149
Modules.EDMIAPI/Document.vb
View File

@ -53,15 +53,17 @@ Public Class Document
''' </summary> ''' </summary>
''' <param name="FilePath">The filename to import</param> ''' <param name="FilePath">The filename to import</param>
''' <returns>A document object</returns> ''' <returns>A document object</returns>
Public Function ImportFile(FilePath As String) As DocumentResult Public Async Function ImportFileAsync(FilePath As String, Optional [ReadOnly] As Boolean = False, Optional RetentionPeriod As Integer = 0) As Task(Of DocumentResult2)
Try Try
Dim oContents As Byte() = File.ReadAllBytes(FilePath)
Dim oInfo As New FileInfo(FilePath) Dim oInfo As New FileInfo(FilePath)
Dim oName As String = oInfo.Name
Dim oExtension As String = oInfo.Extension.Substring(1)
Dim oDocObject = _channel.NewFile(oName, oContents) Using oStream As New FileStream(FilePath, FileMode.Open)
Return oDocObject Dim oContents As Byte() = {}
Dim oBytesRead = Await oStream.ReadAsync(oContents, 0, oStream.Length)
Dim oResult = Await _channel.ImportFileAsync(oInfo, oContents, [ReadOnly], RetentionPeriod)
Return oResult
End Using
Catch ex As Exception Catch ex As Exception
_logger.Error(ex) _logger.Error(ex)
Throw ex Throw ex
@ -73,34 +75,11 @@ Public Class Document
''' </summary> ''' </summary>
''' <param name="FilePath">The filename to import</param> ''' <param name="FilePath">The filename to import</param>
''' <returns>A document object</returns> ''' <returns>A document object</returns>
Public Async Function ImportFile2(FilePath As String, Optional [ReadOnly] As Boolean = False, Optional RetentionPeriod As Integer = -1) As Task(Of DocumentResult2) Public Function ImportFile(FilePath As String, Optional [ReadOnly] As Boolean = False, Optional RetentionPeriod As Integer = 0) As DocumentResult2
Try Try
Dim oContents As Byte() = File.ReadAllBytes(FilePath) Dim oContents As Byte() = File.ReadAllBytes(FilePath)
Dim oInfo As New FileInfo(FilePath) Dim oInfo As New FileInfo(FilePath)
Dim oName As String = oInfo.Name Dim oDocObject = _channel.ImportFile(oInfo, oContents, [ReadOnly], RetentionPeriod)
Dim oExtension As String = oInfo.Extension.Substring(1)
Dim oResult = Await _channel.ImportFileAsync(oInfo, oContents, [ReadOnly], RetentionPeriod)
Return oResult
Catch ex As Exception
_logger.Error(ex)
Throw ex
End Try
End Function
''' <summary>
''' Imports a file by filename
''' </summary>
''' <param name="FilePath">The filename to import</param>
''' <returns>A document object</returns>
Public Async Function ImportFileAsync(FilePath As String) As Task(Of DocumentResult)
Try
Dim oContents As Byte() = File.ReadAllBytes(FilePath)
Dim oInfo As New FileInfo(FilePath)
Dim oName As String = oInfo.Name
Dim oExtension As String = oInfo.Extension.Substring(1)
Dim oDocObject = Await _channel.NewFileAsync(oName, oContents)
Return oDocObject Return oDocObject
Catch ex As Exception Catch ex As Exception
_logger.Error(ex) _logger.Error(ex)
@ -108,45 +87,85 @@ Public Class Document
End Try End Try
End Function End Function
Public Async Function NewFileIndexAsync(DocObject As DocumentObject, Syskey As String, LanguageCode As String, Value As String) As Task(Of IndexResult) '''' <summary>
Try '''' Imports a file by filename
Dim oResult As IndexResult = Await _channel.NewFileIndexAsync(DocObject, Syskey, LanguageCode, Value) '''' </summary>
'''' <param name="FilePath">The filename to import</param>
'''' <returns>A document object</returns>
'Public Function ImportFile(FilePath As String) As DocumentResult
' Try
' Dim oContents As Byte() = File.ReadAllBytes(FilePath)
' Dim oInfo As New FileInfo(FilePath)
' Dim oName As String = oInfo.Name
' Dim oExtension As String = oInfo.Extension.Substring(1)
Return oResult ' Dim oDocObject = _channel.NewFile(oName, oContents)
Catch ex As Exception ' Return oDocObject
_logger.Error(ex) ' Catch ex As Exception
Throw ex ' _logger.Error(ex)
End Try ' Throw ex
End Function ' End Try
'End Function
Public Function NewFileIndex(DocObject As DocumentObject, Syskey As String, LanguageCode As String, Value As String) As IndexResult '''' <summary>
Try '''' Imports a file by filename
Dim oResult As IndexResult = _channel.NewFileIndex(DocObject, Syskey, LanguageCode, Value) '''' </summary>
'''' <param name="FilePath">The filename to import</param>
'''' <returns>A document object</returns>
'Public Async Function ImportFileAsync(FilePath As String) As Task(Of DocumentResult)
' Try
' Dim oContents As Byte() = File.ReadAllBytes(FilePath)
' Dim oInfo As New FileInfo(FilePath)
' Dim oName As String = oInfo.Name
' Dim oExtension As String = oInfo.Extension.Substring(1)
Return oResult ' Dim oDocObject = Await _channel.NewFileAsync(oName, oContents)
Catch ex As Exception ' Return oDocObject
_logger.Error(ex) ' Catch ex As Exception
Throw ex ' _logger.Error(ex)
End Try ' Throw ex
End Function ' End Try
'End Function
Public Function GetDocumentByDocumentId(DocumentId As Int64) As DocumentResult 'Public Async Function NewFileIndexAsync(DocObject As DocumentObject, Syskey As String, LanguageCode As String, Value As String) As Task(Of IndexResult)
Try ' Try
Return _channel.GetDocumentByDocumentId(DocumentId) ' Dim oResult As IndexResult = Await _channel.NewFileIndexAsync(DocObject, Syskey, LanguageCode, Value)
Catch ex As Exception
_logger.Error(ex)
Throw ex
End Try
End Function
Public Function GetDocumentByContainerId(ContainerId As String) As DocumentResult ' Return oResult
Try ' Catch ex As Exception
Return _channel.GetDocumentByContainerId(ContainerId) ' _logger.Error(ex)
Catch ex As Exception ' Throw ex
_logger.Error(ex) ' End Try
Throw ex 'End Function
End Try
End Function 'Public Function NewFileIndex(DocObject As DocumentObject, Syskey As String, LanguageCode As String, Value As String) As IndexResult
' Try
' Dim oResult As IndexResult = _channel.NewFileIndex(DocObject, Syskey, LanguageCode, Value)
' Return oResult
' Catch ex As Exception
' _logger.Error(ex)
' Throw ex
' End Try
'End Function
'Public Function GetDocumentByDocumentId(DocumentId As Int64) As DocumentResult
' Try
' Return _channel.GetDocumentByDocumentId(DocumentId)
' Catch ex As Exception
' _logger.Error(ex)
' Throw ex
' End Try
'End Function
'Public Function GetDocumentByContainerId(ContainerId As String) As DocumentResult
' Try
' Return _channel.GetDocumentByContainerId(ContainerId)
' Catch ex As Exception
' _logger.Error(ex)
' Throw ex
' End Try
'End Function
''' <summary> ''' <summary>
''' Aborts the channel and creates a new connection ''' Aborts the channel and creates a new connection

4
Modules.EDMIAPI/app.config
View File

@ -26,7 +26,7 @@
<bindings> <bindings>
<netTcpBinding> <netTcpBinding>
<binding name="tcpBinding"> <binding name="tcpBinding">
<security mode="None"> <security>
<transport sslProtocols="None" /> <transport sslProtocols="None" />
</security> </security>
</binding> </binding>
@ -37,7 +37,7 @@
binding="netTcpBinding" bindingConfiguration="tcpBinding" binding="netTcpBinding" bindingConfiguration="tcpBinding"
contract="EDMIServiceReference.IEDMIService" name="tcpBinding"> contract="EDMIServiceReference.IEDMIService" name="tcpBinding">
<identity> <identity>
<dns value="localhost" /> <servicePrincipalName value="host/sDD-VMP03-VM09.dd-san01.dd-gan.local.digitaldata.works" />
</identity> </identity>
</endpoint> </endpoint>
</client> </client>

6
Service.EDMIService/App.config
View File

@ -31,13 +31,15 @@
</source> </source>
</sources> </sources>
<sharedListeners> <sharedListeners>
<add initializeData="C:\logs\TracingAndLogging-service.svclog" type="System.Diagnostics.XmlWriterTraceListener" name="xml"/> <add name="xml"
initializeData="C:\logs\TracingAndLogging-service.svclog"
type="System.Diagnostics.XmlWriterTraceListener" />
</sharedListeners> </sharedListeners>
<trace autoflush="true"/> <trace autoflush="true"/>
</system.diagnostics> </system.diagnostics>
<system.serviceModel> <system.serviceModel>
<diagnostics wmiProviderEnabled="true"> <diagnostics wmiProviderEnabled="true">
<messageLogging logEntireMessage="true" logMalformedMessages="true" logMessagesAtTransportLevel="true"/> <messageLogging logEntireMessage="false" logMalformedMessages="true" logMessagesAtTransportLevel="true"/>
<endToEndTracing propagateActivity="true" activityTracing="true" messageFlowTracing="true"/> <endToEndTracing propagateActivity="true" activityTracing="true" messageFlowTracing="true"/>
</diagnostics> </diagnostics>
<bindings> <bindings>