diff --git a/Base/Base.vbproj b/Base/Base.vbproj index ad3ecfe1..bb03a7e8 100644 --- a/Base/Base.vbproj +++ b/Base/Base.vbproj @@ -146,4 +146,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Base/FilesystemEx.vb b/Base/FilesystemEx.vb index a746d93f..cf7bf36c 100644 --- a/Base/FilesystemEx.vb +++ b/Base/FilesystemEx.vb @@ -177,7 +177,6 @@ Public Class FilesystemEx ''' somestring~3 --> somestring~3 --> ['somestring', '3'] --> version 3 ''' ''' The string to versioned - ''' The character to split at ''' Tuple of string and version Public Function GetVersionedString(pString As String) As Tuple(Of String, Integer) Dim oSplitString = pString.Split(VERSION_SEPARATOR).ToList() @@ -265,7 +264,7 @@ Public Class FilesystemEx _Logger.Warn("Deleting files was aborted at file {0}.", oFile.FullName) Return False End If - oUnableToDeleteCounter = oUnableToDeleteCounter + 1 + oUnableToDeleteCounter += 1 _Logger.Warn("File {0} could not be deleted!") End Try Next diff --git a/Base/ModuleExtensions.vb b/Base/ModuleExtensions.vb index 2c0a78aa..28dba280 100644 --- a/Base/ModuleExtensions.vb +++ b/Base/ModuleExtensions.vb @@ -65,6 +65,30 @@ Public Module ModuleExtensions Return (pString.Trim().ToLower() = "true") OrElse (pString.Trim() = "1") End Function + ''' + ''' Checks if a string is null or empty + ''' + ''' The input string + ''' True string is null or empty, otherwise false. + + Public Function IsNullOrEmpty(pString As String) As Boolean + Return String.IsNullOrEmpty(pString) + End Function + + ''' + ''' Checks if a string is NOT null or empty + ''' + ''' The input string + ''' True string is null or empty, otherwise false. + + Public Function IsNotNullOrEmpty(pString As String) As Boolean + If String.IsNullOrEmpty(pString) Then + Return False + Else + Return True + End If + End Function + ' ====================================================== ' === DICTIONARY ' ====================================================== diff --git a/Config/Config.vbproj b/Config/Config.vbproj index 3551eb08..a6a1d69b 100644 --- a/Config/Config.vbproj +++ b/Config/Config.vbproj @@ -119,9 +119,9 @@ - + {8a8f20fc-c46e-41ac-bee7-218366cfff99} - Encryption + NNEncryption {903b2d7d-3b80-4be9-8713-7447b704e1b0} @@ -129,4 +129,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Database/Database.vbproj b/Database/Database.vbproj index 89b6477f..995cb2d6 100644 --- a/Database/Database.vbproj +++ b/Database/Database.vbproj @@ -146,9 +146,9 @@ - + {8a8f20fc-c46e-41ac-bee7-218366cfff99} - Encryption + NNEncryption {903b2d7d-3b80-4be9-8713-7447b704e1b0} @@ -164,4 +164,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/EDMIAPI/DatabaseWithFallback.vb b/EDMIAPI/DatabaseWithFallback.vb index 0dbff6bb..552b8da8 100644 --- a/EDMIAPI/DatabaseWithFallback.vb +++ b/EDMIAPI/DatabaseWithFallback.vb @@ -3,7 +3,6 @@ Imports DigitalData.Modules.EDMI.API Imports DigitalData.Modules.EDMI.API.Constants Imports DigitalData.Modules.EDMI.API.EDMIServiceReference Imports DigitalData.Modules.Logging -Imports DigitalData.Modules.Language.Utils Public Class DatabaseWithFallback Private ReadOnly _Logger As Logger diff --git a/EDMIAPI/EDMI.API.vbproj b/EDMIAPI/EDMI.API.vbproj index 10dfec3b..262ecaaa 100644 --- a/EDMIAPI/EDMI.API.vbproj +++ b/EDMIAPI/EDMI.API.vbproj @@ -314,4 +314,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Encryption/Encryption.vbproj b/Encryption/NNEncryption.vbproj similarity index 100% rename from Encryption/Encryption.vbproj rename to Encryption/NNEncryption.vbproj diff --git a/Interfaces/ActiveDirectoryInterface.vb b/Interfaces/ActiveDirectoryInterface.vb index 5da2f55a..0c8a04c3 100644 --- a/Interfaces/ActiveDirectoryInterface.vb +++ b/Interfaces/ActiveDirectoryInterface.vb @@ -37,23 +37,22 @@ Public Class ActiveDirectoryInterface _logger.Info("Using RootPath {0}", _rootPath) End Sub - Public Function SyncUsersForGroup(GroupName As String, Firebird As Firebird, MSSQL As MSSQLServer) As List(Of ADUser) + Public Function SyncUsersForGroup(GroupName As String, MSSQL As MSSQLServer) As List(Of ADUser) Try - Return SyncUsersForGroup(GroupName, New List(Of AttributeMapping), Firebird, MSSQL) + Return SyncUsersForGroup(GroupName, New List(Of AttributeMapping), MSSQL) Catch ex As Exception _logger.Error(ex) Return Nothing End Try End Function - Public Function SyncUsersForGroup(GroupName As String, AttributeMappings As List(Of AttributeMapping), Firebird As Firebird, MSSQL As MSSQLServer, Optional Filter As String = DEFAULT_USER_FILTER) As List(Of ADUser) + Public Function SyncUsersForGroup(GroupName As String, AttributeMappings As List(Of AttributeMapping), MSSQL As MSSQLServer, Optional Filter As String = DEFAULT_USER_FILTER) As List(Of ADUser) Dim oUsers As New List(Of ADUser) Dim oSyncedUsers As New List(Of ADUser) Dim oGroupId As Int64 = Nothing - Dim oFirebirdSync As New SyncUsers.SyncUsersFirebird(_logConfig, Firebird) Dim oSQLSync As New SyncUsers.SyncUsersMSSQL(_logConfig, MSSQL) - Dim oSyncedUsersFirebird, oSyncedUsersMSSQL As List(Of ADUser) + Dim oSyncedUsersMSSQL As List(Of ADUser) Try _logger.Debug("Fetching users from ActiveDirectory") @@ -64,16 +63,6 @@ Public Class ActiveDirectoryInterface Return Nothing End Try - ' Do the actual sync into firebird - If Firebird IsNot Nothing Then - oSyncedUsersFirebird = oFirebirdSync.SyncUsers(GroupName, oUsers, AttributeMappings) - If oSyncedUsersFirebird.Count > 0 Then - _logger.Debug("Synced {0} users to Firebird", oSyncedUsersFirebird.Count) - End If - Else - _logger.Debug("SyncUsersForGroup: _firebird is nothing. ") - End If - ' Do the actual sync into MSSQL If MSSQL IsNot Nothing Then oSyncedUsersMSSQL = oSQLSync.SyncUsers(GroupName, oUsers, AttributeMappings) @@ -81,7 +70,7 @@ Public Class ActiveDirectoryInterface _logger.Debug("Synced {0} users to MSSQLServer", oSyncedUsersMSSQL.Count) End If Else - _logger.Debug("SyncUsersForGroup: _mssql is nothing. ") + _logger.Debug("SyncUsersForGroup: _mssql is nothing.") End If Return oUsers diff --git a/Interfaces/ActiveDirectoryInterface/SyncUsers.Firebird.vb b/Interfaces/ActiveDirectoryInterface/SyncUsers.Firebird.vb deleted file mode 100644 index 182e2718..00000000 --- a/Interfaces/ActiveDirectoryInterface/SyncUsers.Firebird.vb +++ /dev/null @@ -1,145 +0,0 @@ -Imports DigitalData.Modules.Database -Imports DigitalData.Modules.Interfaces -Imports DigitalData.Modules.Logging - -Namespace SyncUsers - Public Class SyncUsersFirebird - Implements ISyncUsers - - Private ReadOnly _logConfig As LogConfig - Private ReadOnly _logger As Logger - Private ReadOnly _firebird As Database.Firebird - - Public Sub New(LogConfig As LogConfig, Firebird As Database.Firebird) - _logConfig = LogConfig - _logger = LogConfig.GetLogger() - _firebird = Firebird - End Sub - - Public Function SyncUsers(GroupName As String, Users As List(Of ADUser), PropertyMapping As List(Of AttributeMapping)) As List(Of ADUser) Implements ISyncUsers.SyncUsers - Dim oGroupId As Integer - Dim oSyncedUsers As New List(Of ADUser) - - Try - _logger.Debug("Getting group Id for group [{0}]", GroupName) - oGroupId = GetGroupId(GroupName) - - If oGroupId = 0 Then - _logger.Debug("Group [{0}] does not exist in database or is not enabled for sync.", GroupName) - Return oSyncedUsers - End If - - _logger.Debug("Using group Id [{0}]", oGroupId) - Catch ex As Exception - _logger.Error(ex) - Return oSyncedUsers - End Try - - For Each oUser In Users - Dim oUserId As Int64 - Dim oUserExists As Boolean = False - - ' Check if user already exists - Try - _logger.Debug("Checking if user [{0}] exists", oUser) - oUserId = GetUserId(oUser.samAccountName) - oUserExists = Not IsNothing(oUserId) - _logger.Debug("User [{0}] exists in database: ", oUser, oUserExists) - Catch ex As Exception - _logger.Error(ex) - _logger.Warn("Could not get UserId for user. Skipping") - Continue For - End Try - - ' I user does not exist, create a new user - Try - If Not oUserExists Then - _logger.Debug("Creating new user for [{0}]", oUser) - oUserId = CreateUser(oUser) - _logger.Debug("User created with Id [{0}]", oUserId) - End If - Catch ex As Exception - _logger.Error(ex) - _logger.Warn("Could not create user. Skipping") - Continue For - End Try - - ' Add the user to group - Try - AddUserToGroup(oUserId, oGroupId) - Catch ex As Exception - _logger.Error(ex) - _logger.Warn("Could not add user to group. Skipping") - Continue For - End Try - - oSyncedUsers.Add(oUser) - Next - - Return oSyncedUsers - End Function - - Private Function AddUserToGroup(UserId As Integer, GroupId As Integer) As Boolean Implements ISyncUsers.AddUserToGroup - Try - Dim oSQL = $"SELECT FNICM_RADM_NEW_USER2GROUP({UserId}, {GroupId}, 'AD-Sync') from RDB$DATABASE" - Dim oRecordId = _firebird.GetScalarValue(oSQL) - - If IsDBNull(oRecordId) Then - _logger.Warn("UserId {0} - GroupId {1} relation already exists.", UserId, GroupId) - Return False - End If - - Return True - Catch ex As Exception - _logger.Error(ex) - Throw ex - End Try - End Function - Private Function GetGroupId(GroupName As String) As Integer Implements ISyncUsers.GetGroupId - Try - Dim oSQL As String = $"SELECT FNICM_GET_RECORD4SYSKEY('{GroupName}','002-NAME') from RDB$DATABASE" - Dim oGroupId = _firebird.GetScalarValue(oSQL) - - If IsDBNull(oGroupId) OrElse oGroupId = 0 Then - _logger.Debug("Group {0} not found in database", GroupName) - Return Nothing - End If - - Return oGroupId - Catch ex As Exception - _logger.Error(ex) - Throw ex - End Try - End Function - Private Function GetUserId(UserName As String) As Integer Implements ISyncUsers.GetUserId - Try - Dim oSQL As String = $"SELECT FNICM_GET_RECORD4SYSKEY('{UserName}','001-USRNAME') from RDB$DATABASE" - Dim oResult = _firebird.GetScalarValue(oSQL) - - If IsDBNull(oResult) Then - Return Nothing - End If - - Return oResult - Catch ex As Exception - _logger.Error(ex) - Throw ex - End Try - End Function - Private Function CreateUser(User As ADUser) As Integer Implements ISyncUsers.CreateUser - Try - Dim oSQL = $"SELECT FNICM_RADM_NEW_USER('{User?.GivenName}', '{User?.Surname}', '{User?.samAccountName}', 'AD-Sync') from RDB$DATABASE" - Dim oUserId As Integer = _firebird.GetScalarValue(oSQL) - - Return oUserId - Catch ex As Exception - _logger.Error(ex) - Throw ex - End Try - End Function - - Public Sub AddCustomAttributesToUser(User As ADUser, UserId As Integer) Implements ISyncUsers.AddCustomAttributesToUser - Throw New NotImplementedException() - End Sub - End Class -End Namespace diff --git a/Interfaces/Interfaces.vbproj b/Interfaces/Interfaces.vbproj index e8652081..d26f6698 100644 --- a/Interfaces/Interfaces.vbproj +++ b/Interfaces/Interfaces.vbproj @@ -86,7 +86,6 @@ - @@ -170,4 +169,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Interfaces/ZUGFeRDInterface/PDFEmbeds.vb b/Interfaces/ZUGFeRDInterface/PDFEmbeds.vb index f11deaf7..00b41c33 100644 --- a/Interfaces/ZUGFeRDInterface/PDFEmbeds.vb +++ b/Interfaces/ZUGFeRDInterface/PDFEmbeds.vb @@ -149,6 +149,7 @@ Public Class PDFEmbeds GDPicturePDF.DeleteEmbeddedFile(0) End While + Return True End Function Private Function DoExtract(GDPicturePDF As GdPicturePDF, pExtensions As List(Of String)) As List(Of EmbeddedFile) diff --git a/Jobs/ADSync/ADSyncJob.vb b/Jobs/ADSync/ADSyncJob.vb index 5598f275..145787e3 100644 --- a/Jobs/ADSync/ADSyncJob.vb +++ b/Jobs/ADSync/ADSyncJob.vb @@ -8,8 +8,8 @@ Public Class ADSyncJob Inherits JobBase Implements IJob(Of ADSyncArgs) - Public Sub New(LogConfig As LogConfig, Firebird As Firebird, MSSQL As MSSQLServer) - MyBase.New(LogConfig, Firebird, MSSQL) + Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer) + MyBase.New(LogConfig, MSSQL) End Sub Public Sub Start(Arguments As ADSyncArgs) Implements IJob(Of ADSyncArgs).Start @@ -31,7 +31,7 @@ Public Class ADSyncJob For Each oGroup In oGroups _Logger.Debug("Syncing Group [{0}]", oGroup) - Dim oSyncedUsers = oSync.SyncUsersForGroup(oGroup, oAttributeMappings, _Firebird, _MSSQL, Arguments.UserFilter) + Dim oSyncedUsers = oSync.SyncUsersForGroup(oGroup, oAttributeMappings, _MSSQL, Arguments.UserFilter) If oSyncedUsers Is Nothing Then _Logger.Warn("Group [{0}] could not be synced!", oGroup) diff --git a/Jobs/GraphQL/GraphQLJob.vb b/Jobs/GraphQL/GraphQLJob.vb index 60fd33c4..6bc83c49 100644 --- a/Jobs/GraphQL/GraphQLJob.vb +++ b/Jobs/GraphQL/GraphQLJob.vb @@ -1,28 +1,24 @@ Option Explicit On - -Imports System.Collections.Generic -Imports System.Data Imports System.IO -Imports System.Linq -Imports DigitalData.Modules.Base Imports DigitalData.Modules.Config Imports DigitalData.Modules.Database Imports DigitalData.Modules.Interfaces Imports DigitalData.Modules.Jobs.GraphQL Imports DigitalData.Modules.Logging -Imports Newtonsoft.Json.Linq Public Class GraphQLJob Inherits JobBase Implements IJob(Of GraphQLArgs) Private _GraphQL As GraphQLInterface = Nothing + Private _Model As GraphQLModel + Private _Writer As GraphQLWriter Private Const PLACEHOLDER_STATIC = "STATIC:" Private Const JOB_NAME = "GraphQL Job" Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer) - MyBase.New(LogConfig, Nothing, MSSQL) + MyBase.New(LogConfig, MSSQL) End Sub Public Sub Start(Args As GraphQLArgs) Implements IJob(Of GraphQLArgs).Start @@ -34,6 +30,9 @@ Public Class GraphQLJob _GraphQL = New GraphQLInterface(_LogConfig, .BaseUrl, .Email, .Password, .CertificateFingerprint) End With + _Model = New GraphQLModel(_LogConfig, _MSSQL) + _Writer = New GraphQLWriter(_LogConfig, _MSSQL) + ' Login to get cookie _Logger.Debug("Logging in") Dim oLoginResponse = _GraphQL.Login() @@ -43,41 +42,7 @@ Public Class GraphQLJob _Logger.Debug("Loading Queries") - ' Load query data from TBCUST_JOBRUNNER_QUERY - Dim oQueryTable As DataTable = _MSSQL.GetDatatable("SELECT * FROM TBCUST_JOBRUNNER_QUERY ORDER BY SEQUENCE") - Dim oQueryList As New List(Of Query) - - ' Save query data to business objects - For Each oRow As DataRow In oQueryTable.Rows - Dim oQuery As New Query With { - .Id = oRow.Item("GUID"), - .Name = oRow.Item("TITLE"), - .ClearBeforeFill = oRow.ItemEx("CLEAR_BEFORE_FILL", False), - .ConnectionId = oRow.ItemEx("CON_ID", 1), ' TODO: Connection String? - .DestinationTable = oRow.ItemEx("DESTINATION_TABLE", String.Empty), - .OperationName = oRow.ItemEx("OPERATION_NAME", String.Empty), - .MappingBasePath = oRow.ItemEx("MAPPING_BASE_PATH", String.Empty), - .QueryString = oRow.ItemEx("QUERY_STRING", String.Empty) - } - - If oQuery.DestinationTable = String.Empty Then - _Logger.Warn("Value [DestinationTable] could not be read. Configuration incomplete.") - End If - - If oQuery.OperationName = String.Empty Then - _Logger.Warn("Value [OperationName] could not be read. Configuration incomplete.") - End If - - If oQuery.MappingBasePath = String.Empty Then - _Logger.Warn("Value [MappingBasePath] could not be read. Configuration incomplete.") - End If - - If oQuery.QueryString = String.Empty Then - _Logger.Warn("Value [QueryString] could not be read. Configuration incomplete.") - End If - - oQueryList.Add(oQuery) - Next + Dim oQueryList = _Model.GetQueryList() _Logger.Debug("Running [{0}] queries.", oQueryList.Count) @@ -119,13 +84,17 @@ Public Class GraphQLJob ' Clear Table before inserting If pQuery.ClearBeforeFill = True Then - If DeleteWithQueryName(pQuery) = False Then + If DeleteHistoryTable(pQuery) = False Then Throw New ApplicationException($"Error while dropping history table before fill for Query [{pQuery.Name}]") End If If CreateHistoryTable(pQuery) = False Then Throw New ApplicationException($"Error while creating history table before fill for Query [{pQuery.Name}]") End If + + If TruncateTable(pQuery) = False Then + Throw New ApplicationException($"Error while truncating table before fill for Query [{pQuery.Name}]") + End If End If ' Reset all records to status = 0 @@ -137,31 +106,22 @@ Public Class GraphQLJob End If ' get the data from GraphQL - _Logger.Info("Getting data..", pQuery.Name) + _Logger.Info("Getting data from GraphQL..", pQuery.Name) Dim oDataResponse = _GraphQL.GetData(pQuery.QueryString, pQuery.OperationName) - Dim oResult As String + Dim oJsonResult As String ' write data to string Using oStream = oDataResponse.GetResponseStream() Using oReader As New StreamReader(oStream) - oResult = oReader.ReadToEnd() + oJsonResult = oReader.ReadToEnd() End Using End Using - ' Fill the query object with field mapping data from TBCUST_JOBRUNNER_QUERY_MAPPING - Dim oSQL As String = "SELECT t2.* FROM TBCUST_JOBRUNNER_QUERY_MAPPING t - JOIN TBCUST_JOBRUNNER_MAPPING t2 ON t.MAPPING_ID = t2.GUID - WHERE t.QUERY_ID = {0}" - Dim oMappingTable As DataTable = _MSSQL.GetDatatable(String.Format(oSQL, pQuery.Id)) - For Each oMapping As DataRow In oMappingTable.Rows - pQuery.MappingFields.Add(New GraphQL.FieldMapping With { - .DestinationColumn = oMapping.Item("DestinationColumn"), - .SourcePath = oMapping.Item("SourcePath") - }) - Next + _Logger.Debug("Writing JSON data to database..") ' Handle the response from GraphQL and insert Data - Dim oWriteDataResult As GraphQL.Query = WriteNewQueryData(oResult, pQuery, oDatabase) + 'Dim oWriteDataResult As Query = WriteNewQueryData(oResult, pQuery, oDatabase) + Dim oWriteDataResult As Query = _Writer.WriteNewQueryData(oJsonResult, pQuery, JOB_NAME) If IsNothing(oWriteDataResult) Then Throw New ApplicationException($"Error while handling Result of Query [{pQuery.Name}]") @@ -200,7 +160,7 @@ Public Class GraphQLJob End Try End Function - Private Function DeleteWithQueryName(pQuery As GraphQL.Query) As Boolean + Private Function DeleteHistoryTable(pQuery As Query) As Boolean Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY" Dim oDeleteHistorySQL = $" IF (EXISTS (SELECT * @@ -214,6 +174,12 @@ Public Class GraphQLJob Return _MSSQL.ExecuteNonQuery(oDeleteHistorySQL) End Function + Private Function TruncateTable(pQuery As Query) + Dim oDeleteTableSQL = $"TRUNCATE TABLE {pQuery.DestinationTable}" + + Return _MSSQL.ExecuteNonQuery(oDeleteTableSQL) + End Function + Private Function CreateHistoryTable(pQuery As Query) As Boolean Dim oHistoryTableName = $"{pQuery.DestinationTable}_HISTORY" Dim oFillHistorySQL = $"SELECT * INTO {oHistoryTableName} FROM {pQuery.DestinationTable}" @@ -231,72 +197,6 @@ Public Class GraphQLJob Return _MSSQL.ExecuteNonQuery(oResetSQL) End Function - Private Function WriteNewQueryData(JsonString As String, QueryData As GraphQL.Query, DB As Database.MSSQLServer) As GraphQL.Query - Dim oObj As JObject = JObject.Parse(JsonString) - Dim oResultList As JToken - - If _GraphQL.ReadJSONPathFragmented(oObj, QueryData.MappingBasePath) = False Then - _Logger.Warn("There is an error in the MappingBasePath [{1}] configuration of query [{0}]", QueryData.Name, QueryData.MappingBasePath) - End If - - Try - oResultList = oObj.SelectToken(QueryData.MappingBasePath, errorWhenNoMatch:=True) - Catch ex As Exception - _Logger.Warn("WriteNewQueryData: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name) - _Logger.Error(ex) - Return Nothing - End Try - - If oResultList Is Nothing Then - _Logger.Warn("WriteNewQueryData: Could not find BasePath: [{0}] for query [{1}]", QueryData.MappingBasePath, QueryData.Name) - Return Nothing - End If - - _Logger.Info("WriteNewQueryData: Processing Queue [{0}] with [{1}] Items", QueryData.Name, oResultList.Count) - - For Each oResultItem As JToken In oResultList - Try - ' ADDED_WHO, ADDED_QUERY_ID are system fields which are used to correctly fill - ' and delete rows in the destination table without touching rows from other queries - Dim oKeys As New List(Of String) From {"ADDED_WHO", "ADDED_QUERY_ID", "STATUS"} - Dim oValues As New List(Of String) From {JOB_NAME, QueryData.Id, "1"} - - For Each oMapping In QueryData.MappingFields - Dim oValue As String = String.Empty - - If oMapping.SourcePath.StartsWith(PLACEHOLDER_STATIC) Then - oValue = oMapping.SourcePath.Replace(PLACEHOLDER_STATIC, String.Empty) - Else - Dim oToken = oResultItem.SelectToken(oMapping.SourcePath) - - If oToken Is Nothing Then - _Logger.Warn("WriteNewQueryData: Could not find value at SourcePath: {0}", oMapping.SourcePath) - oValue = String.Empty - Else - oValue = oToken.ToString - End If - End If - - oValues.Add(oValue) - oKeys.Add(oMapping.DestinationColumn) - Next - - Dim oColumnString = String.Join(",", oKeys.ToArray) - - Dim oValueList = oValues.Select(Function(Value) $"'{Value.EscapeForSQL}'").ToList() - Dim oValueString = String.Join(",", oValueList) - - Dim oSQL As String = $"INSERT INTO {QueryData.DestinationTable} ({oColumnString}) VALUES ({oValueString})" - - DB.ExecuteNonQuery(oSQL) - Catch ex As Exception - _Logger.Error(ex) - End Try - Next - - Return QueryData - End Function - Public Function ShouldStart(Arguments As GraphQLArgs) As Boolean Implements IJob(Of GraphQLArgs).ShouldStart Return Arguments.Enabled End Function diff --git a/Jobs/GraphQL/GraphQLModel.vb b/Jobs/GraphQL/GraphQLModel.vb new file mode 100644 index 00000000..b79134aa --- /dev/null +++ b/Jobs/GraphQL/GraphQLModel.vb @@ -0,0 +1,86 @@ +Imports System.Collections.Generic +Imports System.Data +Imports DigitalData.Modules.Base +Imports DigitalData.Modules.Database +Imports DigitalData.Modules.Jobs.GraphQL +Imports DigitalData.Modules.Logging + +Public Class GraphQLModel + Private Database As MSSQLServer + Private LogConfig As LogConfig + Private Logger As Logger + + Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer) + Database = pDatabase + LogConfig = pLogConfig + Logger = pLogConfig.GetLogger() + End Sub + + Public Function GetQueryList() As List(Of Query) + Try + Dim oQueryTable As DataTable = Database.GetDatatable("SELECT * FROM TBCUST_JOBRUNNER_QUERY WHERE ACTIVE = 1 ORDER BY SEQUENCE") + Dim oQueryList As New List(Of Query) + + For Each oRow As DataRow In oQueryTable.Rows + Dim oQuery As New Query With { + .Id = oRow.Item("GUID"), + .Name = oRow.Item("TITLE"), + .ClearBeforeFill = oRow.ItemEx("CLEAR_BEFORE_FILL", False), + .ConnectionId = oRow.ItemEx("CON_ID", 1), ' TODO: Connection String? + .DestinationTable = oRow.ItemEx("DESTINATION_TABLE", String.Empty), + .OperationName = oRow.ItemEx("OPERATION_NAME", String.Empty), + .MappingBasePath = oRow.ItemEx("MAPPING_BASE_PATH", String.Empty), + .QueryString = oRow.ItemEx("QUERY_STRING", String.Empty) + } + + If oQuery.DestinationTable = String.Empty Then + Logger.Warn("Value [DestinationTable] could not be read. Configuration incomplete.") + End If + + If oQuery.OperationName = String.Empty Then + Logger.Warn("Value [OperationName] could not be read. Configuration incomplete.") + End If + + If oQuery.MappingBasePath = String.Empty Then + Logger.Warn("Value [MappingBasePath] could not be read. Configuration incomplete.") + End If + + If oQuery.QueryString = String.Empty Then + Logger.Warn("Value [QueryString] could not be read. Configuration incomplete.") + End If + + oQuery.MappingFields = GetQueryMapping(oQuery.Id) + + oQueryList.Add(oQuery) + Next + + Return oQueryList + Catch ex As Exception + Logger.Error(ex) + Return New List(Of Query) + End Try + End Function + + Public Function GetQueryMapping(pQueryId As Integer) As List(Of FieldMapping) + Try + Dim oSQL As String = "SELECT t2.* FROM TBCUST_JOBRUNNER_QUERY_MAPPING t + JOIN TBCUST_JOBRUNNER_MAPPING t2 ON t.MAPPING_ID = t2.GUID + WHERE t.QUERY_ID = {0}" + Dim oMappingTable As DataTable = Database.GetDatatable(String.Format(oSQL, pQueryId)) + Dim oMappings As New List(Of FieldMapping) + + For Each oMapping As DataRow In oMappingTable.Rows + oMappings.Add(New FieldMapping With { + .DestinationColumn = oMapping.Item("DestinationColumn"), + .SourcePath = oMapping.Item("SourcePath") + }) + Next + + Return oMappings + Catch ex As Exception + Logger.Error(ex) + Return New List(Of FieldMapping) + End Try + End Function + +End Class diff --git a/Jobs/GraphQL/GraphQLWriter.vb b/Jobs/GraphQL/GraphQLWriter.vb new file mode 100644 index 00000000..20b46a97 --- /dev/null +++ b/Jobs/GraphQL/GraphQLWriter.vb @@ -0,0 +1,178 @@ +Imports System +Imports System.Collections.Generic +Imports System.Data +Imports System.Data.SqlClient +Imports System.Linq +Imports DigitalData.Modules.Database +Imports DigitalData.Modules.Jobs.GraphQL +Imports DigitalData.Modules.Logging +Imports Newtonsoft.Json.Linq + +Public Class GraphQLWriter + Private ReadOnly Database As MSSQLServer + Private ReadOnly LogConfig As LogConfig + Private ReadOnly Logger As Logger + + Private Const PLACEHOLDER_STATIC = "STATIC:" + + Public Sub New(pLogConfig As LogConfig, pDatabase As MSSQLServer) + Database = pDatabase + LogConfig = pLogConfig + Logger = pLogConfig.GetLogger() + End Sub + + Public Function WriteNewQueryData(pJsonString As String, pQueryData As Query, pJobName As String) As GraphQL.Query + Try + Logger.Debug("Parsing JSON...") + + Dim oObj As JObject = JObject.Parse(pJsonString) + Dim oResultList As JToken + + If ValidateJSONPath(oObj, pQueryData.MappingBasePath) = False Then + Logger.Warn("There is an error in the MappingBasePath [{1}] configuration of query [{0}]", pQueryData.Name, pQueryData.MappingBasePath) + End If + + Try + oResultList = oObj.SelectToken(pQueryData.MappingBasePath, errorWhenNoMatch:=True) + Catch ex As Exception + Logger.Warn("Could not find BasePath: [{0}] for query [{1}]", pQueryData.MappingBasePath, pQueryData.Name) + Logger.Error(ex) + Return Nothing + End Try + + If oResultList Is Nothing Then + Logger.Warn("Could not find BasePath: [{0}] for query [{1}]", pQueryData.MappingBasePath, pQueryData.Name) + Return Nothing + End If + + Logger.Info("Processing Query [{0}] with [{1}] Items", pQueryData.Name, oResultList.Count) + + Dim oTable As New DataTable + Dim oColumnList = pQueryData.MappingFields.Select(Function(f) f.DestinationColumn).ToList() + + For Each oColumnName In oColumnList + oTable.Columns.Add(oColumnName) + Next + + oTable.Columns.Add("ADDED_WHO") + oTable.Columns.Add("ADDED_QUERY_ID") + oTable.Columns.Add("STATUS") + + Logger.Debug("Creating DataTable..") + + For Each oResultItem As JToken In oResultList + Dim oRow = FillRowFromJson(pQueryData, oResultItem, pJobName, oTable) + + If oRow Is Nothing Then + Logger.Error("DataRow could not be created!") + Continue For + End If + + oTable.Rows.Add(oRow) + Next + oTable.AcceptChanges() + + Logger.Debug("Starting Bulk Insert..") + + 'Bulk insert + Dim oBulkResult = BulkInsert(oTable, pQueryData.DestinationTable, oColumnList) + + If oBulkResult = False Then + Logger.Error("Bulk Insert for Query [{0}] failed!", pQueryData.Name) + End If + + Logger.Info("Bulk Insert finished. [{0}] rows inserted.", oTable.Rows.Count) + + Return pQueryData + + Catch ex As Exception + Logger.Error(ex) + Return Nothing + + End Try + End Function + + Private Function FillRowFromJson(pQueryData As Query, pToken As JToken, pJobName As String, pTable As DataTable) As DataRow + Try + Dim oValuesNew As New Dictionary(Of String, String) + Dim oRow As DataRow = pTable.NewRow() + + For Each oMapping In pQueryData.MappingFields + Dim oValue As String = String.Empty + + If oMapping.SourcePath.StartsWith(PLACEHOLDER_STATIC) Then + oValue = oMapping.SourcePath.Replace(PLACEHOLDER_STATIC, String.Empty) + Else + Dim oToken = pToken.SelectToken(oMapping.SourcePath) + + If oToken Is Nothing Then + Logger.Warn("WriteNewQueryData: Could not find value at SourcePath: {0}", oMapping.SourcePath) + oValue = String.Empty + Else + oValue = oToken.ToString + End If + End If + + oValuesNew.Add(oMapping.DestinationColumn, oValue) + Next + + oValuesNew.Add("ADDED_WHO", pJobName) + oValuesNew.Add("ADDED_QUERY_ID", pQueryData.Id) + oValuesNew.Add("STATUS", "1") + + For Each oColumn As DataColumn In pTable.Columns + oRow.Item(oColumn.ColumnName) = oValuesNew.Item(oColumn.ColumnName) + Next + + Return oRow + Catch ex As Exception + Logger.Error(ex) + Return Nothing + End Try + End Function + + Private Function BulkInsert(pTable As DataTable, pDestinationTable As String, pColumns As List(Of String)) As Boolean + Using oConnection = Database.GetConnection() + Using oBulkCopy = New SqlBulkCopy(oConnection) + + oBulkCopy.DestinationTableName = pDestinationTable + For Each oColumn In pColumns + oBulkCopy.ColumnMappings.Add(New SqlBulkCopyColumnMapping(oColumn, oColumn)) + Next + + Try + oBulkCopy.WriteToServer(pTable) + Catch ex As Exception + Logger.Error(ex) + Return False + End Try + End Using + End Using + + Return True + End Function + + Public Function ValidateJSONPath(pObject As Newtonsoft.Json.Linq.JObject, pJsonPath As String) As Boolean + Dim oSplitPath As List(Of String) = pJsonPath.Split(".").ToList() + Dim oCurrentPath As String = String.Empty + + For Each oPart In oSplitPath + If oCurrentPath = String.Empty Then + oCurrentPath = oPart + Else + oCurrentPath &= "." & oPart + End If + + Logger.Debug("Selecting Path Fragment [{0}]", oCurrentPath) + + Try + pObject.SelectToken(oCurrentPath, errorWhenNoMatch:=True) + Catch ex As Exception + Logger.Warn("Path Fragment [{0}] did not return a valid token", oCurrentPath) + Return False + End Try + Next + + Return True + End Function +End Class diff --git a/Jobs/JobBase.vb b/Jobs/JobBase.vb index 0f02b631..8dd4668e 100644 --- a/Jobs/JobBase.vb +++ b/Jobs/JobBase.vb @@ -4,13 +4,11 @@ Imports DigitalData.Modules.Logging Public Class JobBase Protected _LogConfig As LogConfig Protected _Logger As Logger - Protected _Firebird As Firebird Protected _MSSQL As MSSQLServer - Public Sub New(LogConfig As LogConfig, Firebird As Firebird, MSSQL As MSSQLServer) + Public Sub New(LogConfig As LogConfig, MSSQL As MSSQLServer) _LogConfig = LogConfig _Logger = LogConfig.GetLogger() - _Firebird = Firebird _MSSQL = MSSQL End Sub End Class diff --git a/Jobs/JobConfig.vb b/Jobs/JobConfig.vb index 99bc851f..19ef04d9 100644 --- a/Jobs/JobConfig.vb +++ b/Jobs/JobConfig.vb @@ -4,6 +4,8 @@ Public Class JobConfig Public Property Name As JobType Public Property Enabled As Boolean = False Public Property StartWithoutDelay As Boolean = False + + ' https://www.quartz-scheduler.net/documentation/quartz-3.x/how-tos/crontrigger.html Public Property CronSchedule As String = "" Public Property ArgsString As String = "" diff --git a/Jobs/Jobs.vbproj b/Jobs/Jobs.vbproj index 30ed392e..3b82f5ae 100644 --- a/Jobs/Jobs.vbproj +++ b/Jobs/Jobs.vbproj @@ -92,7 +92,9 @@ + + @@ -136,4 +138,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Jobs/My Project/AssemblyInfo.vb b/Jobs/My Project/AssemblyInfo.vb index f1b560ef..5872f1a6 100644 --- a/Jobs/My Project/AssemblyInfo.vb +++ b/Jobs/My Project/AssemblyInfo.vb @@ -13,7 +13,7 @@ Imports System.Runtime.InteropServices - + @@ -30,5 +30,5 @@ Imports System.Runtime.InteropServices ' Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern ' übernehmen, indem Sie "*" eingeben: - - + + diff --git a/Jobs/ZUGFeRD/EmailStrings.vb b/Jobs/ZUGFeRD/EmailStrings.vb index b25f5b98..f5daa1bd 100644 --- a/Jobs/ZUGFeRD/EmailStrings.vb +++ b/Jobs/ZUGFeRD/EmailStrings.vb @@ -3,7 +3,7 @@ Public Const EMAIL_WRAPPING_TEXT = "Sehr geehrte Damen und Herren,

das @NAME_ZUGFERD_PORTAL zur Verarbeitung der Eingangsrechnungen im ZUGFeRD-Format konnte die von Ihnen gesandte Rechnung leider nicht verarbeiten!

Grund: {0}

Bitte prüfen Sie die Datei und nehmen Sie bei Bedarf mit uns Kontakt auf.

- Vielen Dank für Ihr Verständnis.
Mit freundlichen Grüßen
Ihre IT-Abteilung" + Vielen Dank für Ihr Verständnis.
Mit freundlichen Grüßen
Ihre WISAG" Public Const EMAIL_SUBJECT_TEXT = "

Der Betreff der Original-Email war: {0}

" Public Const EMAIL_SUBJECT_REJECTED = "@NAME_ZUGFERD_PORTAL: Beleg abgelehnt" @@ -37,8 +37,9 @@ Public Const EMAIL_INVALID_DOCUMENT = "

Ihre Email ({0}) enthielt ein ZUGFeRD Dokument, welches aber inkorrekt formatiert wurde.

-

Mögliche Gründe für ein inkorrektes Format:

    -
  • Betrags-Werte weisen ungültiges Format auf (z.B. 25,01 anstatt 25.01)
    • +

    Mögliche Gründe für ein inkorrektes Format sind:

      +
    • Betragswerte weisen ein ungültiges Format auf (z.B. 25,01 anstatt 25.01)
      • +
    • Eins der folgenden Zeichen wird nicht XML-codiert verwendet: &, <, > oder ".

    " diff --git a/License/License.vbproj b/License/License.vbproj index 0b95ba02..6c5e1b33 100644 --- a/License/License.vbproj +++ b/License/License.vbproj @@ -128,4 +128,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Logging/Logging.vbproj b/Logging/Logging.vbproj index 72d00292..b818b62b 100644 --- a/Logging/Logging.vbproj +++ b/Logging/Logging.vbproj @@ -116,4 +116,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Messaging/Messaging.vbproj b/Messaging/Messaging.vbproj index 63fd34b6..9a86015f 100644 --- a/Messaging/Messaging.vbproj +++ b/Messaging/Messaging.vbproj @@ -151,4 +151,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Modules.sln b/Modules.sln index 470454c8..42af024d 100644 --- a/Modules.sln +++ b/Modules.sln @@ -11,7 +11,7 @@ Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "Database", "Database\Databa EndProject Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "EDMI.API", "EDMIAPI\EDMI.API.vbproj", "{25017513-0D97-49D3-98D7-BA76D9B251B0}" EndProject -Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "Encryption", "Encryption\Encryption.vbproj", "{8A8F20FC-C46E-41AC-BEE7-218366CFFF99}" +Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "NNEncryption", "Encryption\NNEncryption.vbproj", "{8A8F20FC-C46E-41AC-BEE7-218366CFFF99}" EndProject Project("{F184B08F-C81C-45F6-A57F-5ABD9991F28F}") = "Interfaces", "Interfaces\Interfaces.vbproj", "{AB6F09BF-E794-4F6A-94BB-C97C0BA84D64}" EndProject diff --git a/Patterns/Patterns.vbproj b/Patterns/Patterns.vbproj index 0fe15dfc..2da07c8c 100644 --- a/Patterns/Patterns.vbproj +++ b/Patterns/Patterns.vbproj @@ -153,4 +153,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..95fcf8b2 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Modules + +Diese Projektmappe enthält die Grundfunktionen für alle Produkte in Form von Modulen. + +## Deploy Skript + +Das Skript `copy-binary.ps1` liegt im Hauptverzeichnis und kann über den folgenden Befehl aufgerufen werden. Der Befehl wird pro Projekt in den Projekteinstellungen unter `Kompilieren > Buildereignisse > Postbuildereignis` eingetragen und sorgt dafür, dass die Assembly des Projekts in ein Deploy Verzeichnis kopiert wird. + +```cmd +powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' }" +``` diff --git a/Windows/Windows.vbproj b/Windows/Windows.vbproj index bf125735..8d73efe6 100644 --- a/Windows/Windows.vbproj +++ b/Windows/Windows.vbproj @@ -173,4 +173,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/Windream/Windream.vbproj b/Windream/Windream.vbproj index fbcf6136..6f9ecb0e 100644 --- a/Windream/Windream.vbproj +++ b/Windream/Windream.vbproj @@ -154,4 +154,11 @@ + + + + + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/ZooFlow/ZooFlow.vbproj b/ZooFlow/ZooFlow.vbproj index b10f3f7e..76ca9968 100644 --- a/ZooFlow/ZooFlow.vbproj +++ b/ZooFlow/ZooFlow.vbproj @@ -142,4 +142,7 @@ + + powershell.exe -command "& { &'$(SolutionDir)copy-binary.ps1' '$(TargetPath)' '$(TargetFileName)' '$(ConfigurationName)' '$(ProjectName)' }" + \ No newline at end of file diff --git a/copy-binary.ps1 b/copy-binary.ps1 new file mode 100644 index 00000000..317ca9b5 --- /dev/null +++ b/copy-binary.ps1 @@ -0,0 +1,29 @@ +param( + [String]$SourcePath, + [String]$FileName, + [String]$Configuration, + [String]$ProjectName +) + +$ArchiveFolderName = "Archiv" +$DestinationPath = "P:\Install .Net\0 DD - Bibliotheken\Modules\$ProjectName\" +$DestinationFilePath = [IO.Path]::Combine($DestinationPath, $FileName) +$ArchiveBasePath = [IO.Path]::Combine($DestinationPath, $ArchiveFolderName) + +if ($Configuration -eq "Release") { + $DateFolderName = $((Get-Date).ToString('yyyy-MM-dd_hh-mm')) + $ArchivePath = [IO.Path]::Combine($ArchiveBasePath, $DateFolderName) + + if (!(Test-Path -Path $ArchiveBasePath)) { + New-Item -Path $ArchiveBasePath -Type Directory + } + + if (Test-Path -Path $DestinationFilePath) { + if (!(Test-Path -Path $ArchivePath)) { + New-Item -Path $ArchivePath -Type Directory + } + Copy-Item $DestinationFilePath -Destination $ArchivePath + } + + Copy-Item $SourcePath -Destination $DestinationFilePath +}