Microsoft Sentinel Parser Development: Handling variants

In many cases, events in an event stream include variants that require different parsing logic.

Take an example of Syslog events where it doesn’t follow any particular event scheme. Event messages or logs usually come in a string void of fields as well. Building a parser for a Syslog log source becomes quite complicated because of this shortcoming.

To handle such an issue Sentinel provide a solution in KQL, where variants can be parsed in one function by creating a separate function for each variant and using the union statement to combine the results.

let AzureFirewallNetworkRuleLogs = AzureDiagnostic
 | where Category == "AzureFirewallNetworkRule"
 | where isnotempty(msg_s);
let parseLogs = AzureFirewallNetworkRuleLogs
 | where msg_s has_any("TCP", "UDP")
 | parse-where
 msg_s with networkProtocol:string 
 " request from " srcIpAddr:string
 ":" srcPortNumber:int
 …
 | project-away msg_s;
let parseLogsWithUrls = AzureFirewallNetworkRuleLogs
 | where msg_s has_all ("Url:","ThreatIntel:")
 | parse-where
 msg_s with networkProtocol:string 
 " request from " srcIpAddr:string
 " to " dstIpAddr:string
 …
union parseLogs, parseLogsWithUrls…s        

Taking the above example from Microsoft into consideration we can build a single parser for three separate variants as shown below:

let SyslogAuthenticationSuccess = Syslo
 | where SyslogMessage contains 'accepted password'
 | parse SyslogMessage with Activity: string ' for ' TargetUserName ' from ' IpAddress ' port ' IpPort ' ' Protocol
 | extend EventVendor = 'Linux', EventProduct = 'Syslog', EventCount=int(1), EventSchemaVersion='0.1.0', EventResult = 'Success', EventStartTime = TimeGenerated, EventEndTime= TimeGenerated, EventType= 'Logon', SrcDvcId=tostring(Computer), SrcDvcHostname =tostring(HostName), SrcDvcOs=tostring(Computer)
 | project-rename EventOriginalUid =ProcessID, LogonMethod = ProcessName
 | project-reorder TimeGenerated, EventProduct, EventOriginalUid, EventResult, EventStartTime, EventEndTime, LogonMethod, SrcDvcId, SrcDvcHostname, SrcDvcOs;
let SyslogAuthenticationFailed = Syslog 
| where Facility has "authpriv" and SeverityLevel has "info" and SyslogMessage contains "Failed password for"
| parse SyslogMessage with Activity:string
" for " TargetUserName
" from " IPAddress
" port " IpPort
" " Protocol
| extend
EventVendor = 'Linux'
, EventProduct = 'Syslog'
, EventCount=int(1)
, EventSchemaVersion='0.1.0'
, EventResult = iff (Facility ==0, 'Success', 'Failure')
, EventOriginalResultDetails = coalesce(Facility, SeverityLevel)
, EventStartTime = TimeGenerated
, EventEndTime= TimeGenerated
, EventType= 'Logon'
, SrcDvcId=tostring(Computer)
, SrcDvcHostname =tostring(HostName)
, SrcDvcOs=tostring(SourceSystem)
, EventOriginalUid=tostring(ProcessID)
| project-rename
LogonMethod = ProcessName
| project-reorder
TimeGenerated
,EventProduct
, EventOriginalUid
, EventResult
, EventOriginalResultDetails
, EventStartTime
, EventEndTime
, LogonMethod
, SrcDvcId
, SrcDvcHostname
, SrcDvcOs 
| where TargetUserName !contains "invalid user";
let SyslogAuthenticationFailedwithInvalidUser = Syslog
| where SyslogMessage contains "failed password" and SeverityLevel == "info"
| parse SyslogMessage with Activity:string
 " for " TargetUserName
 " from " IpAddress
 " port " IpPort
 " " Protocol
| where TargetUserName contains "invalid user"
| extend tmp_Username = split(TargetUserName," ")
| extend TargetUserName = tostring(tmp_Username[2])
| extend 
 EventVendor = 'Linux'
 , EventProduct = 'Syslog'
 , EventCount=int(1)
 , EventSchemaVersion='0.1.0'
 , EventResult = iff (Facility ==0, 'Success', 'Failure')
 , EventStartTime = TimeGenerated
 , EventEndTime= TimeGenerated
 , EventType= 'Logon'
 , SrcDvcId=tostring(Computer)
 , SrcDvcHostname =tostring(HostName)
 , SrcDvcOs=tostring(Computer)
 | project-rename
 EventOriginalUid =ProcessID
 , LogonMethod = ProcessName
 | project-reorder
 TimeGenerated
 ,EventProduct
 , EventOriginalUid
 , EventResult
 , EventStartTime
 , EventEndTime
 , LogonMethod 
 , SrcDvcId
 , SrcDvcHostname 
 , SrcDvcOs;
union SyslogAuthenticationFailed, SyslogAuthenticationSuccess, SyslogAuthenticationFailedwithInvalidUserg        

The whole KQL function can be saved as?ASimSyslogAuthentication.

Furthermore, the saved function can then be called in the ASimAuthentication parser according to the standard method of custom parser inclusion:

let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimAuthentication') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser); 2let ASimAuthenticationDisabled=toscalar('ExcludeASimAuthentication' in (DisabledParsers) or 'Any' in (DisabledParsers)); 3union isfuzzy=true 4 vimAuthenticationEmpty 5 , ASimAuthenticationAADManagedIdentitySignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADManagedIdentitySignInLogs' in (DisabledParsers) )) 6 , ASimAuthenticationAADNonInteractiveUserSignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADNonInteractiveUserSignInLogs' in (DisabledParsers) )) 7 , ASimAuthenticationAADServicePrincipalSignInLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAADServicePrincipalSignInLogs' in (DisabledParsers) )) 8 , ASimAuthenticationSigninLogs (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationSigninLogs' in (DisabledParsers) )) 9 , ASimAuthenticationAWSCloudTrail (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationAWSCloudTrail' in (DisabledParsers) )) 10 , ASimAuthenticationOktaSSO (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationOktaSSO' in (DisabledParsers) )) 11 , ASimAuthenticationM365Defender (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationM365Defender' in (DisabledParsers) )) 12 , ASimAuthenticationMicrosoftWindowsEvent (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationMicrosoftWindowsEvent' in (DisabledParsers) )) 13 , ASimAuthenticationMD4IoT (ASimAuthenticationDisabled or ('ExcludeASimAuthenticationMD4IoT' in (DisabledParsers) )) 14 , ASimSyslogAuthentication (ASimSyslogAuthentication or ('ExcludeASimSyslogAuthentication' in (DisabledParsers) ))        

Conclusion

This method will reduce the function count in general and increase the efficiency of the platform as a whole.

Hope it helps the Microsoft Sentinel Community and SIEM Engineers out there!

Feel free to share your thoughts and feedback with us at: [email protected]