Revisiting splunk data pipeline ouroboros : How to make splunk heavy-forwarder reiterate over props.conf

Here is another take on my original article, published exactly one year ago.

This time the requirement was to process a consolidated data stream relayed by McAfee ESM in a CEF format.

As shown in the diagram below, Server A is currently indexing all the events into one index with CEFEVENTS sourcetype.

What I was requested to implement is duplicating the data stream to Server B, but this time break the data into the respective source type and do further parsing, merging and typing operations (see here) for every sourcetype separately - all using the single available heavy forwarder instance.

As always, I hope this was helpful and obviously feel free to drop any question in the comments.

Take care and stay safe?

inputs.conf

[tcp://2000]
index = main
sourcetype = cefevents

[tcp://3000]
index = secondary
sourcetype = siem

[tcp://3010]
index = secondary
sourcetype = cisco:asa

[tcp://3020]
index = secondary
sourcetype = wineventlog:security        

props.conf

[cefevents]
TRANSFORMS-duplicate_siem = send_loopback_siem
TRANSFORMS-duplicate_cisco_asa = send_loopback_cisco_asa
TRANSFORMS-duplicate_wineventlog_security = send_loopback_wineventlog_security

[source::tcp:3*]
TRANSFORMS-duplicate_to_secondary = send_to_secondary

[siem]

[cisco:asa]

[wineventlog:security]
        

transforms.conf

[send_loopback_siem]
REGEX = cat\=\/rule\/Fire
DEST_KEY = _TCP_ROUTING
FORMAT = primary_server,loopback_siem

[send_loopback_cisco_asa]
REGEX = CEF\:1\|CISCO\|ASA\|
DEST_KEY = _TCP_ROUTING
FORMAT = primary_server,loopback_cisco_asa

[send_loopback_wineventlog_security]
REGEX = CEF\:1\|Microsoft\|Microsoft Widnows\|\|Microsoft\-Windows\-Security\-Auditing\:
DEST_KEY = _TCP_ROUTING
FORMAT = primary_server,loopback_wineventlog_security

[send_to_secondary]
REGEX = .
DEST_KEY = _TCP_ROUTING
FORMAT = secondary_server
        

outputs.conf

[tcpout]
defaultgroup = primary_server

[tcpout:primary_server]
server = server_a:4000

[tcpout:secondary_server]
server = server_b:4000

[tcpout:loopback_siem]
server = 127.0.0.1:3000
sendCookedData = false

[tcpout:loopback_cisco_asa]
server = 127.0.0.1:3010
sendCookedData = false

[tcpout:loopback_wineventlog_security]
server = 127.0.0.1:3020
sendCookedData = false]