Here is a dashboard I built to help you understand the activity of services and MSI installs within a Windows machine. This dashboard utilizes Post Processing so there is only 2 searches that are launched when the dashboard is loaded to minimize impact on search queuing.
Add-on’s: Splunk Add-on for Microsoft Windows – https://splunkbase.splunk.com/app/742/
Video overview of the dashboard and how to create this dashboard in your environment: https://youtu.be/WFKcYfaAzAY
<form theme="dark" version="1.0">
<label>Windows service activity & MSI installs</label>
<search id="service_base1">
<query>index=wineventlog $filter1$ SourceName="*Service Control Manager" LogName=System | fillnull value=* Service_Name, Service_File_Name, Service_Type, Service_Start_Type, Service_Account| stats count by _time, host, Service_Name, Service_File_Name, Service_Type, Service_Start_Type, Service_Account, Message, EventCode</query>
<earliest>$time1.earliest$</earliest>
<latest>$time1.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<fieldset submitButton="true" autoRun="false">
<input type="time" token="time1">
<label></label>
<default>
<earliest>-24h@h</earliest>
<latest>now</latest>
</default>
</input>
<input type="text" token="filter1">
<label>Filter:</label>
<default>*</default>
</input>
</fieldset>
<row>
<panel>
<title>Service was installed</title>
<table>
<search base="service_base1">
<query>search EventCode=7045 OR EventCode=7035 Service_Name!=tenable_mw_scan | eval Message=split(Message,".") | eval Short_Message=mvindex(Message,0) | table _time, host, Service_Name, Service_File_Name, Service_Type, Service_Start_Type, Service_Account, Short_Message</query>
</search>
<option name="count">10</option>
<option name="drilldown">none</option>
</table>
</panel>
</row>
<row>
<panel>
<title>Service entered Running/Stopped state</title>
<table>
<search base="service_base1">
<query>search EventCode=7036 | stats sum(count) as count by Message</query>
</search>
<option name="count">10</option>
<option name="drilldown">cell</option>
<drilldown>
<set token="term1">$click.value$</set>
</drilldown>
</table>
<table>
<title>Looking at "$term1$"</title>
<search>
<query>index=wineventlog $term1$ SourceName="*Service Control Manager" EventCode=7036 | stats count by _time,host</query>
<earliest>$time1.earliest$</earliest>
<latest>$time1.latest$</latest>
</search>
<option name="count">10</option>
<option name="drilldown">none</option>
</table>
</panel>
<panel>
<title>Service state changes</title>
<table>
<search base="service_base1">
<query>search EventCode=7040 Message!="The start type of the Windows Modules Installer service*" Message!="The start type of the Background Intelligent Transfer Service service*" | stats sum(count) as count by Message</query>
</search>
<option name="count">10</option>
<option name="drilldown">cell</option>
<drilldown>
<set token="term2">$click.value$</set>
</drilldown>
</table>
<table>
<title>Looking at "$term2$"</title>
<search>
<query>index=wineventlog $term2$ SourceName="*Service Control Manager" LogName=System EventCode=7040 Message!="The start type of the Windows Modules Installer service*" Message!="The start type of the Background Intelligent Transfer Service service*" | stats count by _time,host</query>
<earliest>$time1.earliest$</earliest>
<latest>$time1.latest$</latest>
</search>
<option name="count">10</option>
<option name="drilldown">none</option>
</table>
</panel>
</row>
<row>
<panel>
<title>Windows service activity</title>
<table>
<search base="service_base1">
<query>search EventCode!=7045 EventCode!=7036 EventCode!=7035 | stats sum(count) as count by Message</query>
</search>
<option name="count">10</option>
<option name="drilldown">cell</option>
<drilldown>
<set token="term3">$click.value$</set>
</drilldown>
</table>
<table>
<title>Looking at "$term3$"</title>
<search>
<query>index=wineventlog $term3$ SourceName="*Service Control Manager" EventCode!=7045 EventCode!=7036 EventCode!=7035 | fillnull value=* user | stats count by _time,host,user</query>
<earliest>$time1.earliest$</earliest>
<latest>$time1.latest$</latest>
</search>
<option name="count">10</option>
<option name="drilldown">none</option>
</table>
</panel>
</row>
<row>
<panel>
<title>MSI Installs</title>
<table>
<search>
<query>index=wineventlog $filter1$ "SourceName=MsiInstaller" NOT EventCode="1015" | stats list(Message) by _time,host,User | sort -_time</query>
<earliest>$time1.earliest$</earliest>
<latest>$time1.latest$</latest>
</search>
<option name="count">10</option>
<option name="drilldown">none</option>
<option name="refresh.display">progressbar</option>
</table>
</panel>
</row>
</form>
