Dashboard sourcetype validation

Wanted a dashboard supporting checks on a sourcetype to see at the same time:

  • how data where ingested, where and get an overview of samples and fields summary
  • what is the stanza configuration (following the 9-step sequence)

this dashboard uses internal rest API (| rest)

before running dashboard a lookup table (CSV) and a lookup definition must be configured.

The lookup definition name is LD_DATA_transforms_list

Dashboard code

<form version="1.1">
<init>
<unset token="metadata_condition"></unset>
<set token="static_span_value"></set>
<set token="treillis_mode_enabled"></set>
<set token="event_time_field_format"></set>
<unset token="treillis_mode_disabled"></unset>
<unset token="event_time_field_type"></unset>
</init>
<search id="truncation_base">
<query>index="$index$" sourcetype="$sourcetype$"
| eval raw_size=len(_raw)
| stats count as event_count first(_raw) by index, sourcetype, host, raw_size
| eventstats max(raw_size) as max_raw_size
</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
</search>
<label>004 - DATA sourcetype validation</label>
<description>dashboard to ease the validation of a sourcetype</description>
<fieldset submitButton="false">
<input type="time" token="GlobalTimePicker" searchWhenChanged="true">
<label>Time</label>
<default>
<earliest>-15m</earliest>
<latest>now</latest>
</default>
</input>
<input type="dropdown" token="where_is_stanza">
<label>Where is my stanza</label>
<fieldForLabel>entry</fieldForLabel>
<fieldForValue>entry</fieldForValue>
<search>
<query>| union maxtime=300 timeout=300
[| rest splunk_server="local" "/servicesNS/-/-/data/props/calcfields"]
[| rest splunk_server="local" "/servicesNS/-/-/data/props/fieldaliases"]
[| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions"]
[| rest "/servicesNS/-/-/data/transforms/extractions"]
| rename eai:acl.app as app
| eval entry=app+": "+stanza
| stats count by entry</query>
<earliest>-24h@h</earliest>
<latest>now</latest>
</search>
</input>
<input type="dropdown" token="splunk_ta" searchWhenChanged="true">
<label>TA</label>
<fieldForLabel>tille</fieldForLabel>
<fieldForValue>title</fieldForValue>
<search>
<query>| rest /services/apps/local splunk_server=local
| table title</query>
<earliest>-24h@h</earliest>
<latest>now</latest>
</search>
</input>
<input type="checkbox" token="show_field_conf" searchWhenChanged="true" depends="$splunk_ta$">
<label>Show field configuration</label>
<choice value="1">Show conf</choice>
<delimiter/>
<initialValue>1</initialValue>
</input>
<input type="text" token="metadata" searchWhenChanged="true">
<label>metadata search condition</label>
<change>
<condition match="$value$=&quot;&quot;">
<unset token="metadata_condition"></unset>
</condition>
<condition>
<set token="metadata_condition">$metadata$</set>
</condition>
</change>
</input>
<input type="text" token="sourcetype" searchWhenChanged="true">
<label>sourcetype=</label>
</input>
<input type="text" token="source" searchWhenChanged="true">
<label>source=</label>
<default>*</default>
<initialValue>*</initialValue>
<change>
<condition match="$value$=&quot;&quot;">
<unset token="source_filter"></unset>
</condition>
<condition match="$value$=&quot;*&quot;">
<unset token="source_filter"></unset>
</condition>
<condition>
<set token="source_filter">source="$source$"</set>
</condition>
</change>
</input>
<input type="text" token="index" searchWhenChanged="true">
<label>index=</label>
</input>
<input type="checkbox" token="check_time" searchWhenChanged="true" depends="$index$ $sourcetype$">
<label>Check _time &amp; _indextime</label>
<choice value="1">Check</choice>
</input>
<input type="text" token="event_field" searchWhenChanged="true" depends="$index$ $sourcetype$ $check_time$">
<label>_time validation: field to use</label>
<change>
<condition match="$value$=&quot;&quot;">
<unset token="event_time_field"></unset>
</condition>
<condition>
<set token="event_time_field">$event_field$</set>
</condition>
</change>
</input>
<input type="radio" token="event_time_field_type" searchWhenChanged="true" depends="$index$ $sourcetype$ $check_time$">
<label>event time field type</label>
<choice value="epoch">timestamp</choice>
<choice value="datetime">date &amp; time string</choice>
<default>epoch</default>
<initialValue>epoch</initialValue>
</input>
<input type="text" token="event_time_field_format" searchWhenChanged="true" depends="$index$ $sourcetype$ $check_time$">
<label>event time field format</label>
</input>
<input type="checkbox" token="indexing_delay" searchWhenChanged="true" depends="$index$ $sourcetype$">
<label>Indexing delay overview</label>
<choice value="1">Activate</choice>
</input>
<input type="checkbox" token="reviewhost" searchWhenChanged="true" depends="$index$ $sourcetype$">
<label>Review host</label>
<choice value="1">Review</choice>
</input>
<input type="checkbox" token="log_distribution_timechart" searchWhenChanged="true" depends="$index$ $sourcetype$">
<label>Log timechart</label>
<choice value="1">Activate</choice>
</input>
<input type="checkbox" token="truncation" searchWhenChanged="true" depends="$index$ $sourcetype$">
<label>Check potential truncation</label>
<choice value="1">Activate</choice>
</input>
<input type="checkbox" token="fieldsample" searchWhenChanged="true" depends="$index$ $sourcetype$">
<label>Field samples</label>
<choice value="1">Activate</choice>
</input>
<input type="checkbox" token="fieldsummary" searchWhenChanged="true" depends="$index$ $sourcetype$">
<label>Field summary</label>
<choice value="1">Activate</choice>
<delimiter/>
</input>
</fieldset>
<row>
<panel depends="$metadata_condition$">
<title>Find all available sourcetypes related to topic (click on sourcetype)</title>
<table>
<search>
<query>| metadata index=* type=sourcetypes |search $metadata_condition$
| eval firstEventtime=strftime(firstTime,"%Y-%m-%d %T"), lastEventtime=strftime(lastTime,"%Y-%m-%d %T"), lastIndextime=strftime(recentTime,"%Y-%m-%d %T")
| table sourcetype firstEventtime lastEventtime lastIndextime totalCount</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">cell</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
<drilldown>
<condition match="match('click.name2', &quot;sourcetype&quot;">
<set token="form.sourcetype">$click.value2$</set>
</condition>
</drilldown>
</table>
</panel>
<panel depends="$source_filter$">
<title>Find all available sources related to topic (click on source)</title>
<table>
<search>
<query>| metadata index=* type=sources |search $source_filter$
| eval firstEventtime=strftime(firstTime,"%Y-%m-%d %T"), lastEventtime=strftime(lastTime,"%Y-%m-%d %T"), lastIndextime=strftime(recentTime,"%Y-%m-%d %T")
| table source firstEventtime lastEventtime lastIndextime totalCount</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">cell</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
<drilldown>
<condition match="match('click.name2', &quot;source&quot;">
<set token="form.source">$click.value2$</set>
</condition>
</drilldown>
</table>
</panel>
</row>
<row>
<panel depends="$sourcetype$">
<title>List of indexes containing logs for sourcetype $sourcetype$ (click on index or sourcetype to update filter)</title>
<table>
<search>
<query>| tstats count as totalCount min(_time) as start_date, max(_time) as end_date, max(_indextime) as recent_date dc(host) as hosts where index=* sourcetype="$sourcetype$" by index, sourcetype
| convert timeformat="%Y/%m/%d %H:%M:%S" ctime(start_date)
| convert timeformat="%Y/%m/%d %H:%M:%S" ctime(end_date)
| convert timeformat="%Y/%m/%d %H:%M:%S" ctime(recent_date)
| table index sourcetype start_date end_date recent_date hosts totalCount</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">cell</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
<drilldown>
<condition match="match('click.name2', &quot;index&quot;">
<set token="form.index">$click.value2$</set>
</condition>
<condition match="match('click.name2', &quot;sourcetype&quot;">
<set token="form.sourcetype">$click.value2$</set>
</condition>
</drilldown>
</table>
</panel>
</row>
<row>
<panel depends="$index$">
<title>list of sourcetypes in index(es) $index$ (click on index or sourcetype to update filter)</title>
<table>
<search>
<query>| tstats count as totalCount min(_time) as start_date, max(_time) as end_date, max(_indextime) as recent_date dc(host) as hosts where index="$index$" sourcetype=* by index, sourcetype
| convert timeformat="%Y/%m/%d %H:%M:%S" ctime(start_date)
| convert timeformat="%Y/%m/%d %H:%M:%S" ctime(end_date)
| convert timeformat="%Y/%m/%d %H:%M:%S" ctime(recent_date)
| table index sourcetype start_date end_date recent_date hosts totalCount</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">100</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">cell</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
<drilldown>
<condition match="match('click.name2', &quot;index&quot;">
<set token="form.index">$click.value2$</set>
</condition>
<condition match="match('click.name2', &quot;sourcetype&quot;">
<set token="form.sourcetype">$click.value2$</set>
</condition>
</drilldown>
</table>
</panel>
</row>
<row>
<panel depends="$index$">
<title>Retention Period</title>
<single>
<search>
<query>| rest /services/data/indexes count=0
| search splunk_server=spk-idx* title=$index$
| eval retentionInDays=frozenTimePeriodInSecs/86400
| eval perc=(currentDBSizeMB * 100 / maxTotalDataSizeMB )
| eval currentDBSizeGB=currentDBSizeMB/1024
| eval maxTotalDataSizeGB=maxTotalDataSizeMB/1024
| stats count(splunk_server) as count sum(currentDBSizeGB) as sumDBSizeMB max(currentDBSizeGB) as maxDBSizeGB_per_Idx max(perc) as max_perc avg(perc) as avgperc max(maxTime) as maxTime min(minTime) as minTime by title,retentionInDays,maxTotalDataSizeGB
| eval total_overall=count*maxTotalDataSizeGB, et=strptime(minTime,"%Y-%m-%dT%T%z"), lt=strptime(maxTime,"%Y-%m-%dT%T%z"), collection_days=ceiling((lt-et)/86400)
| stats min(retentionInDays) as retentionInDays</query>
<earliest>-60m@m</earliest>
<latest>now</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="colorBy">value</option>
<option name="colorMode">block</option>
<option name="drilldown">none</option>
<option name="numberPrecision">0</option>
<option name="rangeColors">["0xdc4e41","0xf1813f","0x006d9c","0x53a051"]</option>
<option name="rangeValues">[100,364,365]</option>
<option name="showSparkline">1</option>
<option name="showTrendIndicator">1</option>
<option name="trellis.enabled">0</option>
<option name="trellis.scales.shared">1</option>
<option name="trellis.size">medium</option>
<option name="trendColorInterpretation">standard</option>
<option name="trendDisplayMode">absolute</option>
<option name="unitPosition">after</option>
<option name="useColors">1</option>
<option name="useThousandSeparators">1</option>
</single>
</panel>
<panel depends="$index$">
<title>Index information for $index$</title>
<table>
<search>
<query>| rest /services/data/indexes count=0
| search splunk_server=spk-idx* title=$index$
| eval retentionInDays=frozenTimePeriodInSecs/86400
| eval perc=(currentDBSizeMB * 100 / maxTotalDataSizeMB )
| eval currentDBSizeGB=currentDBSizeMB/1024
| eval maxTotalDataSizeGB=maxTotalDataSizeMB/1024
| stats count(splunk_server) as count sum(currentDBSizeGB) as sumDBSizeMB max(currentDBSizeGB) as maxDBSizeGB_per_Idx max(perc) as max_perc avg(perc) as avgperc max(maxTime) as maxTime min(minTime) as minTime by title,retentionInDays,maxTotalDataSizeGB
| eval total_overall=count*maxTotalDataSizeGB, et=strptime(minTime,"%Y-%m-%dT%T%z"), lt=strptime(maxTime,"%Y-%m-%dT%T%z"), collection_days=ceiling((lt-et)/86400)
| rename title as "Index Name"
retentionInDays as "Retention (days)"
total_overall as "Total size (GB)"
sumDBSizeMB as "Current size (GB)"
max_perc as "Fill Percentage per indexer"
minTime as "Earliest Event"
maxTime as "Latest Event"
collection_days as "Number of days"
| table "Index Name" "Retention (days)" "Total size (GB)" "Current size (GB)"  "Fill Percentage per indexer"  "Earliest Event" "Latest Event" "Number of days"</query>
<earliest>-60m@m</earliest>
<latest>now</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">100</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $event_time_field$ $event_time_field_type$ $check_time$">
<title>Check _time against field $event_time_field$ of type $event_time_field_type$</title>
<table>
<title>($index$ $sourcetype$)</title>
<search>
<query>index="$index$" sourcetype="$sourcetype$"
| eval event_time=case("$event_time_field_type$"=="epoch",$event_time_field$,"$event_time_field_type$"=="datetime",strptime($event_time_field$,"$event_time_field_format$")), delta_time = _time - event_time
| stats count by delta_time</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
<panel depends="$index$ $sourcetype$ $reviewhost$">
<title>Host distribution (index=$index$ sourcetype=$sourcetype$)</title>
<table>
<search>
<query>| tstats count as event_count where index=$index$ sourcetype="$sourcetype$" by index sourcetype host</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">10</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $indexing_delay$">
<title>Event delay assessment sourcetype=$sourcetype$ index=$index$</title>
<input type="time" token="DelayTimePicker" searchWhenChanged="true">
<label>Time</label>
<default>
<earliest>-24h@h</earliest>
<latest>now</latest>
</default>
</input>
<input type="text" token="static_span" searchWhenChanged="true">
<label>Specific span value</label>
<change>
<condition match="$value$=&quot;&quot;">
<set token="static_span_value"></set>
</condition>
<condition>
<set token="static_span_value">span=$static_span$</set>
</condition>
</change>
</input>
<chart>
<search>
<query>| tstats count as event_count where index="$index$" sourcetype="$sourcetype$" by _time _indextime span=1s
| eval indextime = _indextime, delay_sec=indextime - _time
| timechart $static_span_value$ sum(event_count) as event_count, perc99.9(delay_sec) as p1thousand_delay, perc99.9999(delay_sec) as p1million_delay</query>
<earliest>$DelayTimePicker.earliest$</earliest>
<latest>$DelayTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
<option name="charting.axisTitleX.visibility">visible</option>
<option name="charting.axisTitleY.visibility">visible</option>
<option name="charting.axisTitleY2.visibility">visible</option>
<option name="charting.axisX.abbreviation">none</option>
<option name="charting.axisX.scale">linear</option>
<option name="charting.axisY.abbreviation">none</option>
<option name="charting.axisY.scale">linear</option>
<option name="charting.axisY2.abbreviation">none</option>
<option name="charting.axisY2.enabled">1</option>
<option name="charting.axisY2.scale">linear</option>
<option name="charting.chart">column</option>
<option name="charting.chart.bubbleMaximumSize">50</option>
<option name="charting.chart.bubbleMinimumSize">10</option>
<option name="charting.chart.bubbleSizeBy">area</option>
<option name="charting.chart.nullValueMode">gaps</option>
<option name="charting.chart.overlayFields">event_count</option>
<option name="charting.chart.showDataLabels">none</option>
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
<option name="charting.chart.stackMode">default</option>
<option name="charting.chart.style">shiny</option>
<option name="charting.drilldown">none</option>
<option name="charting.layout.splitSeries">0</option>
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
<option name="charting.legend.mode">standard</option>
<option name="charting.legend.placement">right</option>
<option name="charting.lineWidth">2</option>
<option name="refresh.display">progressbar</option>
<option name="trellis.enabled">0</option>
<option name="trellis.scales.shared">1</option>
<option name="trellis.size">medium</option>
</chart>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $indexing_delay$">
<title>Event indexing overview for sourcetype=$sourcetype$ index=$index$</title>
<chart>
<search>
<query>| tstats prestats=t count where index="$index$" sourcetype="$sourcetype$" by _indextime
| eval _time = _indextime
| timechart $static_span_value$ count as event_count</query>
<earliest>$DelayTimePicker.earliest$</earliest>
<latest>$DelayTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
<option name="charting.axisTitleX.visibility">visible</option>
<option name="charting.axisTitleY.visibility">visible</option>
<option name="charting.axisTitleY2.visibility">visible</option>
<option name="charting.axisX.abbreviation">none</option>
<option name="charting.axisX.scale">linear</option>
<option name="charting.axisY.abbreviation">none</option>
<option name="charting.axisY.scale">linear</option>
<option name="charting.axisY2.abbreviation">none</option>
<option name="charting.axisY2.enabled">0</option>
<option name="charting.axisY2.scale">linear</option>
<option name="charting.chart">column</option>
<option name="charting.chart.bubbleMaximumSize">50</option>
<option name="charting.chart.bubbleMinimumSize">10</option>
<option name="charting.chart.bubbleSizeBy">area</option>
<option name="charting.chart.nullValueMode">gaps</option>
<option name="charting.chart.showDataLabels">none</option>
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
<option name="charting.chart.stackMode">default</option>
<option name="charting.chart.style">shiny</option>
<option name="charting.drilldown">none</option>
<option name="charting.layout.splitSeries">0</option>
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
<option name="charting.legend.mode">standard</option>
<option name="charting.legend.placement">right</option>
<option name="charting.lineWidth">2</option>
<option name="refresh.display">progressbar</option>
<option name="trellis.enabled">0</option>
<option name="trellis.scales.shared">1</option>
<option name="trellis.size">medium</option>
</chart>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $log_distribution_timechart$">
<title>Log distribution over time - treillis mode</title>
<input type="radio" token="treillis_mode" searchWhenChanged="true">
<label>treillis mode</label>
<choice value="1">use treillis</choice>
<choice value="0">don't use treillis</choice>
<default>1</default>
<initialValue>1</initialValue>
</input>
<input type="text" token="static_span" searchWhenChanged="true">
<label>Specific span value</label>
<change>
<condition match="$value$=&quot;&quot;">
<set token="static_span_value"></set>
</condition>
<condition>
<set token="static_span_value">span=$static_span$</set>
</condition>
</change>
</input>
<input type="text" token="timechart_limit" searchWhenChanged="true">
<label>limit</label>
<default>30</default>
<initialValue>30</initialValue>
</input>
<input type="dropdown" token="timechart_other" searchWhenChanged="true">
<label>Use other</label>
<choice value="1">other=True</choice>
<choice value="0">other=False</choice>
<default>1</default>
<initialValue>1</initialValue>
</input>
<chart>
<search>
<query>| tstats count as event_count where index="$index$" sourcetype="$sourcetype$" by _time _indextime host span=1s
| eval index_ts = _indextime, delta_ts=index_ts - _time
| timechart limit=$timechart_limit$ useother=$timechart_other$ $static_span_value$ sum(event_count) as event_count by host</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
<option name="charting.axisTitleX.visibility">collapsed</option>
<option name="charting.axisTitleY.visibility">collapsed</option>
<option name="charting.axisTitleY2.visibility">collapsed</option>
<option name="charting.axisX.abbreviation">none</option>
<option name="charting.axisX.scale">linear</option>
<option name="charting.axisY.abbreviation">none</option>
<option name="charting.axisY.scale">linear</option>
<option name="charting.axisY2.abbreviation">none</option>
<option name="charting.axisY2.enabled">1</option>
<option name="charting.axisY2.scale">linear</option>
<option name="charting.chart">column</option>
<option name="charting.chart.bubbleMaximumSize">50</option>
<option name="charting.chart.bubbleMinimumSize">10</option>
<option name="charting.chart.bubbleSizeBy">area</option>
<option name="charting.chart.nullValueMode">gaps</option>
<option name="charting.chart.overlayFields">delay:*</option>
<option name="charting.chart.showDataLabels">none</option>
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
<option name="charting.chart.stackMode">stacked</option>
<option name="charting.chart.style">shiny</option>
<option name="charting.drilldown">none</option>
<option name="charting.layout.splitSeries">0</option>
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
<option name="charting.legend.mode">standard</option>
<option name="charting.legend.placement">none</option>
<option name="charting.lineWidth">2</option>
<option name="refresh.display">progressbar</option>
<option name="trellis.enabled">$treillis_mode$</option>
<option name="trellis.scales.shared">0</option>
<option name="trellis.size">small</option>
</chart>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $truncation$">
<title>Potential truncation assessment due to max event size (distribution) - check if max_row_size (horizont line) seen in logs is closed to the TRUNCATE value</title>
<chart>
<search base="truncation_base">
<query>| chart sum(event_count) as event_count values(max_raw_size) as max_raw_size by raw_size</query>
</search>
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
<option name="charting.axisTitleX.visibility">visible</option>
<option name="charting.axisTitleY.visibility">visible</option>
<option name="charting.axisTitleY2.text">TRUNCATE</option>
<option name="charting.axisTitleY2.visibility">visible</option>
<option name="charting.axisX.abbreviation">none</option>
<option name="charting.axisX.scale">linear</option>
<option name="charting.axisY.abbreviation">none</option>
<option name="charting.axisY.scale">linear</option>
<option name="charting.axisY2.abbreviation">none</option>
<option name="charting.axisY2.enabled">1</option>
<option name="charting.axisY2.scale">linear</option>
<option name="charting.chart">column</option>
<option name="charting.chart.bubbleMaximumSize">50</option>
<option name="charting.chart.bubbleMinimumSize">10</option>
<option name="charting.chart.bubbleSizeBy">area</option>
<option name="charting.chart.nullValueMode">gaps</option>
<option name="charting.chart.overlayFields">max_raw_size</option>
<option name="charting.chart.showDataLabels">none</option>
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
<option name="charting.chart.stackMode">default</option>
<option name="charting.chart.style">shiny</option>
<option name="charting.drilldown">none</option>
<option name="charting.layout.splitSeries">0</option>
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
<option name="charting.legend.mode">standard</option>
<option name="charting.legend.placement">right</option>
<option name="charting.lineWidth">2</option>
<option name="refresh.display">progressbar</option>
<option name="trellis.enabled">0</option>
<option name="trellis.scales.shared">1</option>
<option name="trellis.size">medium</option>
</chart>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $truncation$">
<title>Potential truncaton assessment (samples per event size)</title>
<table>
<search base="truncation_base">
<query>
| sort 0 - raw_size</query>
</search>
<option name="count">100</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $fieldsample$">
<title>Log sample (index="$index$" sourcetype="$sourcetype$")</title>
<input type="text" token="search_field" searchWhenChanged="true">
<label>search ...</label>
<default>*</default>
<initialValue>*</initialValue>
</input>
<input type="dropdown" token="head_count" searchWhenChanged="true">
<label>head ( return X events)</label>
<default>50</default>
<initialValue>50</initialValue>
<choice value="10">10</choice>
<choice value="50">50</choice>
<choice value="100">100</choice>
<choice value="500">500</choice>
<choice value="1000">1000</choice>
</input>
<input type="dropdown" token="sampling" searchWhenChanged="true">
<label>sampling</label>
<default>1</default>
<initialValue>1</initialValue>
<choice value="1">no sampling</choice>
<choice value="10">1:10</choice>
<choice value="100">1:100</choice>
<choice value="1000">1:1000</choice>
<choice value="10000">1:10000</choice>
</input>
<table>
<search>
<query>index="$index$" sourcetype="$sourcetype$"
| search $search_field$
| fillnull value="no_punct"
| stats count first(_raw) as raw_event, first(_time) as _time values(host) as host by punct
| sort - count
| head $head_count$
| table _time, raw_event, punct, count</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="drilldown">none</option>
<option name="refresh.display">progressbar</option>
</table>
</panel>
</row>
<row>
<panel depends="$index$ $sourcetype$ $fieldsummary$">
<title>fieldsummary - click on one field to check _time</title>
<input type="text" token="field_name" searchWhenChanged="true">
<label>Field Name</label>
<default>*</default>
<initialValue>*</initialValue>
</input>
<input type="radio" token="show_key_fields" searchWhenChanged="true">
<label>Columns for sourcetype documentation</label>
<choice value="| sort field">Default columns</choice>
<default>| sort field</default>
<initialValue>| sort field</initialValue>
<fieldForLabel>label</fieldForLabel>
<fieldForValue>option</fieldForValue>
<search>
<query>| makeresults
| eval label="Only key columns"
| eval option="| search NOT field IN (\"field\",\"distinct_count\",\"coverage\",\"data_type\",\"parsing_type\",\"quality_check\",\"description\",\"index\",\"sourcetype\",\"source\",\"splunk*\",\"tag*\",\"eventtype\",\"punct\",\"linecount\",\"timeendpos\",\"timestartpos\")
| table field, distinct_count, coverage, data_type, parsing_type, quality_check, description values | sort field"</query>
</search>
</input>
<table>
<search>
<query>index="$index$" sourcetype="$sourcetype$"
| head 100000
| fields - date_*
| fieldsummary
| eventstats max(count) as event_count
| eval coverage=round(count/event_count*100,2)
| sort - coverage
| rex field=values max_match=0 "value\":\"(?P&lt;value&gt;[^\"]+)\""
| eval data_type=if(match(value,"^\d+$"),"Number","String")
| fields - value
| search field="$field_name$"
$show_key_fields$</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">100</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">cell</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
<drilldown>
<condition match="match('click.name2', &quot;field&quot;">
<set token="form.event_time_field">$click.value2$</set>
</condition>
</drilldown>
</table>
</panel>
</row>
<row>
<panel depends="$splunk_ta$">
<title>List of stanza defined in $splunk_ta$ (click on stanza)</title>
<table>
<search>
<query>| union maxtime=300 timeout=300
[| rest splunk_server="local" "/servicesNS/-/-/data/props/calcfields"]
[| rest splunk_server="local" "/servicesNS/-/-/data/props/fieldaliases"]
[| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions"]
| rename eai:acl.app as app
| where app=="$splunk_ta$"
| stats count by stanza
| fields - count</query>
<earliest>-15m</earliest>
<latest>now</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="percentagesRow">false</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
<option name="drilldown">cell</option>
<drilldown>
<condition match="match('click.name2', &quot;stanza&quot;">
<set token="form.sourcetype">$click.value2$</set>
</condition>
</drilldown>
</table>
</panel>
</row>
<row>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<html>
<body bgcolor="#88FF88">
<h1>Running configuration for $splunk_ta$</h1>
<p>Following panels provide a view on current running configuration for sourcetype $sourcetype$.</p>
<h3>
<a href="https://docs.splunk.com/Documentation/Splunk/9.0.4/Knowledge/Searchtimeoperationssequence">The sequence of search-time operations</a>
</h3>
<table>
<tr>
<th>Search-time operation order</th>
<th>Operation name</th>
<th>Configurable in Splunk Web?</th>
<th>Location of file configuration</th>
</tr>
<tr>
<td>1</td>
<td>Role-based field filtering</td>
<td>No</td>
<td>fieldFilter-&lt;fieldname&gt; in a stanza in the authorize.conf file.</td>
</tr>
<tr>
<td>2</td>
<td>Inline field extraction (no field transform)</td>
<td>Yes</td>
<td>EXTRACT-&lt;class&gt; in a stanza in the props.conf file.</td>
</tr>
<tr>
<td>3</td>
<td>Field extraction that uses a field transform</td>
<td>Yes</td>
<td>REPORT-&lt;class&gt; in a stanza in the props.conf file.</td>
</tr>
<tr>
<td>4</td>
<td>Automatic key-value field extraction</td>
<td>No</td>
<td>In stanzas in the props.conf file, where KV_MODE is set to a valid value other than none. If no KV_MODE value is specified for a stanza, it is set to auto by default.</td>
</tr>
<tr>
<td>5</td>
<td>Field aliasing</td>
<td>Yes</td>
<td>FIELDALIAS-&lt;class&gt; in a stanza in the props.conf file.</td>
</tr>
<tr>
<td>6</td>
<td>Calculated fields</td>
<td>Yes</td>
<td>EVAL-&lt;fieldname&gt; in a stanza in the props.conf file.</td>
</tr>
<tr>
<td>7</td>
<td>Lookups</td>
<td>Yes</td>
<td>LOOKUP-&lt;class&gt; in a stanza in the props.conf file.</td>
</tr>
<tr>
<td>8</td>
<td>Event types</td>
<td>Yes</td>
<td>In a stanza in the eventtypes.conf file.</td>
</tr>
<tr>
<td>9</td>
<td>Tags</td>
<td>Yes</td>
<td>In a stanza in the tags.conf file.</td>
</tr>
</table>
</body>
</html>
</panel>
</row>
<row>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>Configuration files for $splunk_ta$</title>
<input type="text" token="search_conf_keyword" searchWhenChanged="true" depends="$splunk_ta$ $show_field_conf$">
<label>Search in conf</label>
<default>*</default>
<initialValue>*</initialValue>
</input>
</panel>
</row>
<row>
<panel depends="$sourcetype$ $splunk_ta$ $show_field_conf$">
<title>2 - Inline field extraction (no field transform) for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR value="$search_conf_keyword$")</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions" | eval Type="extractions" | rex field=title " : (?&lt;type&gt;(EXTRACT|REPORT))-(?&lt;title&gt;[^\e]+)"
| rename eai:acl.app as app
| where app=="$splunk_ta$" AND stanza LIKE "%$sourcetype$%"
| search attribute="$search_conf_keyword$" OR value="$search_conf_keyword$"
| table stanza, type, attribute, value</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
<panel depends="$sourcetype$ $splunk_ta$ $show_field_conf$">
<title>3a - Transforms for $splunk_ta$ (limited to "$search_conf_keyword$" OR REGEX="$search_conf_keyword$" OR DEST_KEY="$search_conf_keyword$" OR FORMAT="$search_conf_keyword$"))</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/transforms/extractions"
| rename eai:acl.app as app
| where app=="$splunk_ta$"
| eval type=if(isnull(DELIMS),"regex","delimiter")
| search title="$search_conf_keyword$" OR REGEX="$search_conf_keyword$" OR DEST_KEY="$search_conf_keyword$" OR FORMAT="$search_conf_keyword$" OR FIELDS="$search_conf_keyword$"
| table title, type, SOURCE_KEY, DELIMS, REGEX, DEST_KEY, FORMAT, FIELDS</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
<row>
<panel depends="$sourcetype$ $splunk_ta$ $show_field_conf$">
<title>3 - Field extraction that uses a field transform - Combined REPORT (Step1) and TRANSFORMS (Step2) for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR value="$search_conf_keyword$")</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions"
| eval Type="extractions"
| rex field=title " : (?&lt;type&gt;(EXTRACT|REPORT))-(?&lt;title&gt;[^\e]+)"
| rename eai:acl.app as app
| table app, stanza, type, attribute, value
| search app="*" OR (
[| rest splunk_server="local" "/servicesNS/-/-/data/transforms/extractions"
| rename eai:acl.app as app
| where app=="$splunk_ta$"
| eval type=if(isnull(DELIMS),"regex","delimiter")
| table title, type, SOURCE_KEY, DELIMS, REGEX, DEST_KEY, FORMAT, FIELDS
| outputlookup LD_DATA_transforms_list] )
| search app="$splunk_ta$" AND stanza="*$sourcetype$*"
| eval title=split(value,","), value=replace(value,",",", ")
| mvexpand title
| fields - app
| lookup LD_DATA_transforms_list title OUTPUT
| search attribute="$search_conf_keyword$" OR value="$search_conf_keyword$" OR title="$search_conf_keyword$" OR REGEX="$search_conf_keyword$" OR DEST_KEY="$search_conf_keyword$" OR FORMAT="$search_conf_keyword$" OR FIELDS="$search_conf_keyword$"
| table stanza, type, attribute, value, title, *</query>
<earliest>-15m</earliest>
<latest>now</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
<row>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>5 - Field aliases for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR value="$search_conf_keyword$")</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/props/fieldaliases" | rex field=title " : [^\-]+-(?&lt;title&gt;[^\e]+)"
| rename eai:acl.app as app
| where app=="$splunk_ta$" AND stanza LIKE "%$sourcetype$%"
| search attribute="$search_conf_keyword$" OR value="$search_conf_keyword$"
| table stanza, attribute, value</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>6 - Calculated fields for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR field_name="$search_conf_keyword$" OR value="$search_conf_keyword$")</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/props/calcfields" | rex field=title " : [^\-]+-(?&lt;title&gt;[^\e]+)"
| rename eai:acl.app as app, field.name as field_name
| where app=="$splunk_ta$" AND stanza LIKE "%$sourcetype$%"
| search attribute="$search_conf_keyword$" OR field_name="$search_conf_keyword$" OR value="$search_conf_keyword$"
| table stanza, attribute, field_name, value</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
<row>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>7 - Automatic lookups</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/props/lookups" | rex field=title " : [^\-]+-(?&lt;title&gt;[^\e]+)"
| rename eai:acl.app as app, value as Lookup
| where app=="$splunk_ta$"  AND stanza LIKE "%$sourcetype$%"
| eval Name=stanza + " : " + attribute
| table Name, Lookup</query>
<earliest>-15m</earliest>
<latest>now</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>8 - Event types for $splunk_ta$ (limited to search="$search_conf_keyword$") - click on search to launch CIM validator</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/saved/eventtypes" | eval Type="Event Types" | eval tags=mvjoin(tags, ",")
| rename eai:acl.app as app
| where app=="$splunk_ta$"
| search search="$search_conf_keyword$" OR title="$search_conf_keyword$" OR tags="$search_conf_keyword$"
| table title, search, tags</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">cell</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
<drilldown>
<link target="_blank">/app/SA-cim_vladiator/cim_validator?form.cim_search=$click.value2$</link>
</drilldown>
</table>
</panel>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>9 - Tags for $splunk_ta$ (limited to title="$search_conf_keyword$")</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/saved/fvtags"
| rename eai:acl.app as app
| where app=="$splunk_ta$"
| search title="$search_conf_keyword$" OR tags="$search_conf_keyword$"
| table tags, title</query>
<earliest>$GlobalTimePicker.earliest$</earliest>
<latest>$GlobalTimePicker.latest$</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="refresh.display">progressbar</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
<row>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>Lookup tables</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/lookup-table-files"
| rename eai:acl.app as app
| where app=="$splunk_ta$"
| table title</query>
<earliest>-15m</earliest>
<latest>now</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
<panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$">
<title>Lookup definitions</title>
<table>
<search>
<query>| rest splunk_server="local" "/servicesNS/-/-/data/transforms/lookups"
| rename eai:acl.app as app
| where app=="$splunk_ta$"
| table title, type, filename, fields_list</query>
<earliest>-15m</earliest>
<latest>now</latest>
<sampleRatio>1</sampleRatio>
</search>
<option name="count">5</option>
<option name="dataOverlayMode">none</option>
<option name="drilldown">none</option>
<option name="percentagesRow">false</option>
<option name="rowNumbers">false</option>
<option name="totalsRow">false</option>
<option name="wrap">true</option>
</table>
</panel>
</row>
</form>
Share This:

Leave A Comment?