Wanted a dashboard supporting checks on a sourcetype to see at the same time:
- how data where ingested, where and get an overview of samples and fields summary
- what is the stanza configuration (following the 9-step sequence)
this dashboard uses internal rest API (| rest)
before running dashboard a lookup table (CSV) and a lookup definition must be configured.
The lookup definition name is LD_DATA_transforms_list
Dashboard code
<form version="1.1"> <init> <unset token="metadata_condition"></unset> <set token="static_span_value"></set> <set token="treillis_mode_enabled"></set> <set token="event_time_field_format"></set> <unset token="treillis_mode_disabled"></unset> <unset token="event_time_field_type"></unset> </init> <search id="truncation_base"> <query>index="$index$" sourcetype="$sourcetype$" | eval raw_size=len(_raw) | stats count as event_count first(_raw) by index, sourcetype, host, raw_size | eventstats max(raw_size) as max_raw_size </query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> </search> <label>004 - DATA sourcetype validation</label> <description>dashboard to ease the validation of a sourcetype</description> <fieldset submitButton="false"> <input type="time" token="GlobalTimePicker" searchWhenChanged="true"> <label>Time</label> <default> <earliest>-15m</earliest> <latest>now</latest> </default> </input> <input type="dropdown" token="where_is_stanza"> <label>Where is my stanza</label> <fieldForLabel>entry</fieldForLabel> <fieldForValue>entry</fieldForValue> <search> <query>| union maxtime=300 timeout=300 [| rest splunk_server="local" "/servicesNS/-/-/data/props/calcfields"] [| rest splunk_server="local" "/servicesNS/-/-/data/props/fieldaliases"] [| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions"] [| rest "/servicesNS/-/-/data/transforms/extractions"] | rename eai:acl.app as app | eval entry=app+": "+stanza | stats count by entry</query> <earliest>-24h@h</earliest> <latest>now</latest> </search> </input> <input type="dropdown" token="splunk_ta" searchWhenChanged="true"> <label>TA</label> <fieldForLabel>tille</fieldForLabel> <fieldForValue>title</fieldForValue> <search> <query>| rest /services/apps/local splunk_server=local | table title</query> <earliest>-24h@h</earliest> <latest>now</latest> </search> </input> <input type="checkbox" token="show_field_conf" searchWhenChanged="true" depends="$splunk_ta$"> <label>Show field configuration</label> <choice value="1">Show conf</choice> <delimiter/> <initialValue>1</initialValue> </input> <input type="text" token="metadata" searchWhenChanged="true"> <label>metadata search condition</label> <change> <condition match="$value$="""> <unset token="metadata_condition"></unset> </condition> <condition> <set token="metadata_condition">$metadata$</set> </condition> </change> </input> <input type="text" token="sourcetype" searchWhenChanged="true"> <label>sourcetype=</label> </input> <input type="text" token="source" searchWhenChanged="true"> <label>source=</label> <default>*</default> <initialValue>*</initialValue> <change> <condition match="$value$="""> <unset token="source_filter"></unset> </condition> <condition match="$value$="*""> <unset token="source_filter"></unset> </condition> <condition> <set token="source_filter">source="$source$"</set> </condition> </change> </input> <input type="text" token="index" searchWhenChanged="true"> <label>index=</label> </input> <input type="checkbox" token="check_time" searchWhenChanged="true" depends="$index$ $sourcetype$"> <label>Check _time & _indextime</label> <choice value="1">Check</choice> </input> <input type="text" token="event_field" searchWhenChanged="true" depends="$index$ $sourcetype$ $check_time$"> <label>_time validation: field to use</label> <change> <condition match="$value$="""> <unset token="event_time_field"></unset> </condition> <condition> <set token="event_time_field">$event_field$</set> </condition> </change> </input> <input type="radio" token="event_time_field_type" searchWhenChanged="true" depends="$index$ $sourcetype$ $check_time$"> <label>event time field type</label> <choice value="epoch">timestamp</choice> <choice value="datetime">date & time string</choice> <default>epoch</default> <initialValue>epoch</initialValue> </input> <input type="text" token="event_time_field_format" searchWhenChanged="true" depends="$index$ $sourcetype$ $check_time$"> <label>event time field format</label> </input> <input type="checkbox" token="indexing_delay" searchWhenChanged="true" depends="$index$ $sourcetype$"> <label>Indexing delay overview</label> <choice value="1">Activate</choice> </input> <input type="checkbox" token="reviewhost" searchWhenChanged="true" depends="$index$ $sourcetype$"> <label>Review host</label> <choice value="1">Review</choice> </input> <input type="checkbox" token="log_distribution_timechart" searchWhenChanged="true" depends="$index$ $sourcetype$"> <label>Log timechart</label> <choice value="1">Activate</choice> </input> <input type="checkbox" token="truncation" searchWhenChanged="true" depends="$index$ $sourcetype$"> <label>Check potential truncation</label> <choice value="1">Activate</choice> </input> <input type="checkbox" token="fieldsample" searchWhenChanged="true" depends="$index$ $sourcetype$"> <label>Field samples</label> <choice value="1">Activate</choice> </input> <input type="checkbox" token="fieldsummary" searchWhenChanged="true" depends="$index$ $sourcetype$"> <label>Field summary</label> <choice value="1">Activate</choice> <delimiter/> </input> </fieldset> <row> <panel depends="$metadata_condition$"> <title>Find all available sourcetypes related to topic (click on sourcetype)</title> <table> <search> <query>| metadata index=* type=sourcetypes |search $metadata_condition$ | eval firstEventtime=strftime(firstTime,"%Y-%m-%d %T"), lastEventtime=strftime(lastTime,"%Y-%m-%d %T"), lastIndextime=strftime(recentTime,"%Y-%m-%d %T") | table sourcetype firstEventtime lastEventtime lastIndextime totalCount</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">cell</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> <drilldown> <condition match="match('click.name2', "sourcetype""> <set token="form.sourcetype">$click.value2$</set> </condition> </drilldown> </table> </panel> <panel depends="$source_filter$"> <title>Find all available sources related to topic (click on source)</title> <table> <search> <query>| metadata index=* type=sources |search $source_filter$ | eval firstEventtime=strftime(firstTime,"%Y-%m-%d %T"), lastEventtime=strftime(lastTime,"%Y-%m-%d %T"), lastIndextime=strftime(recentTime,"%Y-%m-%d %T") | table source firstEventtime lastEventtime lastIndextime totalCount</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">cell</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> <drilldown> <condition match="match('click.name2', "source""> <set token="form.source">$click.value2$</set> </condition> </drilldown> </table> </panel> </row> <row> <panel depends="$sourcetype$"> <title>List of indexes containing logs for sourcetype $sourcetype$ (click on index or sourcetype to update filter)</title> <table> <search> <query>| tstats count as totalCount min(_time) as start_date, max(_time) as end_date, max(_indextime) as recent_date dc(host) as hosts where index=* sourcetype="$sourcetype$" by index, sourcetype | convert timeformat="%Y/%m/%d %H:%M:%S" ctime(start_date) | convert timeformat="%Y/%m/%d %H:%M:%S" ctime(end_date) | convert timeformat="%Y/%m/%d %H:%M:%S" ctime(recent_date) | table index sourcetype start_date end_date recent_date hosts totalCount</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">cell</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> <drilldown> <condition match="match('click.name2', "index""> <set token="form.index">$click.value2$</set> </condition> <condition match="match('click.name2', "sourcetype""> <set token="form.sourcetype">$click.value2$</set> </condition> </drilldown> </table> </panel> </row> <row> <panel depends="$index$"> <title>list of sourcetypes in index(es) $index$ (click on index or sourcetype to update filter)</title> <table> <search> <query>| tstats count as totalCount min(_time) as start_date, max(_time) as end_date, max(_indextime) as recent_date dc(host) as hosts where index="$index$" sourcetype=* by index, sourcetype | convert timeformat="%Y/%m/%d %H:%M:%S" ctime(start_date) | convert timeformat="%Y/%m/%d %H:%M:%S" ctime(end_date) | convert timeformat="%Y/%m/%d %H:%M:%S" ctime(recent_date) | table index sourcetype start_date end_date recent_date hosts totalCount</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">100</option> <option name="dataOverlayMode">none</option> <option name="drilldown">cell</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> <drilldown> <condition match="match('click.name2', "index""> <set token="form.index">$click.value2$</set> </condition> <condition match="match('click.name2', "sourcetype""> <set token="form.sourcetype">$click.value2$</set> </condition> </drilldown> </table> </panel> </row> <row> <panel depends="$index$"> <title>Retention Period</title> <single> <search> <query>| rest /services/data/indexes count=0 | search splunk_server=spk-idx* title=$index$ | eval retentionInDays=frozenTimePeriodInSecs/86400 | eval perc=(currentDBSizeMB * 100 / maxTotalDataSizeMB ) | eval currentDBSizeGB=currentDBSizeMB/1024 | eval maxTotalDataSizeGB=maxTotalDataSizeMB/1024 | stats count(splunk_server) as count sum(currentDBSizeGB) as sumDBSizeMB max(currentDBSizeGB) as maxDBSizeGB_per_Idx max(perc) as max_perc avg(perc) as avgperc max(maxTime) as maxTime min(minTime) as minTime by title,retentionInDays,maxTotalDataSizeGB | eval total_overall=count*maxTotalDataSizeGB, et=strptime(minTime,"%Y-%m-%dT%T%z"), lt=strptime(maxTime,"%Y-%m-%dT%T%z"), collection_days=ceiling((lt-et)/86400) | stats min(retentionInDays) as retentionInDays</query> <earliest>-60m@m</earliest> <latest>now</latest> <sampleRatio>1</sampleRatio> </search> <option name="colorBy">value</option> <option name="colorMode">block</option> <option name="drilldown">none</option> <option name="numberPrecision">0</option> <option name="rangeColors">["0xdc4e41","0xf1813f","0x006d9c","0x53a051"]</option> <option name="rangeValues">[100,364,365]</option> <option name="showSparkline">1</option> <option name="showTrendIndicator">1</option> <option name="trellis.enabled">0</option> <option name="trellis.scales.shared">1</option> <option name="trellis.size">medium</option> <option name="trendColorInterpretation">standard</option> <option name="trendDisplayMode">absolute</option> <option name="unitPosition">after</option> <option name="useColors">1</option> <option name="useThousandSeparators">1</option> </single> </panel> <panel depends="$index$"> <title>Index information for $index$</title> <table> <search> <query>| rest /services/data/indexes count=0 | search splunk_server=spk-idx* title=$index$ | eval retentionInDays=frozenTimePeriodInSecs/86400 | eval perc=(currentDBSizeMB * 100 / maxTotalDataSizeMB ) | eval currentDBSizeGB=currentDBSizeMB/1024 | eval maxTotalDataSizeGB=maxTotalDataSizeMB/1024 | stats count(splunk_server) as count sum(currentDBSizeGB) as sumDBSizeMB max(currentDBSizeGB) as maxDBSizeGB_per_Idx max(perc) as max_perc avg(perc) as avgperc max(maxTime) as maxTime min(minTime) as minTime by title,retentionInDays,maxTotalDataSizeGB | eval total_overall=count*maxTotalDataSizeGB, et=strptime(minTime,"%Y-%m-%dT%T%z"), lt=strptime(maxTime,"%Y-%m-%dT%T%z"), collection_days=ceiling((lt-et)/86400) | rename title as "Index Name" retentionInDays as "Retention (days)" total_overall as "Total size (GB)" sumDBSizeMB as "Current size (GB)" max_perc as "Fill Percentage per indexer" minTime as "Earliest Event" maxTime as "Latest Event" collection_days as "Number of days" | table "Index Name" "Retention (days)" "Total size (GB)" "Current size (GB)" "Fill Percentage per indexer" "Earliest Event" "Latest Event" "Number of days"</query> <earliest>-60m@m</earliest> <latest>now</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">100</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $event_time_field$ $event_time_field_type$ $check_time$"> <title>Check _time against field $event_time_field$ of type $event_time_field_type$</title> <table> <title>($index$ $sourcetype$)</title> <search> <query>index="$index$" sourcetype="$sourcetype$" | eval event_time=case("$event_time_field_type$"=="epoch",$event_time_field$,"$event_time_field_type$"=="datetime",strptime($event_time_field$,"$event_time_field_format$")), delta_time = _time - event_time | stats count by delta_time</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> <panel depends="$index$ $sourcetype$ $reviewhost$"> <title>Host distribution (index=$index$ sourcetype=$sourcetype$)</title> <table> <search> <query>| tstats count as event_count where index=$index$ sourcetype="$sourcetype$" by index sourcetype host</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">10</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $indexing_delay$"> <title>Event delay assessment sourcetype=$sourcetype$ index=$index$</title> <input type="time" token="DelayTimePicker" searchWhenChanged="true"> <label>Time</label> <default> <earliest>-24h@h</earliest> <latest>now</latest> </default> </input> <input type="text" token="static_span" searchWhenChanged="true"> <label>Specific span value</label> <change> <condition match="$value$="""> <set token="static_span_value"></set> </condition> <condition> <set token="static_span_value">span=$static_span$</set> </condition> </change> </input> <chart> <search> <query>| tstats count as event_count where index="$index$" sourcetype="$sourcetype$" by _time _indextime span=1s | eval indextime = _indextime, delay_sec=indextime - _time | timechart $static_span_value$ sum(event_count) as event_count, perc99.9(delay_sec) as p1thousand_delay, perc99.9999(delay_sec) as p1million_delay</query> <earliest>$DelayTimePicker.earliest$</earliest> <latest>$DelayTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option> <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option> <option name="charting.axisTitleX.visibility">visible</option> <option name="charting.axisTitleY.visibility">visible</option> <option name="charting.axisTitleY2.visibility">visible</option> <option name="charting.axisX.abbreviation">none</option> <option name="charting.axisX.scale">linear</option> <option name="charting.axisY.abbreviation">none</option> <option name="charting.axisY.scale">linear</option> <option name="charting.axisY2.abbreviation">none</option> <option name="charting.axisY2.enabled">1</option> <option name="charting.axisY2.scale">linear</option> <option name="charting.chart">column</option> <option name="charting.chart.bubbleMaximumSize">50</option> <option name="charting.chart.bubbleMinimumSize">10</option> <option name="charting.chart.bubbleSizeBy">area</option> <option name="charting.chart.nullValueMode">gaps</option> <option name="charting.chart.overlayFields">event_count</option> <option name="charting.chart.showDataLabels">none</option> <option name="charting.chart.sliceCollapsingThreshold">0.01</option> <option name="charting.chart.stackMode">default</option> <option name="charting.chart.style">shiny</option> <option name="charting.drilldown">none</option> <option name="charting.layout.splitSeries">0</option> <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option> <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option> <option name="charting.legend.mode">standard</option> <option name="charting.legend.placement">right</option> <option name="charting.lineWidth">2</option> <option name="refresh.display">progressbar</option> <option name="trellis.enabled">0</option> <option name="trellis.scales.shared">1</option> <option name="trellis.size">medium</option> </chart> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $indexing_delay$"> <title>Event indexing overview for sourcetype=$sourcetype$ index=$index$</title> <chart> <search> <query>| tstats prestats=t count where index="$index$" sourcetype="$sourcetype$" by _indextime | eval _time = _indextime | timechart $static_span_value$ count as event_count</query> <earliest>$DelayTimePicker.earliest$</earliest> <latest>$DelayTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option> <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option> <option name="charting.axisTitleX.visibility">visible</option> <option name="charting.axisTitleY.visibility">visible</option> <option name="charting.axisTitleY2.visibility">visible</option> <option name="charting.axisX.abbreviation">none</option> <option name="charting.axisX.scale">linear</option> <option name="charting.axisY.abbreviation">none</option> <option name="charting.axisY.scale">linear</option> <option name="charting.axisY2.abbreviation">none</option> <option name="charting.axisY2.enabled">0</option> <option name="charting.axisY2.scale">linear</option> <option name="charting.chart">column</option> <option name="charting.chart.bubbleMaximumSize">50</option> <option name="charting.chart.bubbleMinimumSize">10</option> <option name="charting.chart.bubbleSizeBy">area</option> <option name="charting.chart.nullValueMode">gaps</option> <option name="charting.chart.showDataLabels">none</option> <option name="charting.chart.sliceCollapsingThreshold">0.01</option> <option name="charting.chart.stackMode">default</option> <option name="charting.chart.style">shiny</option> <option name="charting.drilldown">none</option> <option name="charting.layout.splitSeries">0</option> <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option> <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option> <option name="charting.legend.mode">standard</option> <option name="charting.legend.placement">right</option> <option name="charting.lineWidth">2</option> <option name="refresh.display">progressbar</option> <option name="trellis.enabled">0</option> <option name="trellis.scales.shared">1</option> <option name="trellis.size">medium</option> </chart> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $log_distribution_timechart$"> <title>Log distribution over time - treillis mode</title> <input type="radio" token="treillis_mode" searchWhenChanged="true"> <label>treillis mode</label> <choice value="1">use treillis</choice> <choice value="0">don't use treillis</choice> <default>1</default> <initialValue>1</initialValue> </input> <input type="text" token="static_span" searchWhenChanged="true"> <label>Specific span value</label> <change> <condition match="$value$="""> <set token="static_span_value"></set> </condition> <condition> <set token="static_span_value">span=$static_span$</set> </condition> </change> </input> <input type="text" token="timechart_limit" searchWhenChanged="true"> <label>limit</label> <default>30</default> <initialValue>30</initialValue> </input> <input type="dropdown" token="timechart_other" searchWhenChanged="true"> <label>Use other</label> <choice value="1">other=True</choice> <choice value="0">other=False</choice> <default>1</default> <initialValue>1</initialValue> </input> <chart> <search> <query>| tstats count as event_count where index="$index$" sourcetype="$sourcetype$" by _time _indextime host span=1s | eval index_ts = _indextime, delta_ts=index_ts - _time | timechart limit=$timechart_limit$ useother=$timechart_other$ $static_span_value$ sum(event_count) as event_count by host</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option> <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option> <option name="charting.axisTitleX.visibility">collapsed</option> <option name="charting.axisTitleY.visibility">collapsed</option> <option name="charting.axisTitleY2.visibility">collapsed</option> <option name="charting.axisX.abbreviation">none</option> <option name="charting.axisX.scale">linear</option> <option name="charting.axisY.abbreviation">none</option> <option name="charting.axisY.scale">linear</option> <option name="charting.axisY2.abbreviation">none</option> <option name="charting.axisY2.enabled">1</option> <option name="charting.axisY2.scale">linear</option> <option name="charting.chart">column</option> <option name="charting.chart.bubbleMaximumSize">50</option> <option name="charting.chart.bubbleMinimumSize">10</option> <option name="charting.chart.bubbleSizeBy">area</option> <option name="charting.chart.nullValueMode">gaps</option> <option name="charting.chart.overlayFields">delay:*</option> <option name="charting.chart.showDataLabels">none</option> <option name="charting.chart.sliceCollapsingThreshold">0.01</option> <option name="charting.chart.stackMode">stacked</option> <option name="charting.chart.style">shiny</option> <option name="charting.drilldown">none</option> <option name="charting.layout.splitSeries">0</option> <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option> <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option> <option name="charting.legend.mode">standard</option> <option name="charting.legend.placement">none</option> <option name="charting.lineWidth">2</option> <option name="refresh.display">progressbar</option> <option name="trellis.enabled">$treillis_mode$</option> <option name="trellis.scales.shared">0</option> <option name="trellis.size">small</option> </chart> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $truncation$"> <title>Potential truncation assessment due to max event size (distribution) - check if max_row_size (horizont line) seen in logs is closed to the TRUNCATE value</title> <chart> <search base="truncation_base"> <query>| chart sum(event_count) as event_count values(max_raw_size) as max_raw_size by raw_size</query> </search> <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option> <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option> <option name="charting.axisTitleX.visibility">visible</option> <option name="charting.axisTitleY.visibility">visible</option> <option name="charting.axisTitleY2.text">TRUNCATE</option> <option name="charting.axisTitleY2.visibility">visible</option> <option name="charting.axisX.abbreviation">none</option> <option name="charting.axisX.scale">linear</option> <option name="charting.axisY.abbreviation">none</option> <option name="charting.axisY.scale">linear</option> <option name="charting.axisY2.abbreviation">none</option> <option name="charting.axisY2.enabled">1</option> <option name="charting.axisY2.scale">linear</option> <option name="charting.chart">column</option> <option name="charting.chart.bubbleMaximumSize">50</option> <option name="charting.chart.bubbleMinimumSize">10</option> <option name="charting.chart.bubbleSizeBy">area</option> <option name="charting.chart.nullValueMode">gaps</option> <option name="charting.chart.overlayFields">max_raw_size</option> <option name="charting.chart.showDataLabels">none</option> <option name="charting.chart.sliceCollapsingThreshold">0.01</option> <option name="charting.chart.stackMode">default</option> <option name="charting.chart.style">shiny</option> <option name="charting.drilldown">none</option> <option name="charting.layout.splitSeries">0</option> <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option> <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option> <option name="charting.legend.mode">standard</option> <option name="charting.legend.placement">right</option> <option name="charting.lineWidth">2</option> <option name="refresh.display">progressbar</option> <option name="trellis.enabled">0</option> <option name="trellis.scales.shared">1</option> <option name="trellis.size">medium</option> </chart> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $truncation$"> <title>Potential truncaton assessment (samples per event size)</title> <table> <search base="truncation_base"> <query> | sort 0 - raw_size</query> </search> <option name="count">100</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $fieldsample$"> <title>Log sample (index="$index$" sourcetype="$sourcetype$")</title> <input type="text" token="search_field" searchWhenChanged="true"> <label>search ...</label> <default>*</default> <initialValue>*</initialValue> </input> <input type="dropdown" token="head_count" searchWhenChanged="true"> <label>head ( return X events)</label> <default>50</default> <initialValue>50</initialValue> <choice value="10">10</choice> <choice value="50">50</choice> <choice value="100">100</choice> <choice value="500">500</choice> <choice value="1000">1000</choice> </input> <input type="dropdown" token="sampling" searchWhenChanged="true"> <label>sampling</label> <default>1</default> <initialValue>1</initialValue> <choice value="1">no sampling</choice> <choice value="10">1:10</choice> <choice value="100">1:100</choice> <choice value="1000">1:1000</choice> <choice value="10000">1:10000</choice> </input> <table> <search> <query>index="$index$" sourcetype="$sourcetype$" | search $search_field$ | fillnull value="no_punct" | stats count first(_raw) as raw_event, first(_time) as _time values(host) as host by punct | sort - count | head $head_count$ | table _time, raw_event, punct, count</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="drilldown">none</option> <option name="refresh.display">progressbar</option> </table> </panel> </row> <row> <panel depends="$index$ $sourcetype$ $fieldsummary$"> <title>fieldsummary - click on one field to check _time</title> <input type="text" token="field_name" searchWhenChanged="true"> <label>Field Name</label> <default>*</default> <initialValue>*</initialValue> </input> <input type="radio" token="show_key_fields" searchWhenChanged="true"> <label>Columns for sourcetype documentation</label> <choice value="| sort field">Default columns</choice> <default>| sort field</default> <initialValue>| sort field</initialValue> <fieldForLabel>label</fieldForLabel> <fieldForValue>option</fieldForValue> <search> <query>| makeresults | eval label="Only key columns" | eval option="| search NOT field IN (\"field\",\"distinct_count\",\"coverage\",\"data_type\",\"parsing_type\",\"quality_check\",\"description\",\"index\",\"sourcetype\",\"source\",\"splunk*\",\"tag*\",\"eventtype\",\"punct\",\"linecount\",\"timeendpos\",\"timestartpos\") | table field, distinct_count, coverage, data_type, parsing_type, quality_check, description values | sort field"</query> </search> </input> <table> <search> <query>index="$index$" sourcetype="$sourcetype$" | head 100000 | fields - date_* | fieldsummary | eventstats max(count) as event_count | eval coverage=round(count/event_count*100,2) | sort - coverage | rex field=values max_match=0 "value\":\"(?P<value>[^\"]+)\"" | eval data_type=if(match(value,"^\d+$"),"Number","String") | fields - value | search field="$field_name$" $show_key_fields$</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">100</option> <option name="dataOverlayMode">none</option> <option name="drilldown">cell</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> <drilldown> <condition match="match('click.name2', "field""> <set token="form.event_time_field">$click.value2$</set> </condition> </drilldown> </table> </panel> </row> <row> <panel depends="$splunk_ta$"> <title>List of stanza defined in $splunk_ta$ (click on stanza)</title> <table> <search> <query>| union maxtime=300 timeout=300 [| rest splunk_server="local" "/servicesNS/-/-/data/props/calcfields"] [| rest splunk_server="local" "/servicesNS/-/-/data/props/fieldaliases"] [| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions"] | rename eai:acl.app as app | where app=="$splunk_ta$" | stats count by stanza | fields - count</query> <earliest>-15m</earliest> <latest>now</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="percentagesRow">false</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> <option name="drilldown">cell</option> <drilldown> <condition match="match('click.name2', "stanza""> <set token="form.sourcetype">$click.value2$</set> </condition> </drilldown> </table> </panel> </row> <row> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <html> <body bgcolor="#88FF88"> <h1>Running configuration for $splunk_ta$</h1> <p>Following panels provide a view on current running configuration for sourcetype $sourcetype$.</p> <h3> <a href="https://docs.splunk.com/Documentation/Splunk/9.0.4/Knowledge/Searchtimeoperationssequence">The sequence of search-time operations</a> </h3> <table> <tr> <th>Search-time operation order</th> <th>Operation name</th> <th>Configurable in Splunk Web?</th> <th>Location of file configuration</th> </tr> <tr> <td>1</td> <td>Role-based field filtering</td> <td>No</td> <td>fieldFilter-<fieldname> in a stanza in the authorize.conf file.</td> </tr> <tr> <td>2</td> <td>Inline field extraction (no field transform)</td> <td>Yes</td> <td>EXTRACT-<class> in a stanza in the props.conf file.</td> </tr> <tr> <td>3</td> <td>Field extraction that uses a field transform</td> <td>Yes</td> <td>REPORT-<class> in a stanza in the props.conf file.</td> </tr> <tr> <td>4</td> <td>Automatic key-value field extraction</td> <td>No</td> <td>In stanzas in the props.conf file, where KV_MODE is set to a valid value other than none. If no KV_MODE value is specified for a stanza, it is set to auto by default.</td> </tr> <tr> <td>5</td> <td>Field aliasing</td> <td>Yes</td> <td>FIELDALIAS-<class> in a stanza in the props.conf file.</td> </tr> <tr> <td>6</td> <td>Calculated fields</td> <td>Yes</td> <td>EVAL-<fieldname> in a stanza in the props.conf file.</td> </tr> <tr> <td>7</td> <td>Lookups</td> <td>Yes</td> <td>LOOKUP-<class> in a stanza in the props.conf file.</td> </tr> <tr> <td>8</td> <td>Event types</td> <td>Yes</td> <td>In a stanza in the eventtypes.conf file.</td> </tr> <tr> <td>9</td> <td>Tags</td> <td>Yes</td> <td>In a stanza in the tags.conf file.</td> </tr> </table> </body> </html> </panel> </row> <row> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>Configuration files for $splunk_ta$</title> <input type="text" token="search_conf_keyword" searchWhenChanged="true" depends="$splunk_ta$ $show_field_conf$"> <label>Search in conf</label> <default>*</default> <initialValue>*</initialValue> </input> </panel> </row> <row> <panel depends="$sourcetype$ $splunk_ta$ $show_field_conf$"> <title>2 - Inline field extraction (no field transform) for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR value="$search_conf_keyword$")</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions" | eval Type="extractions" | rex field=title " : (?<type>(EXTRACT|REPORT))-(?<title>[^\e]+)" | rename eai:acl.app as app | where app=="$splunk_ta$" AND stanza LIKE "%$sourcetype$%" | search attribute="$search_conf_keyword$" OR value="$search_conf_keyword$" | table stanza, type, attribute, value</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> <panel depends="$sourcetype$ $splunk_ta$ $show_field_conf$"> <title>3a - Transforms for $splunk_ta$ (limited to "$search_conf_keyword$" OR REGEX="$search_conf_keyword$" OR DEST_KEY="$search_conf_keyword$" OR FORMAT="$search_conf_keyword$"))</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/transforms/extractions" | rename eai:acl.app as app | where app=="$splunk_ta$" | eval type=if(isnull(DELIMS),"regex","delimiter") | search title="$search_conf_keyword$" OR REGEX="$search_conf_keyword$" OR DEST_KEY="$search_conf_keyword$" OR FORMAT="$search_conf_keyword$" OR FIELDS="$search_conf_keyword$" | table title, type, SOURCE_KEY, DELIMS, REGEX, DEST_KEY, FORMAT, FIELDS</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> <row> <panel depends="$sourcetype$ $splunk_ta$ $show_field_conf$"> <title>3 - Field extraction that uses a field transform - Combined REPORT (Step1) and TRANSFORMS (Step2) for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR value="$search_conf_keyword$")</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/props/extractions" | eval Type="extractions" | rex field=title " : (?<type>(EXTRACT|REPORT))-(?<title>[^\e]+)" | rename eai:acl.app as app | table app, stanza, type, attribute, value | search app="*" OR ( [| rest splunk_server="local" "/servicesNS/-/-/data/transforms/extractions" | rename eai:acl.app as app | where app=="$splunk_ta$" | eval type=if(isnull(DELIMS),"regex","delimiter") | table title, type, SOURCE_KEY, DELIMS, REGEX, DEST_KEY, FORMAT, FIELDS | outputlookup LD_DATA_transforms_list] ) | search app="$splunk_ta$" AND stanza="*$sourcetype$*" | eval title=split(value,","), value=replace(value,",",", ") | mvexpand title | fields - app | lookup LD_DATA_transforms_list title OUTPUT | search attribute="$search_conf_keyword$" OR value="$search_conf_keyword$" OR title="$search_conf_keyword$" OR REGEX="$search_conf_keyword$" OR DEST_KEY="$search_conf_keyword$" OR FORMAT="$search_conf_keyword$" OR FIELDS="$search_conf_keyword$" | table stanza, type, attribute, value, title, *</query> <earliest>-15m</earliest> <latest>now</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> <row> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>5 - Field aliases for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR value="$search_conf_keyword$")</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/props/fieldaliases" | rex field=title " : [^\-]+-(?<title>[^\e]+)" | rename eai:acl.app as app | where app=="$splunk_ta$" AND stanza LIKE "%$sourcetype$%" | search attribute="$search_conf_keyword$" OR value="$search_conf_keyword$" | table stanza, attribute, value</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>6 - Calculated fields for $sourcetype$ and $splunk_ta$ (limited to attribute="$search_conf_keyword$" OR field_name="$search_conf_keyword$" OR value="$search_conf_keyword$")</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/props/calcfields" | rex field=title " : [^\-]+-(?<title>[^\e]+)" | rename eai:acl.app as app, field.name as field_name | where app=="$splunk_ta$" AND stanza LIKE "%$sourcetype$%" | search attribute="$search_conf_keyword$" OR field_name="$search_conf_keyword$" OR value="$search_conf_keyword$" | table stanza, attribute, field_name, value</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> <row> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>7 - Automatic lookups</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/props/lookups" | rex field=title " : [^\-]+-(?<title>[^\e]+)" | rename eai:acl.app as app, value as Lookup | where app=="$splunk_ta$" AND stanza LIKE "%$sourcetype$%" | eval Name=stanza + " : " + attribute | table Name, Lookup</query> <earliest>-15m</earliest> <latest>now</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>8 - Event types for $splunk_ta$ (limited to search="$search_conf_keyword$") - click on search to launch CIM validator</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/saved/eventtypes" | eval Type="Event Types" | eval tags=mvjoin(tags, ",") | rename eai:acl.app as app | where app=="$splunk_ta$" | search search="$search_conf_keyword$" OR title="$search_conf_keyword$" OR tags="$search_conf_keyword$" | table title, search, tags</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">cell</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> <drilldown> <link target="_blank">/app/SA-cim_vladiator/cim_validator?form.cim_search=$click.value2$</link> </drilldown> </table> </panel> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>9 - Tags for $splunk_ta$ (limited to title="$search_conf_keyword$")</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/saved/fvtags" | rename eai:acl.app as app | where app=="$splunk_ta$" | search title="$search_conf_keyword$" OR tags="$search_conf_keyword$" | table tags, title</query> <earliest>$GlobalTimePicker.earliest$</earliest> <latest>$GlobalTimePicker.latest$</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="refresh.display">progressbar</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> <row> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>Lookup tables</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/lookup-table-files" | rename eai:acl.app as app | where app=="$splunk_ta$" | table title</query> <earliest>-15m</earliest> <latest>now</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> <panel depends="$splunk_ta$ $sourcetype$ $show_field_conf$"> <title>Lookup definitions</title> <table> <search> <query>| rest splunk_server="local" "/servicesNS/-/-/data/transforms/lookups" | rename eai:acl.app as app | where app=="$splunk_ta$" | table title, type, filename, fields_list</query> <earliest>-15m</earliest> <latest>now</latest> <sampleRatio>1</sampleRatio> </search> <option name="count">5</option> <option name="dataOverlayMode">none</option> <option name="drilldown">none</option> <option name="percentagesRow">false</option> <option name="rowNumbers">false</option> <option name="totalsRow">false</option> <option name="wrap">true</option> </table> </panel> </row> </form>