Description: Use this Splunk search to find Base64 encoded content in DNS queries. The goal is to examine the DNS query field of the dns events to find subdomain streams that contain only Base64 valid characters. Utilizing DNS queries with encoded information is a known method to exfiltrate data. But you do not know if […]
Show cron frequency and scheduling of all scheduled searches
This search shows you all scheduled searches and their respective cron frequency and cron schedule. This also helps finding frequently running saved searches.
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 |
| rest splunk_server=local "/servicesNS/-/-/saved/searches/" search="is_scheduled=1" search="disabled=0" | fields title, cron_schedule, eai:acl.app | rename title as savedsearch_name | eval pieces=split(cron_schedule, " ") | eval c_min=mvindex(pieces, 0), c_h=mvindex(pieces, 1), c_d=mvindex(pieces, 2), c_mday=mvindex(pieces, 3), c_wday=mvindex(pieces, 4) | eval c_min_div=if(match(c_min, "/"), replace(c_min, "^.*/(\d+)$", "\1"), null()) | eval c_mins=if(match(c_min, ","), split(c_min, ","), null()) | eval c_min_div=if(isnotnull(c_mins), abs(tonumber(mvindex(c_mins, 1)) - tonumber(mvindex(c_mins, 0))), c_min_div) | eval c_hs=if(match(c_h, ","), split(c_h, ","), null()) | eval c_h_div=case(match(c_h, "/"), replace(c_h, "^.*/(\d+)$", "\1"), isnotnull(c_hs), abs(tonumber(mvindex(c_hs, 1)) - tonumber(mvindex(c_hs, 0))), 1=1, null()) | eval c_wdays=if(match(c_wday, ","), split(c_wday, ","), null()) | eval c_wday_div=case(match(c_wday, "/"), replace(c_wday, "^.*/(\d+)$", "\1"), isnotnull(c_wdays), abs(tonumber(mvindex(c_wdays, 1)) - tonumber(mvindex(c_wdays, 0))), 1=1, null()) | eval i_m=case(c_d < 29, 86400 * 28, c_d = 31, 86400 * 31, 1=1, null()) | eval i_h=case(isnotnull(c_h_div), c_h_div * 3600, c_h = "*", null(), match(c_h, "^\d+$"), 86400) | eval i_min=case(isnotnull(c_min_div), c_min_div * 60, c_min = "*", 60, match(c_min, "^\d+$"), 3600) | eval i_wk=case(isnotnull(c_wday_div), c_wday_div * 86400, c_wday = "*", null(), match(c_wday, "^\d+$"), 604800) | eval cron_minimum_freq=case(isnotnull(i_m), i_m, isnotnull(i_wk) AND isnotnull(c_min_div), i_min, isnotnull(i_wk) AND isnull(c_min_div), i_wk, isnotnull(i_h), i_h, 1=1, min(i_min)) | fields - c_d c_h c_hs c_h_div c_mday c_min c_min_div c_mins c_wday c_wdays c_wday_div pieces i_m i_min i_h i_wk | fields savedsearch_name cron_minimum_freq cron_schedule eai:acl.app |
Data model Acceleration Details
This Splunk Search shows you a lot of good information about your data model acceleration and performance.
|
1 |
| rest /services/admin/summarization by_tstats=t splunk_server=local count=0 <br />| eval key=replace(title,(("tstats:DM_" . 'eai:acl.app') . "_"),""), datamodel=replace('summary.id',(("DM_" . 'eai:acl.app') . "_"),"") <br />| join type=left key <br /> [| rest /services/data/models splunk_server=local count=0 <br /> | table title, "acceleration.cron_schedule", "eai:digest" <br /> | rename title as key <br /> | rename "acceleration.cron_schedule" as cron] <br />| table datamodel, "eai:acl.app", "summary.access_time", "summary.is_inprogress", "summary.size", "summary.latest_time", "summary.complete", "summary.buckets_size", "summary.buckets", cron, "summary.last_error", "summary.time_range", "summary.id", "summary.mod_time", "eai:digest", "summary.earliest_time", "summary.last_sid", "summary.access_count" <br />| rename "summary.id" as summary_id, "summary.time_range" as retention, "summary.earliest_time" as earliest, "summary.latest_time" as latest, "eai:digest" as digest <br />| rename "summary.*" as "*", "eai:acl.*" as "*" <br />| sort datamodel <br />| rename access_count as "Datamodel_Acceleration.access_count", access_time as "Datamodel_Acceleration.access_time", app as "Datamodel_Acceleration.app", buckets as "Datamodel_Acceleration.buckets", buckets_size as "Datamodel_Acceleration.buckets_size", cron as "Datamodel_Acceleration.cron", complete as "Datamodel_Acceleration.complete", datamodel as "Datamodel_Acceleration.datamodel", digest as "Datamodel_Acceleration.digest", earliest as "Datamodel_Acceleration.earliest", is_inprogress as "Datamodel_Acceleration.is_inprogress", last_error as "Datamodel_Acceleration.last_error", last_sid as "Datamodel_Acceleration.last_sid", latest as "Datamodel_Acceleration.latest", mod_time as "Datamodel_Acceleration.mod_time", retention as "Datamodel_Acceleration.retention", size as "Datamodel_Acceleration.size", summary_id as "Datamodel_Acceleration.summary_id" <br />| fields + "Datamodel_Acceleration.access_count", "Datamodel_Acceleration.access_time", "Datamodel_Acceleration.app", "Datamodel_Acceleration.buckets", "Datamodel_Acceleration.buckets_size", "Datamodel_Acceleration.cron", "Datamodel_Acceleration.complete", "Datamodel_Acceleration.datamodel", "Datamodel_Acceleration.digest", "Datamodel_Acceleration.earliest", "Datamodel_Acceleration.is_inprogress", "Datamodel_Acceleration.last_error", "Datamodel_Acceleration.last_sid", "Datamodel_Acceleration.latest", "Datamodel_Acceleration.mod_time", "Datamodel_Acceleration.retention", "Datamodel_Acceleration.size", "Datamodel_Acceleration.summary_id" <br />| rename "Datamodel_Acceleration.*" as "*" <br />| join type=outer last_sid <br /> [| rest splunk_server=local count=0 /services/search/jobs reportSearch=summarize* <br /> | rename sid as last_sid <br /> | fields + last_sid, runDuration] <br />| eval "size(MB)"=round((size / 1048576),1), "retention(days)"=if((retention == 0),"unlimited",round((retention / 86400),1)), "complete(%)"=round((complete * 100),1), "runDuration(s)"=round(runDuration,1) <br />| sort 100 + datamodel <br />| table datamodel, app, cron, "retention(days)", earliest, latest, is_inprogress, "complete(%)", "size(MB)", "runDuration(s)", last_error |
Remove mulitple values from a multivalue field
This Splunk search is an example on how to remove unwanted values from multivalue fields using mvfilter.
|
1 |
| gentimes start=-1 | eval field1="pink,fluffy,unicorns" | table field1 | makemv field1 delim="," | eval field1_filtered=mvfilter(NOT match(field1,"pink") AND NOT match(field1,"fluffy")) |
List all your existing indexes or check if index exists
With this spl you can check what indexes exist or if you want to search for a specific index. List all indexes:
|
1 |
|rest /services/data/indexes | fields title | rename title AS index |
Or check if a specific index exist use:
|
1 |
|rest /services/data/indexes | fields title | rename title AS index | search index=yourindex |
Datamodel Search Performance
See how well your DM searches are running. Run this search using the Line Chart visualization:
|
1 2 3 |
index=_internal sourcetype=scheduler component=SavedSplunker ACCELERATE NOT skipped run_time=* | rex field=savedsearch_id "ACCELERATE_(?:[A-F0-9\-]{36}_)?(?<acceleration>.*?)_ACCELERATE" | timechart span=5m max(run_time) AS run_time by acceleration |
Breathing Fire Dragon when Starting dbx_task_server
|
1 |
index=_internal sourcetype=dbx_server Starting dbx_task_server |
Will return events that display a little dragon ascii art:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
|\___/| (,\ /,)\ / / \ (@_^_@)/ \ W//W_/ \ (//) | \ (/ /) _|_ / ) \ (// /) '/,_ _ _/ (~^-. (( // )) ,-{ _ `. (( /// )) '/\ / | (( ///)) `. { } ((/ )) .----~-.\ \-' ///.----..> \ ///-._ _ _ _} |
Show your triggered alerts
This search shows all the alerts that where triggered in your splunk environment:
|
1 |
index=_audit action=alert_fired ss_app=* | eval ttl=expiration-now() | search ttl>0 | convert ctime(trigger_time) | table trigger_time ss_name severity | rename trigger_time as "Alert Time" ss_name as "Alert Name" severity as "Severity" |
Evaluate Fahrenheit to Celsius
Quick snippet to evaluate temperature Fahrentheit to Celsius:
|
1 |
| eval Temperature_Fahrenheit=Temperature_Celsius*1.8+32 |
Find unused dashboards
Use this search to find unused dashboards:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| rest /servicesNS/-/-/data/ui/views splunk_server=* | search isDashboard=1 | rename eai:acl.app as app | fields title app | join type=left title [| search index=_internal sourcetype=splunk_web_access host=* user=* | rex field=uri_path ".*/(?<title>[^/]*)$" | stats latest(_time) as Time latest(user) as user by title ] | where isnotnull(Time) | eval Now=now() | eval "Days since last accessed"=round((Now-Time)/86400,2) | sort - "Days since last accessed" | convert ctime(Time) | fields - Now |
Admin Notes – Fantastic query! I modified the SPL slightly as I had an issue when I copied it to my two test environments.
Check your strftime is correct in the props.conf
A simple method on checking if your strftime (TIME_FORMAT=) in the props.conf matches your log file timestamp format. strftime(X,Y) This function takes a UNIX time value, X, as the first argument and renders the time as a string using the format specified by Y. The UNIX time must be in seconds. Use the first 10 […]
Saved Search Scheduler Activity
I use this query a lot to tune and adjust scheduling, find out what searches need attention:
|
1 |
index=_internal sourcetype=scheduler result_count | extract pairdelim=",", kvdelim="=", auto=f | stats avg(result_count) min(result_count) max(result_count), sparkline avg(run_time) min(run_time) max(run_time) sum(run_time) values(host) AS hosts count AS execution_count by savedsearch_name, app | join savedsearch_name type=outer [| rest /servicesNS/-/-/saved/searches | fields title eai:acl.owner cron_schedule dispatch.earliest_time dispatch.latest_time search | rename title AS savedsearch_name eai:acl.app AS App eai:acl.owner AS Owner cron_schedule AS "Cron Schedule" dispatch.earliest_time AS "Dispatch Earliest Time" dispatch.latest_time AS "Dispatch Latest Time"]| rename savedsearch_name AS "Saved Search Name" search AS "SPL Query" app AS App | makemv delim="," values(host) | sort - avg(run_time) | table "Saved Search Name", App, Owner, "SPL Query", "Dispatch Earliest Time" "Dispatch Latest Time" "Cron Schedule" hosts, execution_count, sparkline, *(result_count), sum(run_time) *(run_time) |
Show indexing queue sizes
Use a linechart with this search to show you the indexing queue sizes:
|
1 |
index=_internal source=*metrics.log group=queue (name=parsingqueue OR name=indexqueue OR name=typingqueue OR name=aggqueue) | timechart avg(current_size) by name |
Percentage of skipped searches
This query will give you a table with a percentage of skipped searches and an evaluation with 3 ranges
|
1 |
index=_internal sourcetype=scheduler | stats count as total, count(eval(status="skipped")) as skipped | eval pct=round(skipped/total * 100, 0) | rangemap field=pct low=0-10, elevated=10-20 severe=20-100 | eval pct = pct . "%" | fields pct, range |
Retention Period in days per index
This query will give you a table of all indexes and their respective retention period in days:
|
1 2 3 4 |
| rest splunk_server=* /services/data/indexes | join type=outer title [ | rest splunk_server=* /services/data/indexes-extended ] | eval retentionInDays=frozenTimePeriodInSecs/86400 | table title retentionInDays |
Bucket Count by indexer/index
This search displays the amount of buckets per indexer/index To learn more about the | dbinspect command go to: http://docs.splunk.com/Documentation/Splunk/7.1.2/SearchReference/Dbinspect
|
1 |
|dbinspect index=* | search index!=_* | chart dc(bucketId) over splunk_server by index |
Bucket Count by State over Index
This search counts the amount of buckets per state for each index. To learn more about | dbinspect go to: http://docs.splunk.com/Documentation/Splunk/7.1.2/SearchReference/Dbinspect
|
1 |
|dbinspect index=* | eval state=case(state=="warm" OR state=="hot","hot/warm",1=1, state) | chart dc(bucketId) over index by state |
Memory Usage (MB) per Splunk Process Class
Use the following search with a column chart visualisation. It will give you a good overview of what Splunk processes use the most memory:
|
1 |
index=_introspection sourcetype=splunk_resource_usage component=PerProcess host=* | eval process = 'data.process' | eval args = 'data.args' | eval sid = 'data.search_props.sid' | eval elapsed = 'data.elapsed' | eval mem_used = 'data.mem_used' | eval mem = 'data.mem' | eval pct_memory = 'data.pct_memory' | eval app = 'data.search_props.app' | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval role = 'data.search_props.role' | eval process_class = case( process=="splunk-optimize","index service", process=="sh" OR process=="ksh" OR process=="bash" OR like(process,"python%") OR process=="powershell","scripted input", process=="mongod", "KVStore") | eval process_class = case( process=="splunkd" AND (like(args,"-p %start%") OR like(args,"service")),"splunkd server", process=="splunkd" AND isnotnull(sid),"search", process=="splunkd" AND (like(args,"fsck%") OR like(args,"recover-metadata%") OR like(args,"cluster_thing")),"index service", process=="splunkd" AND args=="instrument-resource-usage", "scripted input", (like(process,"python%") AND like(args,"%/appserver/mrsparkle/root.py%")) OR like(process,"splunkweb"),"Splunk Web", isnotnull(process_class), process_class) | eval process_class = if(isnull(process_class),"other",process_class) | stats latest(data.mem_used) AS resource_usage_dedup latest(process_class) AS process_class by data.pid, _time | stats sum(resource_usage_dedup) AS resource_usage by _time, process_class | timechart minspan=10s median(resource_usage) AS "Resource Usage" by process_class |
|
1 |
Universal Forwarder Throughput Limit Hit Count
This search counts the amount of times the UF’s throughput limit is hit. I also threw in a sparkline:
|
1 |
index=_internal sourcetype=splunkd "current data throughput" | rex "Current data throughput \((?<kb>\S+)" | eval rate=case(kb < 500, "256", kb > 499 AND kb < 520, "512", kb > 520 AND kb < 770 ,"768", kb>771 AND kb<1210, "1024", 1=1, ">1024") | stats count as Count sparkline as Trend by host, rate | where Count > 4 | rename host as "Host" rate as "Throughput rate(kb)" count as "Hit Count"| sort -"Throughput rate(kb)",-Count |
Top 10 Accessed Dashboards
Where “host=”your_sh_host”” you could specify a host, or put a wildcard * in place.
|
1 2 3 |
index="_internal" source=*access.log user!="-" */app/* (host="your_sh_host") | rex field=referer "/en-US/app/(?<app>[^/]+)/(?<dashboard>[^?/\s]+)" | top 10 dashboard |
