Pastebin
Paste #2521: perfmon.py
< previous paste - next paste>
Pasted by tdn@barbera
#!/usr/bin/python
#encoding=utf8
import os
import sys
import time
import datetime
import subprocess
from time import sleep
DATE_FMT = "%m/%d/%Y %H:%M:%S.%f"
def ts():
"""Return timestamp"""
return datetime.datetime.now().strftime(DATE_FMT)
#return time.strftime(DATE_FMT, time.gmtime())
def loadavg():
"""Returns a tuple of:
1: 1 minute load average
2: 5 minute load average
3: 15 minute load average
"""
# The first three fields in this file are load average figures giving the
# number of jobs in the run queue (state R) or waiting for disk I/O (state D)
# averaged over 1, 5, and 15 minutes. They are the same as the load average
# numbers given by uptime(1) and other programs.
#
# The fourth field consists of two numbers separated by a slash (/).
# The first of these is the number of
# currently executing kernel scheduling entities (processes, threads);
# this will be less than or equal to the number of CPUs. The value after the
# slash is the number of kernel scheduling entities that currently exist on
# the system.
#
# The fifth field is the PID of the process that was most recently created
# on the system.
fields = open("/proc/loadavg").read().strip().split(" ")
return (fields[0], fields[1], fields[2])
def netoutq():
"""Return network output queue length"""
return None
def disk_queue_length(disks=[]):
"""Returns number of I/Os currently in progress"""
lines = open("/proc/diskstats").readlines()
# 8 0 sda 51091 19748 3923782 457240 174218 3000 9840090 357040 0 124156 812280
# 8 1 sda1 450 189 3558 688 9 0 18 24 0 712 712
# 8 2 sda2 2 0 12 8 0 0 0 0 0 8 8
# 8 5 sda5 50472 19559 3918876 456464 166441 3000 9840072 353168 0 120024 807640
# 11 0 sr0 65 65 520 44 0 0 0 0 0 44 44
# 252 0 dm-0 70030 0 3917802 1562240 177209 0 9840072 2981668 0 201388 4543920
# 252 1 dm-1 69704 0 3915194 1561504 171045 0 9839992 2982604 0 201508 4544116
# 252 2 dm-2 210 0 1680 320 10 0 80 108 0 116 428
r = {}
for line in lines:
t = line.split()
if t[2] in disks:
r[t[2]] = t[11]
return r
def enumerate_partitions():
"""Returns a list of partitions"""
p = []
for line in open("/proc/partitions").readlines()[2:]:
p.append(line.strip().split()[3])
return p
def shell_exec(cmd):
"""Executes a command in a shell and returns STDOUT"""
return subprocess.check_output(cmd, shell=True)
def vmstat():
# memory pages in/out: si/so columns from vmstat output
# procs -----------memory---------- ---swap-- -----io---- -system-- ----cpu----
# r b swpd free buff cache si so bi bo in cs us sy id wa
# 1 0 0 2087724 121672 871612 0 0 14 15 234 192 2 5 93 0
#
# FIELD DESCRIPTION FOR VM MODE
# Procs
# r: The number of processes waiting for run time.
# b: The number of processes in uninterruptible sleep.
#
# Memory
# swpd: the amount of virtual memory used.
# free: the amount of idle memory.
# buff: the amount of memory used as buffers.
# cache: the amount of memory used as cache.
# inact: the amount of inactive memory. (-a option)
# active: the amount of active memory. (-a option)
#
# Swap
# si: Amount of memory swapped in from disk (/s).
# so: Amount of memory swapped to disk (/s).
#
# IO
# bi: Blocks received from a block device (blocks/s).
# bo: Blocks sent to a block device (blocks/s).
#
# System
# in: The number of interrupts per second, including the clock.
# cs: The number of context switches per second.
#
# CPU
# These are percentages of total CPU time.
# us: Time spent running non-kernel code. (user time, including nice time)
# sy: Time spent running kernel code. (system time)
# id: Time spent idle. Prior to Linux 2.5.41, this includes IO-wait time.
# wa: Time spent waiting for IO. Prior to Linux 2.5.41, included in idle.
# st: Time stolen from a virtual machine. Prior to Linux 2.6.11, unknown.
cmd = "vmstat"
lines = shell_exec(cmd).split("\n")
fields = lines[2].split()
r = {
"num_proc_wait" : fields[ 0],
"num_proc_unintr_sleep" : fields[ 1],
"mem_swpd" : fields[ 2],
"mem_free" : fields[ 3],
"mem_buff" : fields[ 4],
"mem_cache" : fields[ 5],
"swapin_rate" : fields[ 6],
"swapout_rate" : fields[ 7],
"io_blks_recv" : fields[ 8],
"io_blks_sent" : fields[ 9],
"sys_intrs_rate" : fields[10],
"sys_ctx_switch_rate" : fields[11],
"cpu_user" : fields[12],
"cpu_sys" : fields[13],
"cpu_idle" : fields[14],
"cpu_wait" : fields[15]
}
return r
def iostat(disks):
#Linux 3.2.0-26-generic (barbera) 2012-07-12 _x86_64_ (8 CPU)
#
#avg-cpu: %user %nice %system %iowait %steal %idle
# 1,71 0,12 5,25 0,05 0,00 92,87
#
#Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
#sda 1,68 0,15 2,05 9,26 80,31 88,06 29,77 0,06 5,23 15,50 2,97 0,48 0,54
#scd0 0,01 0,00 0,01 0,00 0,03 0,00 8,00 0,00 1,04 1,04 0,00 1,04 0,00
#dm-0 0,00 0,00 3,67 9,42 80,11 88,06 25,71 0,22 17,03 29,38 12,21 0,55 0,72
#dm-1 0,00 0,00 3,64 9,15 79,99 88,06 26,29 0,22 17,41 29,56 12,58 0,57 0,72
#dm-2 0,00 0,00 0,02 0,00 0,08 0,00 8,00 0,00 11,75 11,75 0,00 0,53 0,00
# rrqm/s
# The number of read requests merged per second that were queued to
# the device.
# wrqm/s
# The number of write requests merged per second that were queued to
# the device.
# r/s
# The number of read requests that were issued to the device per
# second.
# w/s
# The number of write requests that were issued to the device per
# second.
# rkB/s
# The number of kilobytes read from the device per second.
# wkB/s
# The number of kilobytes written to the device per second.
# avgrq-sz
# The average size (in sectors) of the requests that were issued to
# the device.
# avgqu-sz
# The average queue length of the requests that were issued to the
# device.
# await
# The average time (in milliseconds) for I/O requests issued to the
# device to be served. This includes the time spent by the requests
# in queue and the time spent servicing them.
# svctm
# The average service time (in milliseconds) for I/O requests that
# were issued to the device.
# %util
# Percentage of CPU time during which I/O requests were issued to
# the device (bandwidth utilization for the device). Device satura‐
# tion occurs when this value is close to 100%.
cmd = "iostat -x"
lines = shell_exec(cmd).strip().split("\n")
r = {}
for line in lines[6:]:
fields = line.split()
if fields[0] in disks:
t = {
"io_read_rate" : fields[5],
"io_write_rate" : fields[6],
"io_avg_queue_length" : fields[8],
"io_utilization_pct" : fields[13]
}
r[fields[0]] = t
return r
def vmstat_s():
cmd = "vmstat -s"
lines = shell_exec(cmd).strip().split("\n")
r = {}
for line in lines:
val = line.strip().split()[0]
key = "_".join(line.strip().split()[1:]).lower()
r[key] = val
return r
disks = enumerate_partitions()
# Timestamp
# 1 minute load average
# 5 minute load average
# 15 minute load average
# disk queue lenghts per disk
delay = None
max_iterations = False
iteration = 0
try:
delay = float(sys.argv[1])
max_iterations = int(sys.argv[2])
except Exception, e:
pass
available_counters = {
"loadavg_1" : "Average load last 1 minutes. Load average is a gauge of how many processes are on average, concurrently demanding CPU attention.",
"loadavg_5" : "Average load last 5 minutes. Load average is a gauge of how many processes are on average, concurrently demanding CPU attention.",
"loadavg_15" : "Average load last 15 minutes. Load average is a gauge of how many processes are on average, concurrently demanding CPU attention.",
"disk_queue_length" : "I/Os currently in progress for each drive. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.",
"num_proc_wait" : "The number of processes waiting for run time.",
"num_proc_unintr_sleep" : "The number of processes in uninterruptible sleep.",
"mem_swpd" : "the amount of virtual memory used.",
"mem_free" : "The amount of idle memory.",
"mem_buff" : "the amount of memory used as buffers.",
"mem_cache" : "the amount of memory used as cache.",
"swapin_rate" : "Amount of memory swapped in from disk (/s).",
"swapout_rate" : "Amount of memory swapped to disk (/s).",
"io_blks_recv" : "Blocks received from a block device (blocks/s)",
"io_blks_sent" : "Blocks sent to a block device (blocks/s).",
"sys_intrs_rate": "The number of interrupts per second, including the clock.",
"sys_ctx_switch_rate" : "The number of context switches per second.",
"cpu_user" : "% Time spent running non-kernel code. (user time, including nice time)",
"cpu_sys" : "% time spent running kernel code. (system time)",
"cpu_idle" : "% Time spent idle. Prior to Linux 2.5.41, this includes IO-wait time.",
"cpu_wait" : "% time spent waiting for IO. Prior to Linux 2.5.41, included in idle.",
"io_read_rate" : "The number of kilobytes read from the device per second.",
"io_write_rate" : "The number of kilobytes written to the device per second.",
"io_avg_queue_length": "The average queue length of the requests that were issued to the device.",
"io_utilization_pct" : "Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%."
}
selected_counters = [x for x in available_counters]
#print selected_counters
def addcol(c, first=False):
"""Add column to CSV line"""
r = '"%s"' % (c)
if not first: r = ","+r
return r
while not max_iterations or max_iterations > iteration:
try:
iteration += 1
head = ""
line = ""
line += addcol(ts(), first=True)
head += addcol("Timestamp", first=True)
loadavg_col= loadavg()
if "loadavg_1" in selected_counters:
head += addcol("loadavg_1")
line += addcol(loadavg_col[0])
if "loadavg_5" in selected_counters:
head += addcol("loadavg_5")
line += addcol(loadavg_col[1])
if "loadavg_15" in selected_counters:
head += addcol("loadavg_15")
line += addcol(loadavg_col[2])
vmstat_cols = vmstat()
iostat_cols = iostat(disks)
mem_cols = vmstat_s()
if "cpu_user" in selected_counters:
head += addcol("cpu_user")
line += addcol(vmstat_cols["cpu_user"])
if "cpu_idle" in selected_counters:
head += addcol("cpu_idle")
line += addcol(vmstat_cols["cpu_idle"])
if "cpu_sys" in selected_counters:
head += addcol("cpu_sys")
line += addcol(vmstat_cols["cpu_sys"])
if "cpu_wait" in selected_counters:
head += addcol("cpu_wait")
line += addcol(vmstat_cols["cpu_wait"])
if "disk_queue_length" in selected_counters:
disk_ql_col = disk_queue_length(disks)
for disk in disks:
line += addcol((disk_ql_col[disk]))
head += addcol("disk_queue_length: %s" % (disk))
if "io_avg_queue_length" in selected_counters:
for disk in disks:
try:
line += addcol(iostat_cols[disk]["io_avg_queue_length"])
head += addcol("io_avg_queue_length: %s" % (disk))
except KeyError:
pass
if "io_read_rate" in selected_counters:
for disk in disks:
try:
line += addcol(iostat_cols[disk]["io_read_rate"])
head += addcol("io_read_rate: %s" % (disk))
except KeyError:
pass
if "io_utilization_pct" in selected_counters:
for disk in disks:
try:
line += addcol(iostat_cols[disk]["io_utilization_pct"])
head += addcol("io_utilization_pct: %s" % (disk))
except KeyError:
pass
if "io_write_rate" in selected_counters:
for disk in disks:
try:
line += addcol(iostat_cols[disk]["io_write_rate"])
head += addcol("io_write_rate: %s" % (disk))
except KeyError:
pass
if "io_avg_queue_length" in selected_counters:
for disk in disks:
try:
line += addcol(iostat_cols[disk]["io_avg_queue_length"])
head += addcol("io_avg_queue_length: %s" % (disk))
except KeyError:
pass
if "io_avg_queue_length" in selected_counters:
for disk in disks:
try:
line += addcol(iostat_cols[disk]["io_avg_queue_length"])
head += addcol("io_avg_queue_length: %s" % (disk))
except KeyError:
pass
if "io_blks_recv" in selected_counters:
head += addcol("io_blks_recv")
line += addcol(vmstat_cols["io_blks_recv"])
if "io_blks_sent" in selected_counters:
head += addcol("io_blks_sent")
line += addcol(vmstat_cols["io_blks_sent"])
if "mem_buff" in selected_counters:
head += addcol("mem_buff")
line += addcol(vmstat_cols["mem_buff"])
if "mem_cache" in selected_counters:
head += addcol("mem_cache")
line += addcol(vmstat_cols["mem_cache"])
if "mem_free" in selected_counters:
head += addcol("mem_free")
line += addcol(vmstat_cols["mem_free"])
if "mem_swpd" in selected_counters:
head += addcol("mem_swpd")
line += addcol(vmstat_cols["mem_swpd"])
if "num_proc_unintr_sleep" in selected_counters:
head += addcol("num_proc_unintr_sleep")
line += addcol(vmstat_cols["num_proc_unintr_sleep"])
if "num_proc_wait" in selected_counters:
head += addcol("num_proc_wait")
line += addcol(vmstat_cols["num_proc_wait"])
if "swapin_rate" in selected_counters:
head += addcol("swapin_rate")
line += addcol(vmstat_cols["swapin_rate"])
if "swapout_rate" in selected_counters:
head += addcol("swapout_rate")
line += addcol(vmstat_cols["swapout_rate"])
if "sys_ctx_switch_rate" in selected_counters:
head += addcol("sys_ctx_switch_rate")
line += addcol(vmstat_cols["sys_ctx_switch_rate"])
if "sys_intrs_rate" in selected_counters:
head += addcol("sys_intrs_rate")
line += addcol(vmstat_cols["sys_intrs_rate"])
#print """"%s","%s","%s","%s",%s""" % (t, t_load[0], t_load[1], t_load[2], disk_ql_cols[:-1] )
if iteration == 1:
print head
print line
if max_iterations and max_iterations <= iteration:
sys.exit()
if delay:
sleep(delay)
else:
sys.exit()
except KeyboardInterrupt:
sys.exit()
New Paste
Go to most recent paste.
