#!/usr/bin/python #encoding=utf8 import os import sys import time import datetime import subprocess from time import sleep DATE_FMT = "%m/%d/%Y %H:%M:%S.%f" def ts(): """Return timestamp""" return datetime.datetime.now().strftime(DATE_FMT) #return time.strftime(DATE_FMT, time.gmtime()) def loadavg(): """Returns a tuple of: 1: 1 minute load average 2: 5 minute load average 3: 15 minute load average """ # The first three fields in this file are load average figures giving the # number of jobs in the run queue (state R) or waiting for disk I/O (state D) # averaged over 1, 5, and 15 minutes. They are the same as the load average # numbers given by uptime(1) and other programs. # # The fourth field consists of two numbers separated by a slash (/). # The first of these is the number of # currently executing kernel scheduling entities (processes, threads); # this will be less than or equal to the number of CPUs. The value after the # slash is the number of kernel scheduling entities that currently exist on # the system. # # The fifth field is the PID of the process that was most recently created # on the system. fields = open("/proc/loadavg").read().strip().split(" ") return (fields[0], fields[1], fields[2]) def netoutq(): """Return network output queue length""" return None def disk_queue_length(disks=[]): """Returns number of I/Os currently in progress""" lines = open("/proc/diskstats").readlines() # 8 0 sda 51091 19748 3923782 457240 174218 3000 9840090 357040 0 124156 812280 # 8 1 sda1 450 189 3558 688 9 0 18 24 0 712 712 # 8 2 sda2 2 0 12 8 0 0 0 0 0 8 8 # 8 5 sda5 50472 19559 3918876 456464 166441 3000 9840072 353168 0 120024 807640 # 11 0 sr0 65 65 520 44 0 0 0 0 0 44 44 # 252 0 dm-0 70030 0 3917802 1562240 177209 0 9840072 2981668 0 201388 4543920 # 252 1 dm-1 69704 0 3915194 1561504 171045 0 9839992 2982604 0 201508 4544116 # 252 2 dm-2 210 0 1680 320 10 0 80 108 0 116 428 r = {} for line in lines: t = line.split() if t[2] in disks: r[t[2]] = t[11] return r def enumerate_partitions(): """Returns a list of partitions""" p = [] for line in open("/proc/partitions").readlines()[2:]: p.append(line.strip().split()[3]) return p def shell_exec(cmd): """Executes a command in a shell and returns STDOUT""" return subprocess.check_output(cmd, shell=True) def vmstat(): # memory pages in/out: si/so columns from vmstat output # procs -----------memory---------- ---swap-- -----io---- -system-- ----cpu---- # r b swpd free buff cache si so bi bo in cs us sy id wa # 1 0 0 2087724 121672 871612 0 0 14 15 234 192 2 5 93 0 # # FIELD DESCRIPTION FOR VM MODE # Procs # r: The number of processes waiting for run time. # b: The number of processes in uninterruptible sleep. # # Memory # swpd: the amount of virtual memory used. # free: the amount of idle memory. # buff: the amount of memory used as buffers. # cache: the amount of memory used as cache. # inact: the amount of inactive memory. (-a option) # active: the amount of active memory. (-a option) # # Swap # si: Amount of memory swapped in from disk (/s). # so: Amount of memory swapped to disk (/s). # # IO # bi: Blocks received from a block device (blocks/s). # bo: Blocks sent to a block device (blocks/s). # # System # in: The number of interrupts per second, including the clock. # cs: The number of context switches per second. # # CPU # These are percentages of total CPU time. # us: Time spent running non-kernel code. (user time, including nice time) # sy: Time spent running kernel code. (system time) # id: Time spent idle. Prior to Linux 2.5.41, this includes IO-wait time. # wa: Time spent waiting for IO. Prior to Linux 2.5.41, included in idle. # st: Time stolen from a virtual machine. Prior to Linux 2.6.11, unknown. cmd = "vmstat" lines = shell_exec(cmd).split("\n") fields = lines[2].split() r = { "num_proc_wait" : fields[ 0], "num_proc_unintr_sleep" : fields[ 1], "mem_swpd" : fields[ 2], "mem_free" : fields[ 3], "mem_buff" : fields[ 4], "mem_cache" : fields[ 5], "swapin_rate" : fields[ 6], "swapout_rate" : fields[ 7], "io_blks_recv" : fields[ 8], "io_blks_sent" : fields[ 9], "sys_intrs_rate" : fields[10], "sys_ctx_switch_rate" : fields[11], "cpu_user" : fields[12], "cpu_sys" : fields[13], "cpu_idle" : fields[14], "cpu_wait" : fields[15] } return r def iostat(disks): #Linux 3.2.0-26-generic (barbera) 2012-07-12 _x86_64_ (8 CPU) # #avg-cpu: %user %nice %system %iowait %steal %idle # 1,71 0,12 5,25 0,05 0,00 92,87 # #Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util #sda 1,68 0,15 2,05 9,26 80,31 88,06 29,77 0,06 5,23 15,50 2,97 0,48 0,54 #scd0 0,01 0,00 0,01 0,00 0,03 0,00 8,00 0,00 1,04 1,04 0,00 1,04 0,00 #dm-0 0,00 0,00 3,67 9,42 80,11 88,06 25,71 0,22 17,03 29,38 12,21 0,55 0,72 #dm-1 0,00 0,00 3,64 9,15 79,99 88,06 26,29 0,22 17,41 29,56 12,58 0,57 0,72 #dm-2 0,00 0,00 0,02 0,00 0,08 0,00 8,00 0,00 11,75 11,75 0,00 0,53 0,00 # rrqm/s # The number of read requests merged per second that were queued to # the device. # wrqm/s # The number of write requests merged per second that were queued to # the device. # r/s # The number of read requests that were issued to the device per # second. # w/s # The number of write requests that were issued to the device per # second. # rkB/s # The number of kilobytes read from the device per second. # wkB/s # The number of kilobytes written to the device per second. # avgrq-sz # The average size (in sectors) of the requests that were issued to # the device. # avgqu-sz # The average queue length of the requests that were issued to the # device. # await # The average time (in milliseconds) for I/O requests issued to the # device to be served. This includes the time spent by the requests # in queue and the time spent servicing them. # svctm # The average service time (in milliseconds) for I/O requests that # were issued to the device. # %util # Percentage of CPU time during which I/O requests were issued to # the device (bandwidth utilization for the device). Device satura‐ # tion occurs when this value is close to 100%. cmd = "iostat -x" lines = shell_exec(cmd).strip().split("\n") r = {} for line in lines[6:]: fields = line.split() if fields[0] in disks: t = { "io_read_rate" : fields[5], "io_write_rate" : fields[6], "io_avg_queue_length" : fields[8], "io_utilization_pct" : fields[13] } r[fields[0]] = t return r def vmstat_s(): cmd = "vmstat -s" lines = shell_exec(cmd).strip().split("\n") r = {} for line in lines: val = line.strip().split()[0] key = "_".join(line.strip().split()[1:]).lower() r[key] = val return r disks = enumerate_partitions() # Timestamp # 1 minute load average # 5 minute load average # 15 minute load average # disk queue lenghts per disk delay = None max_iterations = False iteration = 0 try: delay = float(sys.argv[1]) max_iterations = int(sys.argv[2]) except Exception, e: pass available_counters = { "loadavg_1" : "Average load last 1 minutes. Load average is a gauge of how many processes are on average, concurrently demanding CPU attention.", "loadavg_5" : "Average load last 5 minutes. Load average is a gauge of how many processes are on average, concurrently demanding CPU attention.", "loadavg_15" : "Average load last 15 minutes. Load average is a gauge of how many processes are on average, concurrently demanding CPU attention.", "disk_queue_length" : "I/Os currently in progress for each drive. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", "num_proc_wait" : "The number of processes waiting for run time.", "num_proc_unintr_sleep" : "The number of processes in uninterruptible sleep.", "mem_swpd" : "the amount of virtual memory used.", "mem_free" : "The amount of idle memory.", "mem_buff" : "the amount of memory used as buffers.", "mem_cache" : "the amount of memory used as cache.", "swapin_rate" : "Amount of memory swapped in from disk (/s).", "swapout_rate" : "Amount of memory swapped to disk (/s).", "io_blks_recv" : "Blocks received from a block device (blocks/s)", "io_blks_sent" : "Blocks sent to a block device (blocks/s).", "sys_intrs_rate": "The number of interrupts per second, including the clock.", "sys_ctx_switch_rate" : "The number of context switches per second.", "cpu_user" : "% Time spent running non-kernel code. (user time, including nice time)", "cpu_sys" : "% time spent running kernel code. (system time)", "cpu_idle" : "% Time spent idle. Prior to Linux 2.5.41, this includes IO-wait time.", "cpu_wait" : "% time spent waiting for IO. Prior to Linux 2.5.41, included in idle.", "io_read_rate" : "The number of kilobytes read from the device per second.", "io_write_rate" : "The number of kilobytes written to the device per second.", "io_avg_queue_length": "The average queue length of the requests that were issued to the device.", "io_utilization_pct" : "Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%." } selected_counters = [x for x in available_counters] #print selected_counters def addcol(c, first=False): """Add column to CSV line""" r = '"%s"' % (c) if not first: r = ","+r return r while not max_iterations or max_iterations > iteration: try: iteration += 1 head = "" line = "" line += addcol(ts(), first=True) head += addcol("Timestamp", first=True) loadavg_col= loadavg() if "loadavg_1" in selected_counters: head += addcol("loadavg_1") line += addcol(loadavg_col[0]) if "loadavg_5" in selected_counters: head += addcol("loadavg_5") line += addcol(loadavg_col[1]) if "loadavg_15" in selected_counters: head += addcol("loadavg_15") line += addcol(loadavg_col[2]) vmstat_cols = vmstat() iostat_cols = iostat(disks) mem_cols = vmstat_s() if "cpu_user" in selected_counters: head += addcol("cpu_user") line += addcol(vmstat_cols["cpu_user"]) if "cpu_idle" in selected_counters: head += addcol("cpu_idle") line += addcol(vmstat_cols["cpu_idle"]) if "cpu_sys" in selected_counters: head += addcol("cpu_sys") line += addcol(vmstat_cols["cpu_sys"]) if "cpu_wait" in selected_counters: head += addcol("cpu_wait") line += addcol(vmstat_cols["cpu_wait"]) if "disk_queue_length" in selected_counters: disk_ql_col = disk_queue_length(disks) for disk in disks: line += addcol((disk_ql_col[disk])) head += addcol("disk_queue_length: %s" % (disk)) if "io_avg_queue_length" in selected_counters: for disk in disks: try: line += addcol(iostat_cols[disk]["io_avg_queue_length"]) head += addcol("io_avg_queue_length: %s" % (disk)) except KeyError: pass if "io_read_rate" in selected_counters: for disk in disks: try: line += addcol(iostat_cols[disk]["io_read_rate"]) head += addcol("io_read_rate: %s" % (disk)) except KeyError: pass if "io_utilization_pct" in selected_counters: for disk in disks: try: line += addcol(iostat_cols[disk]["io_utilization_pct"]) head += addcol("io_utilization_pct: %s" % (disk)) except KeyError: pass if "io_write_rate" in selected_counters: for disk in disks: try: line += addcol(iostat_cols[disk]["io_write_rate"]) head += addcol("io_write_rate: %s" % (disk)) except KeyError: pass if "io_avg_queue_length" in selected_counters: for disk in disks: try: line += addcol(iostat_cols[disk]["io_avg_queue_length"]) head += addcol("io_avg_queue_length: %s" % (disk)) except KeyError: pass if "io_avg_queue_length" in selected_counters: for disk in disks: try: line += addcol(iostat_cols[disk]["io_avg_queue_length"]) head += addcol("io_avg_queue_length: %s" % (disk)) except KeyError: pass if "io_blks_recv" in selected_counters: head += addcol("io_blks_recv") line += addcol(vmstat_cols["io_blks_recv"]) if "io_blks_sent" in selected_counters: head += addcol("io_blks_sent") line += addcol(vmstat_cols["io_blks_sent"]) if "mem_buff" in selected_counters: head += addcol("mem_buff") line += addcol(vmstat_cols["mem_buff"]) if "mem_cache" in selected_counters: head += addcol("mem_cache") line += addcol(vmstat_cols["mem_cache"]) if "mem_free" in selected_counters: head += addcol("mem_free") line += addcol(vmstat_cols["mem_free"]) if "mem_swpd" in selected_counters: head += addcol("mem_swpd") line += addcol(vmstat_cols["mem_swpd"]) if "num_proc_unintr_sleep" in selected_counters: head += addcol("num_proc_unintr_sleep") line += addcol(vmstat_cols["num_proc_unintr_sleep"]) if "num_proc_wait" in selected_counters: head += addcol("num_proc_wait") line += addcol(vmstat_cols["num_proc_wait"]) if "swapin_rate" in selected_counters: head += addcol("swapin_rate") line += addcol(vmstat_cols["swapin_rate"]) if "swapout_rate" in selected_counters: head += addcol("swapout_rate") line += addcol(vmstat_cols["swapout_rate"]) if "sys_ctx_switch_rate" in selected_counters: head += addcol("sys_ctx_switch_rate") line += addcol(vmstat_cols["sys_ctx_switch_rate"]) if "sys_intrs_rate" in selected_counters: head += addcol("sys_intrs_rate") line += addcol(vmstat_cols["sys_intrs_rate"]) #print """"%s","%s","%s","%s",%s""" % (t, t_load[0], t_load[1], t_load[2], disk_ql_cols[:-1] ) if iteration == 1: print head print line if max_iterations and max_iterations <= iteration: sys.exit() if delay: sleep(delay) else: sys.exit() except KeyboardInterrupt: sys.exit()