ad_monitor_top

ad_monitor_top

ad_monitor_top parses the output from "top" and stores the results in a table. This is useful to later create reports of load averages and memory usage in a time-sensitive way. Required Parameters in the monitoring section: * TopLocation: Full path to execute top Optional Parameters: * LoadAverageAlertThreshold: Minimum load average before emailing persons to notify that the machine is overloaded

Source code:

    
    # list indicating the order in which top outputs data
    set proc_var_list [list pid username threads priority nice proc_size  resident_memory state cpu_total_time cpu_pct command]
    # location of the desired top function
    set top_location [ad_parameter TopLocation monitoring]

    # make sure we have a path to top and that the file exists
    if { [empty_string_p $top_location] } {
	ns_log Error "ad_monitor_top: cannot find top; TopLocation parameter in monitoring 
                      is not defined"
	return
    } elseif { ![file exists $top_location] } {
	ns_log Error "ad_monitor_top: Specified location for top ($top_location) does not exist"
	return
    }

    # set top_output [exec $top_location]
    if [catch { set top_output [exec $top_location] } errmsg] {
        # couldn't exec top at TopLocation
        if { ![file exists $top_location] } {
                ad_return_error "top not found" "
    The top procedure could not be found at $top_location:
    <blockquote><pre> $errmsg </pre></blockquote>"
                return
        }
        ad_return_error "insufficient top permissions" "
    The top procedure at $top_location cannot be run:
    <blockquote><pre> $errmsg </pre></blockquote>"
        return
    }
    # else top execution went ok
    set top_list [split $top_output "\n"]

    ## Run through the output of top, grab header lines and leave the rest. 
    # number of header lines grabbed
    set ctr 0   	
 
    # greet the database
    set db [ns_db gethandle]

    # id for this iteration of top-parsing
    set top_id [database_to_tcl_string $db "select ad_monitoring_top_top_id.nextval from dual"]

    # have we reached the list of process info yet?    
    set procflag 0	
    foreach line $top_list {

	if { $procflag > 0 } {
	    #compress multiple spaces
    	    regsub -all {[ ]+} [string trim $line] " " line    
            set proc_list [split $line]

	    #skip blank lines
            if { [llength $proc_list] < 2 } { continue } 
            if { [llength $proc_list] < 11 } { 
		ns_log Notice "skipping top process line: 
                     element list shorter than variable list."
		continue }
		
	    #set proc-related vars
 	    set pid       [lindex $proc_list 0]
	    set username  [lindex $proc_list 1]
	    set threads   [lindex $proc_list 2]
	    set priority  [lindex $proc_list 3]
	    set nice      [lindex $proc_list 4]
	    set proc_size [lindex $proc_list 5]
	    set resident_memory [lindex $proc_list 6]
	    set state           [lindex $proc_list 7]
	    set cpu_total_time  [lindex $proc_list 8]
	    set cpu_pct   [lindex $proc_list 9]
	    set command   [lindex $proc_list 10]

	    ns_db dml $db "insert into ad_monitoring_top_proc
  	     (proc_id, top_id, pid, command, username, 
	      threads, priority, nice, proc_size, 
	      resident_memory, state, cpu_total_time, cpu_pct) 
 	    values 
      	     (ad_monitoring_top_proc_proc_id.nextval, $top_id, $pid, '$command', 
	      '$username', $threads, $priority, $nice, '$proc_size',  
	      '$resident_memory', '$state', '$cpu_total_time', '$cpu_pct')" 

	    continue   

	} elseif { [regexp -nocase {(.*PID.USERNAME.*)} $line match top_header] } {
	    ## this is the start of proc info lines
            incr procflag

	    continue

        } elseif { [regexp -nocase {load av[a-z]*: (.*)} $line match load] } {
	    ## this is the load header 
	    incr ctr

	    # remove commas, multiple spaces
    	    regsub -all {,} [string trim $load] "" load   
    	    regsub -all {[ ]+} $load " " load             

    	    set load_list [split $load " "]
    
    	    # We keep all three load avgs, ignore the time at the end of the line
    	    set load_1  [lindex load_list 0]
	    set load_5  [lindex load_list 1]
	    set load_15 [lindex load_list 2]

	    # send out any high-load alerts
   	    if { $load_5 > [ad_parameter LoadAverageAlertThreshold monitoring 2.0] } {
              wd_email_notify_list "High Load Average on [ad_url]"  "The load average over the last 5 minutes for [ad_url] was $load_5"
    	     } 
	} elseif { [regexp -nocase {mem[a-z]*: (.*)} $line match memory] } {
	    ## this is the memory header 
	    incr ctr
            foreach mem [split $memory ","] {
                regexp {^ *([^ ]*) (.*)} $mem match amount type
                set amount [string trim [string toupper $amount]]
               	# convert all mem values to Kilobytes 
                regsub {K$} $amount "" amount
                regsub {M$} $amount "000" amount

                set type [string trim [string tolower $type]]
                switch $type {
                    "real" { set memory_real $amount }
                    "free" { set memory_free $amount }
                    "swap free" { set memory_swap_free $amount }
                    "swap in use" { set memory_swap_in_use $amount }
                }
            } 
	}

	if {$ctr == 2 } {
 	    ## we have all the header information we currently store.
	    ## we should also store the rest of top's output... 
	    ## some of it can be gotten from the zoom package. 
	    ns_db dml $db "insert into ad_monitoring_top
             (top_id, timestamp, timehour, load_avg_1, load_avg_5, load_avg_15,
              memory_real, memory_free, memory_swap_in_use, memory_swap_free)
            values
             ($top_id, sysdate, to_char(sysdate, 'HH24'), $load_1, $load_5, $load_15,
              $memory_real, $memory_free, $memory_swap_in_use, $memory_swap_free)"
	    incr ctr
	}
    # end of the foreach loop
    }

    if { $ctr < 2 } {
	# didn't even get load and memory lines from top
	ns_log Error "ad_monitor_top: Cannot parse output from top"
	return
    }