openstatus-client

#!/usr/bin/python
import socket, select, signal, os, sys, atexit, re, getopt, time, logging, commands
from configobj import ConfigObj, ConfigObjError
from validate import Validator

logger = logging.getLogger('openstatus-client')
hdlr = logging.FileHandler('/var/log/openstatus-client.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr) 
logger.setLevel(logging.INFO)

ch  = logging.StreamHandler()
logger.addHandler(ch)

cfg = """
[server]
port = integer(min=1, max=65535, default=12999)
[[__many__]]
port = integer(min=1, max=65535, default=12999)
[misc]
interval = integer(min=1, default=60)
[load]
warning = float(min=1, default=1)
critical = float(min=2, default=2)
[memory]
warning = integer(min=0, default=512)
critical = integer(min=0, default=1024)
[disk]
warning = integer(min=0, max=100, default=80)
critical = integer(min=0, max=100, default=90)
[transfer]
interface = string(default='eth0')
"""

debug = False

try:
	options, remainder = getopt.getopt(sys.argv[1:], 'dc:', ['debug',])
except getopt.GetoptError:
	print "usage: ", sys.argv[0]," [-c configfile] [-d | --debug]"
	sys.exit(2)

configfile = None
for opt, args in options:
	if opt in ('-d', '--debug'):
		debug = True
	elif opt in ('-c'):
		if args in ('-d', '--debug'):
			print "usage: ", sys.argv[0]," [-c configfile] [-d | --debug]"
			sys.exit(2)
		configfile = args

if debug == True:
	logger.setLevel(logging.DEBUG)

if configfile == None:
	configfile = '/etc/openstatus/openstatus-client.conf'

try:
	config = ConfigObj(configfile, file_error = True, configspec=cfg.split('\n'), list_values=False)
except (ConfigObjError, IOError), e:
	logger.error('%s' % (e))
	try:
		configfile = '/etc/openstatus/openstatus-client.conf'
		config = ConfigObj(configfile, file_error = True, configspec=cfg.split('\n'), list_values=False)
	except (ConfigObjError, IOError), e:
		logger.error('%s' % (e))
		logger.error('Exiting...')
		sys.exit(3)

validator = Validator()
result = config.validate(validator)
if result != True:
	logger.error('Config file validation failed!')
	sys.exit(1)

def getTransfer():
	global config, logger
	rp = os.popen("ifconfig %s | grep \"RX bytes\"" % (config['transfer']['interface']))
	line = rp.readline()
	if (line == ""):
		logger.error("Interface for transfer accounting not found.")
		return None
	else:
		words = line.strip().split(' ')
		rx = words[1].split(':')[1]
		tx = words[6].split(':')[1]
		return {"rx" : int(rx), "tx": int(tx)}

starttime = time.time()
transferbytes = getTransfer()

try:
	import setproctitle
	setproctitle.setproctitle('openstatus-client')
except ImportError:
	logger.debug('setproctitle module not found')
try:
	import json
except:
	try:
		import simplejson as json
	except ImportError:
		logger.error('Could not import json module')
		sys.exit(1)

# Let's make this a daemon... this is a very basic daemon fork, taken from: http://code.activestate.com/recipes/66012-fork-a-daemon-process-on-unix/#c9
# Do the first fork
def daemonize(stdout, stderr, stdin, pidfile, startmsg = "Started with PID %s"):

	try:
		pid = os.fork()
		if pid > 0: sys.exit(0) # Exit first parent.
	except OSError, e:
		logger.error("fork #1 failed: (%d) %s" % (e.errno, e.strerror))
		sys.exit(1)

	# Decouple from parent environment.
	# os.chdir("/") # I don't think we need to do this - it probably breaks using relative paths with the -c switch
	os.umask(0)
	os.setsid()

	# Do second fork.
	try:
		pid = os.fork()
		if pid > 0: sys.exit(0) # Exit second parent.
	except OSError, e:
		logger.error("fork #2 failed: (%d) %s" % (e.errno, e.strerror))
		sys.exit(1)

	# Open file descriptors and print start message
	if not stderr: stderr = stdout
	# We don't need to read from stdin, do we?
	si = file(stdin, 'r')
	so = file(stdout, 'a+')
	se = file(stderr, 'a+', 0)
	pid = str(os.getpid())
	logger.info("%s" % startmsg % pid)
	if pidfile: file(pidfile,'w+').write("%s" % pid)
	if debug == False:
		logger.removeHandler(ch)
	# Redirect standard file descriptors.
	os.dup2(si.fileno(), sys.stdin.fileno())
	os.dup2(so.fileno(), sys.stdout.fileno())
	os.dup2(se.fileno(), sys.stderr.fileno())

def sigHdl(signum, frm):

	if signum == signal.SIGTERM:
		logger.critical('Received SIGTERM... Exiting.')
		try:
			s.close()
		except:
			logger.debug('No connection to close')
		os._exit(0)
	else:
		logger.critical('Received signal %s... Exiting.' % (signum))
		try:
			s.close()
		except:
			logger.debug('No connection to close')
		os._exit(0)

def checkProcs():

	procs = []
	for section in configprocs.sections:
		procs.append({'name': section, 'proc': configprocs[section]['proc'], 'level': configprocs[section]['level']})

	# Get list of processes and process it
	rp = os.popen("/bin/ps ax -o command=")
	plines = rp.readlines()
	lines = []
	procstatus = {}
	for l in plines:

		# Output of `ps ax -o command=` contains the full path for some things, and the command line args, so we need to get the basename.
		# There's a : stuck on the end of some stuff, like nginx, so we have to get rid of that too
		lines.append(os.path.basename((l.strip().split(' '))[0]).replace(':', ''))
		for pn in procs:
			if pn['proc'] in lines:
				procstatus[pn['name']] = { 'name': pn['name'], 'status': 0 }
			else:
				if pn['level'] == "warn": procstatus[pn['name']] = { 'name': pn['name'], 'status': 1 }
				elif pn['level'] == "critical": procstatus[pn['name']] = { 'name': pn['name'], 'status': 2 }
	return procstatus


def checkCommands():
	
	coms = []
	comstatus = {}
	for section in configcommands.sections:
		if configcommands[section]['type'] == 'integer':
			coms.append({'name': section, 'command': configcommands[section]['command'], 'type': configcommands[section]['type'], 'warning': configcommands[section]['warning'], 'critical': configcommands[section]['critical']})
		else:
			coms.append({'name': section, 'command': configcommands[section]['command'], 'type': configcommands[section]['type'], 'level': configcommands[section]['level'], 'string' : configcommands[section]['string']})
	for com in coms:
		logger.debug('Running command: %s' % com['command'])
		co = commands.getoutput(str(com['command']))
		if com['type'] == 'integer':
			# Check integer values
			cstatus = 0
			if int(co) > int(com['warning']):
				cstatus += 1;
			if int(co) > int(com['critical']):
				cstatus += 1;
			comstatus[com['name']] = { 'name' : com['name'], 'status': cstatus, 'value' : co }
		elif com['type'] == 'string':
			# Check if string is in output
			cstatus = 0
			if com['string'] in co:
				pass
			else:
				if com['level'] == 'critical':
					cstatus = 2
				else:
					cstatus = 1
			comstatus[com['name']] = { 'name' : com['name'], 'status': cstatus, 'value' : 0 }
		elif com['type'] == 'notstring':
			# Check if string is NOT in output
			# Check if string is in output
			cstatus = 0
			if com['string'] in co:
				if com['level'] == 'critical':
					cstatus = 2
				else:
					cstatus = 1
			else:
				pass
			comstatus[com['name']] = { 'name' : com['name'], 'status': cstatus, 'value' : 0 }
	return comstatus
					
def getUptime():

	rp = os.popen("/usr/bin/uptime")
	line = rp.readline()
	sects = line.split(', ', 2)
	timeuptime = sects[0]
	textload = sects[2]
	uptime = (timeuptime.split('up '))[1].strip()
	loads = (textload.split(': '))[1].split(', ')
	load1 = float(loads[0].strip())
	load5 = float(loads[1].strip())
	load15 = float(loads[2].strip())
	status = 0
	if (load1 > config['load']['warning']): status += 1
	if (load1 > config['load']['critical']): status += 1
	return {'uptime': uptime, 'load1': load1, 'load5': load5, 'load15': load15, 'status': status}

def getMem():

	rp = os.popen("/usr/bin/free -m")
	rp.readline()
	line = rp.readline()
	words1 = line.split(' ')
	words = []
	for w in words1:
		if w != '': words.append(w)
	total = int(words[1])
	bufcac = int(words[5])+int(words[6])
	line2 = rp.readline()
	words2 = line2.split(' ')
	words3 = []
	for b in words2:
		if b != '': words3.append(b)

	free = int(words3[3])
	used = int(words3[2])
	status = 0
	if (used > int(config['memory']['warning'])): status += 1
	if (used > int(config['memory']['critical'])): status += 1
	return {'used': used, 'free': free, 'total': total, 'bufcac': bufcac, 'status': status}

def getDisk():

	dics = {'single': [], 'total': {'avail': 0, 'used': 0, 'total': 0}}
	last = []
	rp = os.popen("/bin/df -TP")
	rp.readline()
	lines = rp.readlines()
	for l in lines:
		m = re.match(r"(.+?)\s+([\w\-]+)\s+([\d.]+\w?)\s+([0-9.]+\w?)\s+([0-9.]+\w?)\s+(?:\d+%)\s*(.*)", l).groups()
		# TYPE:FS:MP:TOTAL:USED:AVAIL
		dics['single'].append({'type': m[1], 'fs': m[0], 'mount': m[5], 'total': int(m[2]), 'used': int(m[3]), 'avail': int(m[4])})
	for item in dics['single']:
		dics['total']['total'] += item['total']
		dics['total']['used'] += item['used']
		dics['total']['avail'] += item['avail']
	status = 0
	if ((int(dics['total']['used'])/int(dics['total']['avail'])) > config['disk']['warning']): status += 1
	if ((int(dics['total']['used'])/int(dics['total']['avail'])) > config['disk']['critical']): status += 1
	dics['total']['status'] = status
	return dics

def getHostname():
	rp = os.popen("/bin/hostname")
	line = rp.readline()
	return line.strip()

def getIPs():
	ary = []
	rp = os.popen("/sbin/ifconfig | grep 'inet addr:' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{print $1}'")
	lines = rp.readlines()
	for line in lines:
		ip = line.strip()
		host = socket.getfqdn(ip)
		ary.append({'ip': ip, 'host': host})
	return ary

	
def getWho():
	users = {}
	rp = os.popen("/usr/bin/who -q")
	line = rp.readline()
	words = line.strip().split(' ')
	for v in words:
		try: users[v] += 1
		except KeyError: users[v] = 1
	return users

def main():

	global configprocs, configcommands, transferbytes, exectime
	if "processes" in config:
		configprocs = config['processes']
	else:
		logger.info('Not monitoring any processes')
	if "commands" in config:
		configcommands = config['commands']
	else:
		logger.info('Not monitoring any commands')
	while 1:

		starttime = time.time()
		output = {}
		
		transferbytes2 = getTransfer()
		if (transferbytes2 != None):
			if transferbytes2['rx'] < transferbytes['rx']:
				rxdiff = int(((4294967295 - transferbytes['rx']) + transferbytes2['rx'])/(config['misc']['interval']-exectime))
			else:
				rxdiff = int((transferbytes2['rx']-transferbytes['rx'])/(config['misc']['interval']-exectime))
			if transferbytes2['tx'] < transferbytes['tx']:
				txdiff = int(((4294967295 - transferbytes['tx']) + transferbytes2['tx'])/(config['misc']['interval']-exectime))
			else:
				txdiff = int((transferbytes2['tx']-transferbytes['tx'])/(config['misc']['interval']-exectime))
			rxstatus = 0
			txstatus = 0
			if rxdiff > config['transfer']['rx-warning']: rxstatus += 1
			if rxdiff > config['transfer']['rx-critical']: rxstatus += 1
			if txdiff > config['transfer']['tx-warning']: txstatus += 1
			if txdiff > config['transfer']['tx-critical']: txstatus += 1
			transferdiff = { "rx": rxdiff, "tx": txdiff, "rx-status": rxstatus, "tx-status": txstatus }
			transferbytes = transferbytes2
			output['transfer'] = transferdiff

		output['time'] = int(starttime)
		output['hostname'] = getHostname()
		output['ips'] = getIPs()
		output['who'] = getWho()
		if "processes" in config:
			output['procs'] = checkProcs()
		output['uptime'] = getUptime()
		output['ram'] = getMem()
		output['disk'] = getDisk()

		if "commands" in config:
			output['commands'] = checkCommands()


		if (config['server'].has_key('ip')):
			try:
				s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
				s.connect((config['server']['ip'], int(config['server']['port'])))
				s.sendall(json.dumps(output))
				logger.debug("JSON sent: %s", (json.dumps(output)))
				s.close()
			except socket.error, (errno, strerror):
				logger.warning('Could not connect to server %s:%d: %s' % (config['server']['ip'], config['server']['port'], strerror))
		else:
			for srv in config['server']:
				if (srv != "port" and srv != "ip"):
					logger.debug("Connecting to server at %s, port %i" % (config['server'][srv]['ip'], config['server'][srv]['port']))
					#sys.exit(0)
					try:
						s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
						s.connect((config['server'][srv]['ip'], int(config['server'][srv]['port'])))
						s.sendall(json.dumps(output))
						logger.debug("JSON sent to %s:%i : %s" % (config['server'][srv]['ip'], config['server'][srv]['port'], json.dumps(output)))
						s.close()
					except socket.error, (errno, strerror):
						logger.warning('Could not connect to server %s:%d: %s' % (config['server'][srv]['ip'], config['server'][srv]['port'], strerror))
		endtime = time.time()
		exectime = endtime-starttime
		sleeptime = float(config['misc']['interval']) - exectime
		if sleeptime > 0:
			time.sleep(sleeptime)
		else:
			logger.warning("Sleep time is negative! Value: %d" % (sleeptime))

if __name__ == "__main__":

	signal.signal(signal.SIGTERM, sigHdl)
	signal.signal(signal.SIGHUP, sigHdl)
	signal.signal(signal.SIGINT, sigHdl)

	if debug == False:
		daemonize('/var/log/openstatus-client.log', '/var/log/openstatus-client.log', '/dev/null', '/var/run/openstatus-client.pid')
		exectime = time.time()-starttime
		main()
	elif debug == True:
		logger.info("OpenStatus client started in debug mode with PID %s using config file %s" % (os.getpid(), configfile))
		exectime = time.time()-starttime
		main()