andreas/heartbeat
Public Access
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 44 Wiki Activity

major rework for ver 3.0

Browse Source
This commit is contained in:
Andreas Wrede
2016-04-22 00:00:51 -04:00
parent b3fe352b13
commit f90038beae
2 changed files with 528 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hbc
+293 -129
View File
@@ -11,6 +11,11 @@ import string
import select
import errno
import traceback
import urllib2
import md5
import shutil
import zlib
import subprocess
try:
import lockfile
@@ -33,123 +38,269 @@ import syslog
PORT = 50003
INTERVAL = 10
PIDFILE = '/tmp/hbc.pid'
DBG = False
VER = 1
MAXRECV = 32767
socks = None
up = True
ackcount = 0
running = True
dorestart = False
class NullDevice:
def write(self, s):
pass
class Conn:
def __init__(self, conId, addr, port, af):
self.conId = conId
self.addr = addr
self.port = port
self.af = af
self.ackcount = 0 # num of accks received
self.lastack = 0 # time() last ACK was received
self.send = 0
self.lastsend = 0 # time() last msg was sent
self.rtts = []
self.sock=socket.socket(af, socket.SOCK_DGRAM)
self.sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR, \
self.sock.getsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR) | 1)
def sendto(self, msg, ID = 'HTB'): # default ID is HearTBeat
msg['name'] = iam
msg['id'] = self.conId
msg['ver'] = VER
msg['time'] = time.time()
m = dicttos(ID, msg)
mz = zlib.compress(m,9)
if verbose: print "conn.send('%s', (%s:%s) %s>%s)" % (msg, self.addr, self.port, len(m), len(mz))
self.sock.sendto(m, (self.addr, self.port))
self.send += 1
self.lastsend = time.time()
def ack(self, msgDict):
self.lastack = time.time()
self.lastacksent = float(msgDict.get('time','0'))
if verbose: print "ack RTT: %0.1f ms" % ((self.lastack - self.lastacksent) * 1000.0)
self.ackcount += 1
def close(self):
if self.sock:
self.sock.close()
self.sock = None
def dicttos(ID, d, compress=False):
s = []
for k in d:
if type(d[k]) == type(1.2):
s.append("%s=%0.5f" % (k, d[k]))
else:
s.append("%s=%s" % (k, d[k]))
pk = ";".join(s)
if compress:
zpk = zlib.compress(pk, 6)
ID = "!"+ID
else:
zpk = pk
return ID + ":" + zpk
def stodict(msg):
d = {}
r0 = msg.split(':',1)
if len(r0) == 1:
return None
if r0[0][0] == '!': # compressed
pk = zlib.decompress(msg[len(r0[0])+1:])
d['ID'] = r0[0][1:]
else:
pk = r0[1]
d['ID'] = r0[0]
r = pk.split(';')
for v in r:
vr = v.split('=', 1)
k = vr[0].strip()
if len(vr) == 1:
d[k] = None
else:
v = vr[1].strip()
try:
if v[0].isdigit():
v = eval(v)
except:
pass
d[k] = v
return d
def syslogtrace(note):
logm = '%s hbc died: \n%s' % (note, traceback.format_exc())
for l in logm.split('\n'):
syslog.syslog(syslog.LOG_ERR, ' tb: %s' % l)
if verbose:
print logm
def getsock(host):
try:
rs=socket.getaddrinfo(host, 50001, 0, 0, socket.SOL_UDP)
except socket.gaierror:
logm = '%s hbc died: \n%s' % ('getsock', traceback.format_exc())
if DBG: print logm
return None
socks = []
for r in rs:
if DBG: print "address %s" % str(r)
if r[0] in [10, 28, 30]:
af_type=socket.AF_INET6
elif r[0] == 2:
af_type=socket.AF_INET
else:
print "dont know this net type: %s" % r[0][0]
sys.exit(1)
if verbose:
syslog.syslog(syslog.LOG_ERR, "socktype: %s" % af_type)
sock=socket.socket(af_type, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR, \
sock.getsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR) | 1)
if verbose: syslog.syslog(syslog.LOG_ERR, "get socket %s" % sock)
socks.append(sock)
conId = 1
def createConnections(hosts):
global conId
for host in hosts:
if verbose: print "createConnections for %s" % host
try:
rs=socket.getaddrinfo(host, hb_port, 0, 0, socket.SOL_UDP)
except socket.gaierror:
logm = '%s hbc died: \n%s' % ('createConnections', traceback.format_exc())
if verbose: print logm
return None
for r in rs:
if verbose: print "address %s" % str(r)
if r[0] in [10, 28, 30]:
af=socket.AF_INET6
elif r[0] == 2:
af=socket.AF_INET
else:
print "dont know this net type: %s" % r[0][0]
sys.exit(1)
return socks
addr = r[4][0]
conns[conId] = Conn(conId, addr, hb_port, af)
conId += 1
def socksend(msg, tohost):
global socks
def doexec(conn, data):
try:
ro = subprocess.check_output(data, stderr=subprocess.STDOUT, shell=True)
fail = "OK"
except subprocess.CalledProcessError as e:
ro = str(e)
fail = "CalledProcessError"
except Exception as e:
syslogtrace('System')
ro = None
fail = "cmd failed: %s" % e
msg={'service': 'command', 'msg': fail+" "+ro}
conns[conn].sendto(msg)
if socks == None:
socks = getsock(tohost[0])
for sock in socks:
if DBG: print "socksend: sending msg=%s on socket=%s" % (msg, sock)
sock.sendto(msg, tohost)
if verbose: syslog.syslog(syslog.LOG_ERR, "msg %s sent" % msg)
def doupdate(conn, msgDict):
fail = None
try:
code = msgDict['code'].decode('base64')
csum = msgDict['csum']
except:
fail = "csum/code missing"
if not fail:
fail = doupdateone(code, csum)
msg={'service': 'update', 'msg': fail if fail else "OK"}
conns[conn].sendto(msg)
return fail
def doupdateone(code, csum):
m = md5.new()
m.update(code)
icsum = m.hexdigest()
if icsum != csum:
return "checksum error"
fn = sys.argv[0]
ofn = "%.sav" % fn
try:
shutil.copy2(fn, ofn)
except Exception as e:
return "cannot make backup copy: %s" % e
try:
fh=open(fn, "w")
fh.write(code)
fh.close()
except Exception as e:
return "cannot write new code: %s" % e
return None
def restart():
if verbose: print "restart: execv %s %s" % (sys.argv[0], [sys.argv[0]]+cmdargs)
os.execv(sys.argv[0], [sys.argv[0]]+cmdargs)
print "should not be here"
def process():
global up, socks, ackcount
global running, dorestart
if socks == None:
socks=getsock(tohost[0])
ifiles = {}
conIds = {}
for conn in conns:
ifiles[conns[conn].sock.fileno()] = conns[conn].sock
conIds[conns[conn].sock.fileno()] = conn
nextReport = time.time()
ackcount=0
lastT=time.time()
ifiles = []
for sock in socks:
ifiles.append(sock.fileno())
while up:
sleep=(lastT+interval) - time.time()
if verbose: syslog.syslog(syslog.LOG_ERR, "sleep %s" % sleep)
if sleep > 0:
while running:
while time.time() < nextReport:
sleep=nextReport - time.time()
if verbose: print "process: sleep %s" % sleep
if sleep <= 0:
break
try:
r=select.select(ifiles,[],[],sleep)
r=select.select(ifiles.keys(),[],[],sleep)
except KeyboardInterrupt:
running = False
break
except:
if up:
if running:
syslogtrace('select')
break
if verbose: syslog.syslog(syslog.LOG_ERR, "r is %s" % str(r))
cont = False
for sock in socks:
if sock.fileno() in r[0]:
data, addr = sock.recvfrom(1024)
if data == "ACK":
ackcount+=1
else:
try:
os.system(data)
except:
syslogtrace('System')
pass
cont = True
continue
if cont:
continue
lastT=time.time()
for hb_host in hb_hosts:
try:
msg="interval=%s;name=%s;time=%s;acks=%s" % (interval, iam, time.time(), ackcount)
if verbose: syslog.syslog(syslog.LOG_ERR, "sock.send('%s', (%s, %s))" % (msg, hb_host, hb_port))
socksend(msg, (hb_host, hb_port))
except:
logm = '%s hbc died: \n%s' % ('socksend', traceback.format_exc())
if DBG: print logm
pass
if verbose: print "process: r is %s" % str(r)
for rfh in r[0]:
conn = conIds[rfh]
data, addr = ifiles[rfh].recvfrom(MAXRECV)
if verbose: print "sock.recvfrom: %s (%s) %s" % (addr, len(data), data[:4])
msgDict = stodict(data)
if verbose: print "sock.recvfrom: %s (%s) %s" % (addr, len(data), str(msgDict)[:80])
if msgDict['ID'] == "ACK":
conns[conn].ack(msgDict)
elif msgDict['ID'] == "UPD":
if doupdate(conn, msgDict) == None:
if verbose: print "process: restart after update"
dorestart = True
break
elif msgDict['ID'] == "CMD":
doexec(conn, msgDict['cmd'])
else:
doexec(conn, data) # deprecated until no more VER - hbc
if dorestart:
running = False
break
if not running:
break
for conn in conns:
msg={'interval': interval, 'acks': conns[conn].ackcount}
conns[conn].sendto(msg)
nextReport += interval
def cleanup(a, b):
global up, socks, ackcount
up = False
syslog.syslog(syslog.LOG_ERR, 'exit a=%s b=%s' % (str(a), str(b)))
msg="shutdown=1;name=%s;acks=%s" % (iam, ackcount)
for hb_host in hb_hosts:
if verbose: syslog.syslog(syslog.LOG_ERR, "hbc: sock.send('%s', (%s, %s))" % (msg, hb_host, hb_port))
socksend(msg, (hb_host, hb_port))
if verbose: print "process: done running"
def cleanup():
global running
running = False
for conn in conns:
msg={'shutdown': 1, 'acks': conns[conn].ackcount}
conns[conn].sendto(msg)
conns[conn].close()
time.sleep(1)
for sock in socks:
sock.close()
closeall()
def closeall():
for conn in conns:
conns[conn].close()
syslog.syslog(syslog.LOG_ERR, 'exit')
msgonly=False
@@ -158,9 +309,11 @@ verbose=False
fdaemon=False
optlist=[]
args=[]
msgboot=[]
msgboot={}
home=os.environ['HOME']
configfile="%s/.hbrc" % home
cmdargs = []
try:
optlist, args = getopt.getopt(sys.argv[1:], 'bc:dhm:v')
@@ -169,22 +322,28 @@ except:
for o,a in optlist:
if o == '-b':
msgboot.append("boot=1")
msgboot['boot']=1
msgonly=True
elif o == '-c':
configfile=a
cmdargs += [o, a]
elif o == '-d':
fdaemon=True
cmdargs += [o]
elif o == '-h':
helpflag=True
elif o == '-m':
msgboot.append("service=%s" % "service")
a.replace(';',':')
msgboot.append("msg=%s" % a)
msgboot['service'] = "service"
msgboot['msg'] = a
msgonly=True
elif o == '-v':
verbose=True
cmdargs += [o]
cmdargs += args
if verbose: print "cmdargs for restart are %s" % cmdargs
if helpflag:
print "hbc HeartBeatClient"
print "usage: hbc [-bdhv] [-c configfile] [-m msg][host1 [..]]"
@@ -217,7 +376,7 @@ hb_hosts=[]
iam=socket.gethostname()
try:
f=open(configfile,"r")
f=open(configfile,"r")
if verbose: print "notice: using config file %s" % configfile
except:
if verbose: print "warning: running without config file: %s" % configfile
@@ -244,50 +403,43 @@ if f:
if len(args) != 0:
hb_hosts=args
if len(hb_hosts) == 0:
print "no hb server specified"
sys.exit(1)
#
if verbose:
print "notice: hb_hosts: %s" % str(hb_hosts)
print "notice: hb_port: %s" % hb_port
print "notice: interval: %s" % interval
print "notice: iam: %s" % iam
print "notice: msgonly: %s" % msgonly
print "notice: msgboot: %s" % msgboot
if not msgonly:
msgboot.append("interval=%s" % interval)
msgboot['interval'] = interval
conns = {}
createConnections(hb_hosts)
if verbose: print "%s connections created" % (len(conns))
if len(msgboot) > 0:
if DBG: print "on boot"
msgboot.append("name=%s" % iam)
msgboot.append("time=%s" % time.time())
msgboot.append("acks=0")
msg=";".join(msgboot)
while 1:
fail=0
for hb_host in hb_hosts:
try:
if DBG: print "sock.send('%s', (%s, %s))" % (msg, hb_host, hb_port)
socksend(msg, (hb_host, hb_port))
except:
logm = '%s hbc died: \n%s' % ('socksend2', traceback.format_exc())
if DBG: print logm
fail=1
if fail:
time.sleep(10)
else:
break
if verbose: print "msgboot done msgonly=%s" % msgonly
if verbose: print "on boot"
msgboot['acks'] = 0
for conn in conns:
conns[conn].sendto(msgboot)
if msgonly:
sys.exit(0)
if verbose: print "msgboot done msgonly=%s" % msgonly
closeall()
sys.exit(0)
#
#
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_DAEMON)
if fdaemon:
@@ -326,22 +478,34 @@ if fdaemon:
}
context.files_preserve = []
for sock in socks:
context.files_preserve += [sock, sock.fileno()]
for conn in conns:
context.files_preserve += [conns[conn].sock, conns[conn].sock.fileno()]
with context:
syslog.syslog(syslog.LOG_ERR, 'starting heartbeat to %s' % ','.join(hb_hosts))
up = True
running = True
try:
process()
except:
syslogtrace('process')
cleanup(0, None)
if verbose: print "main: cleanup"
cleanup()
if dorestart:
if verbose: print "main: restart"
restart()
else:
up = True
running = True
try:
if verbose: print "starting loop process"
process()
except:
except Exception as e:
if verbose: print "err: process exit: %s" % e
syslogtrace('process')
cleanup(0, None)
if verbose: print "main: cleanup"
cleanup()
if dorestart:
if verbose: print "main: restart"
restart()
hbd
+235 -116
View File
@@ -22,6 +22,8 @@ import urllib
import httplib
import threading
import Queue
import md5
import zlib
from subprocess import Popen, STDOUT, PIPE
@@ -31,6 +33,7 @@ SEND_PUSHOVER=True
DEBUG = 0
MAXRECV = 32767
LOGFILE = "/home/andreas/public_html/messages/andreas"
PICKFILE = "/var/tmp/hbd.pick"
AEMAIL = ["andreas@wrede.ca"]
@@ -92,6 +95,50 @@ class LogDevice:
self.fh.flush()
def dicttos(ID, d, compress=False):
s = []
for k in d:
if type(d[k]) == type(1.2):
s.append("%s=%0.5f" % (k, d[k]))
else:
s.append("%s=%s" % (k, d[k]))
pk = ";".join(s)
if compress:
zpk = zlib.compress(pk, 6)
ID = "!"+ID
else:
zpk = pk
return ID + ":" + zpk
def stodict(msg):
d = {}
r0 = msg.split(':',1)
if len(r0) == 1:
return None
if r0[0][0] == '!': # compressed
pk = zlib.uncompress(r0[1])
d['ID'] = r0[0][1:]
else:
pk = r0[1]
d['ID'] = r0[0]
r = pk.split(';')
for v in r:
vr = v.split('=', 1)
k = vr[0].strip()
if len(vr) == 1:
d[k] = None
else:
v = vr[1].strip()
if v[0].isdigit():
v = eval(v)
d[k] = v
return d
def oldmtodict(msg):
return stodict('HTB:'+msg)
class Host:
up = "up"
down = "down"
@@ -112,11 +159,21 @@ class Host:
self.interval = 0
self.doesack = -1
self.cmds = []
self.cver = 0
self.latency = 0
self.hdwcounts = [[0,0],[0,0],[0,0]]
num += 1
hosts[name] = self
def setcver(self, cver):
self.cver = cver
def isDynDns(self):
return self.name in dyndnshosts
def newaddr(self, addr):
if isIPv4(addr):
if self.addr4:
@@ -127,10 +184,14 @@ class Host:
del htab[self.addr4]
self.addr4 = addr
htab[addr] = self.name
if self.isDynDns():
dnsQ.put((self.name, self.addr4))
else:
r = "new addr %s" % (addr)
self.addr4 = addr
htab[addr] = self.name
if self.isDynDns():
dnsQ.put((self.name, self.addr4))
else:
if self.addr6:
if self.addr6 == addr:
@@ -140,10 +201,14 @@ class Host:
del htab[self.addr6]
self.addr6 = addr
htab[addr] = self.name
if self.isDynDns():
dnsQ.put((self.name, self.addr6))
else:
r = "new addr %s" % (addr)
self.addr6 = addr
htab[addr] = self.name
if self.isDynDns():
dnsQ.put((self.name, self.addr6))
return r
@@ -182,6 +247,12 @@ class Host:
self.addr6 = self.addr
del self.addr
try:
a=self.latency
except:
self.latency = 0
def getstate(self):
return self.state
@@ -190,6 +261,8 @@ class Host:
def dispstate(self):
if self.state in ["down", "overdue"]:
state = "<b>%s</b>" % self.state
elif self.state in ["up", "UP"]:
state = "%0.1f" % self.latency
else:
state = "%s" % self.state
return state
@@ -224,21 +297,22 @@ class Host:
return s
def email(s, msg):
if not SEND_EMAIL:
return
ret = "OK"
toaddrs = AEMAIL
fromaddr = "aew.heartbeat@wrede.ca"
fromemail = "aew.heartbeat@wrede.ca"
subj = "Info from %s: %s" % (NAME, s)
date = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime())
body = "To: %s\nFrom: %s\nSubject: %s\nDate: %s\n\n%s" % (toaddrs[0], fromaddr, subj, date, msg)
body = "To: %s\nFrom: %s\nSubject: %s\nDate: %s\n\n%s" % (toaddrs[0], fromemail, subj, date, msg)
try:
server = smtplib.SMTP(SMTPSERVER)
if DEBUG > 0: server.set_debuglevel(1)
server.sendmail(fromaddr, toaddrs, body)
server.sendmail(fromemail, toaddrs, body)
except smtplib.SMTPRecipientsRefused, errs:
log("cannot send email: %s\n" % (errs))
log(None, "cannot send email: %s\n" % (errs))
ret = "Fail"
except:
print("smtp error: "+traceback.format_exc())
@@ -293,9 +367,9 @@ answer
""" % D
if DEBUG > 0: log("DBG: nsup %s" % nsup)
if DEBUG > 0: log(None, "DBG: nsup %s" % nsup)
cmd = ["/usr/local/bin/nsupdate", "-k", "/etc/dhcpc/K%(domain)s.+157+00000." % D, "-v"]
if DEBUG > 0: log("DBG: cmd %s" % cmd)
if DEBUG > 0: log(None, "DBG: cmd %s" % cmd)
try:
p = Popen(cmd, shell=False, bufsize=1, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
except OSError, e:
@@ -357,17 +431,26 @@ def checkoverdue():
continue
timeout = hosts[h].interval+grace
if hosts[h].state == Host.up and now-hosts[h].lastbeat > timeout:
m = "%s is overdue" % h
if h in watchhosts:
email("overdue", m)
pushover(m)
email("overdue", "%s is overdue" % h)
pushover("%s is overdue" % h)
hosts[h].newstate(Host.overdue, grace)
log(m)
log(h, "overdue")
def log(m, service="heartbeat"):
def log(host, m, service=None):
if DEBUG > 0: print "Log: %s" % m
msg = time.strftime("%b %d %H:%M:%S", time.localtime(time.time()))+": "+m+"\n"
ts = time.strftime("%b %d %H:%M:%S", time.localtime(time.time()))
if service:
srv = "service %s: " % service
else:
srv = ""
if host:
hst = "%s " % host
else:
hst = ""
msg = "%s: %s%s%s\n" % (ts, hst, srv, m)
msgs.append(msg)
if logfmt == "msg":
m2 = "%d|%s|%s\n" % (now, service, m)
@@ -393,34 +476,63 @@ def dnsupdatethread():
#
#
def fromaddr(name, addr, boot, interval, acks):
global htab
newh=False
#
#
def readsock(sock):
if DEBUG > 3: sys.stderr.write("readsock recfrom start")
data, addr = sock.recvfrom(MAXRECV)
# now = time.time()
if DEBUG > 2: sys.stderr.write("readsock = %s, %s\n" % (data,addr))
msg = stodict(data)
if not msg: # Old hbc client
if verbose: print "old hbc:", data
oldclient = True
msg = oldmtodict(data)
else:
oldclient = False
if verbose: print "readsock = %s, %s" % (msg,addr)
name = msg.get('name', "unknown")
if not name in hosts: # was: hosts.has_key(name):
host = Host(name)
newh=True
else:
host = hosts[name]
newh=False
host.doesack = acks
res = host.newaddr(addr)
if res:
m = "%s %s" % (host.name, res)
log(m)
if name in dyndnshosts and isIPv4(addr): # don't try and cat ptr to IPv6 addr
if DEBUG > 0: print "dbg: dynhost and %s: %s" % (isIPv4(addr), addr)
dnsQ.put((name, addr))
host.doesack = msg.get('acks', -1)
host.setcver(msg.get('ver', 0))
host.lastbeat = now
interval = msg.get('interval', 0)
shutdown = msg.get('shutdown', 0)
service = msg.get('service', "unknown")
message = msg.get('msg', None)
boot = msg.get('boot', 0)
host.latency = now - msg.get('time', 0)
if boot:
log(name, "booted")
if name in watchhosts:
email("address change", m)
email("booted", m)
pushover(m)
if message:
log(name, "msg: %s" % message, service=service)
if name in watchhosts:
email("msg", m)
pushover(m)
res = host.newaddr(addr[0])
if res:
log(name, res)
if name in watchhosts:
email("address change", "%s %s" % (host.name, res))
pushover(m)
host.lastbeat = now
if host.getstate() != Host.up and interval > 0:
lasts = host.state
d = host.newstate(Host.up)
m = "%s, back after being %s for %s" % (host.name, lasts, dur(d))
log(m)
m = "back after being %s for %s" % (lasts, dur(d))
log(name, m)
if name in watchhosts:
email("back", name)
pushover("%s is back" % name)
@@ -430,101 +542,78 @@ def fromaddr(name, addr, boot, interval, acks):
host.upcount += 1
#
#
def readsock(sock):
global htab
if DEBUG > 3: sys.stderr.write("readsock recfrom start")
data, addr = sock.recvfrom(1024)
if DEBUG > 2: sys.stderr.write("readsock = %s, %s\n" % (data,addr))
pairs = string.split(data, ';')
boot = 0
shutdown = 0
name = "unknown"
service = "unknown"
msg = None
interval = 0
deltaT = 0.0
acks = -1
for pair in pairs:
l = string.split(pair, "=")
key = l[0]
if len(l) != 2:
val = "0"
else:
val = l[1]
if key == 'boot':
boot += 1
elif key == 'shutdown':
shutdown += 1
elif key == 'interval':
interval = int(val)
elif key == 'name':
name = shortname(val)
elif key == 'msg':
msg = val
elif key == 'service':
service = val
elif key == 'time':
try:
deltaT = now-float(val)
except:
pass
elif key == 'acks':
try:
acks = int(val)
except:
acks = -1
if boot:
if acks == -1:
a = "(%s)" % acks
else:
a = ""
m = "%s booted, deltaT %0.2g sec %s" % (name, deltaT, a)
log(m)
if name in watchhosts:
email("booted", m)
pushover(m)
if msg:
m = "%s msg: %s" % (name, msg)
log(m, service=service)
if name in watchhosts:
email("msg", m)
pushover(m)
fromaddr(name, addr[0], boot, interval, acks)
if shutdown:
m = "%s shutdown" % name
log(m)
log(name, "shutdown")
if name in watchhosts:
email("shutdown", m)
email("shutdown", "%s shutdown" % name)
pushover(m)
try:
hosts[name].newstate(Host.down)
host.newstate(Host.down)
except:
pass
if interval > 0:
try:
hosts[name].interval = interval
host.interval = interval
except:
pass
rmsg="ACK"
if len(hosts[name].cmds):
rmsg=hosts[name].cmds[0]
msg="command '%s' initiated" % hosts[name].cmds[0]
email("%s cmd exec" % name, msg)
pushover(msg)
del hosts[name].cmds[0]
log("%s command initiated" % name)
try:
ss=sock.sendto(rmsg, addr)
rmsg = {'time': time.time()}
op = 'ACK'
if host.cver < 1:
opkt = 'ACK'
rmsg = 'ACK'
else:
opkt = dicttos('ACK', rmsg)
ss=sock.sendto(opkt, addr)
if verbose: print "sendto1: %s (%s) %s %s" % (addr, len(opkt), op, str(rmsg)[:50])
# send any commands we have queued
while len(host.cmds):
op, rmsg = host.cmds[0]
if op == 'CMD':
email("%s cmd exec" % name, "command '%s' sent" % rmsg)
del host.cmds[0]
log(name, "command sent")
if host.cver < 1:
rmsg = rmsg['cmd']
elif op == 'UPD':
del host.cmds[0]
log(name, "update initiated")
if host.cver < 1:
log(name," ver 0 does not support UPD")
continue
if host.cver < 1:
opkt = rmsg
op = ""
else:
opkt = dicttos(op, rmsg, True)
try:
ss=sock.sendto(opkt, addr)
except Exception as e:
print "opkt len is %s" % len(opkt)
print "cannot send: %s" % e
if verbose: print "sendto2: %s (%s) %s %s" % (addr, len(opkt), op, str(rmsg)[:50])
if DEBUG > 2: print "msg from %s,%s, sent %s bytes back" % (addr[0], addr[1], ss)
except:
pass
def updatecode(ucode, uname):
fail = None
try:
fh = open(ucode, "r")
new_code = fh.read()
fh.close()
except Exception as e:
fail = "cannot read new code: %s" % e
if not fail:
m = md5.new()
m.update(new_code)
icsum = m.hexdigest()
rmsg = {'csum': icsum, 'code': new_code.encode('base64','strict') }
hosts[uname].cmds.append(('UPD',rmsg))
return fail
#
#
@@ -563,6 +652,7 @@ class HttpHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def buildpage(self):
res=self.buildhead(refresh=60)
res.append("<H2>Heartbeat status %s</h2><h4> %s (%s)</H4>" % (VER, time.strftime("%H:%M:%S", time.localtime(now)), os.environ.get('TZ', 'CET-1CDT')))
res.append("<table>")
res.append("<tr><th>Host</th><th>State</th><th>Hr</th><th>Dy</th><th>Wk</th><th>IP4 Addr</th><th>IP6 Addr</th><th>Last change</th></tr>\n")
hosts_sorted = hosts.keys()
@@ -571,6 +661,7 @@ class HttpHandler(BaseHTTPServer.BaseHTTPRequestHandler):
res.append("<tr><td>%-24s</td><td>%-7s</td>%s<td>%-16s</td><td>%-16s</td><td>%-17s</td></tr>\n" % \
(h, hosts[h].dispstate(), hosts[h].dispstats(), hosts[h].addr4, hosts[h].addr6, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(hosts[h].statetime))))
res.append("</table>")
res.append("<h4>Log of Events</h4>")
for m in msgs[len(msgs)-30:]:
res.append("%s<BR>" % m)
@@ -624,9 +715,14 @@ class HttpHandler(BaseHTTPServer.BaseHTTPRequestHandler):
if uarg[1][:2] == "c=":
ucmd=uarg[1][2:]
if ucmd != "" and uname != "" and hosts.has_key(uname):
hosts[uname].cmds.append(urllib.unquote(ucmd))
rmsg = {'cmd': urllib.unquote(ucmd)}
hosts[uname].cmds.append(('CMD', rmsg))
res=self.buildhead()
res.append("2Done")
else:
code=400
cause='arg 2 error'
res=self.builderror(code, cause, "ucmd %s uname %s" % (ucmd, uname))
elif upar[0] == "/d": # drop host /d?h=melschserver
if len(uarg) != 1 or len(uarg[0]) < 3:
@@ -637,8 +733,8 @@ class HttpHandler(BaseHTTPServer.BaseHTTPRequestHandler):
if uarg[0][:2] == "h=":
uname=uarg[0][2:]
if uname != "" and hosts.has_key(uname):
log(uname, "dropped")
del hosts[uname]
log("%s dropped" % uname)
res=self.buildhead()
res.append("Done")
@@ -657,13 +753,36 @@ class HttpHandler(BaseHTTPServer.BaseHTTPRequestHandler):
else:
ll="name %s not found" % uname
res.append(ll)
log(ll)
log(uname, ll)
elif upar[0] == "/u": # update
uname=""
ucode=""
if len(uarg) != 2 or len(uarg[0]) < 3 or len(uarg[1]) < 3:
code=400
cause='Argument error'
res=self.builderror(code, cause, "need h= and c= arguments")
else:
if uarg[0][:2] == "h=":
uname=uarg[0][2:]
if uarg[1][:2] == "c=":
ucode=uarg[1][2:]
if ucode != "" and uname != "" and hosts.has_key(uname):
err = updatecode(ucode, urllib.unquote(uname))
res=self.buildhead()
res.append("3 Done: %s" % err if err else "OK")
else:
code=400
cause='Argument error'
res=self.builderror(code, cause, "host not found")
elif upar[0] == "/r": # restart
res=self.buildhead()
res.append("restart request")
xsig=signal.SIGHUP
log("restart request")
log(None, "restart request")
else:
code=404
@@ -715,7 +834,7 @@ def closeup():
except:
pass
log("restarting")
log(None, "restarting")
try:
logf.close()
except:
@@ -879,7 +998,7 @@ if os.path.exists(PICKFILE):
for h in hosts.keys():
hosts[h].fixup()
for h in drophosts:
if h in hosts: # was: hosts.has_key(h):
if h in hosts:
del hosts[h]
@@ -887,7 +1006,7 @@ now = time.time()
startsec = int(now) % interval
log("Starting %s" % VER)
log(None, "Starting %s" % VER)
atexit.register(on_exit)
ilist = []