diff --git a/hbc b/hbc index 3463bce..e3c0510 100755 --- a/hbc +++ b/hbc @@ -33,7 +33,7 @@ import syslog PORT = 50003 INTERVAL = 10 PIDFILE = '/tmp/hbc.pid' -DBG = True +DBG = False sock = None up = True @@ -54,13 +54,16 @@ def getsock(host): try: r=socket.getaddrinfo(host, 50001, 0, 0, socket.SOL_UDP) except socket.gaierror: + logm = '%s hbc died: \n%s' % ('getsock', traceback.format_exc()) + if DBG: print logm return None - if r[0][0] == 28: + if r[0][0] in [10, 28, 30]: af_type=socket.AF_INET6 elif r[0][0] == 2: af_type=socket.AF_INET else: - return None + print "dont know this net type: %s" % r[0][0] + sys.exit(1) if verbose: syslog.syslog("socktype: %s" % af_type) sock=socket.socket(af_type, socket.SOCK_DGRAM) @@ -76,6 +79,7 @@ def socksend(msg, tohost): if sock == None: sock=getsock(tohost[0]) + if DBG: print "socksend: sending msg=%s on socket=%s" % (msg, sock) sock.sendto(msg, tohost) if verbose: syslog.syslog("msg %s sent" % msg) @@ -117,6 +121,8 @@ def process(): if verbose: syslog.syslog("sock.send('%s', (%s, %s))" % (msg, hb_host, hb_port)) socksend(msg, (hb_host, hb_port)) except: + logm = '%s hbc died: \n%s' % ('socksend', traceback.format_exc()) + if DBG: print logm pass @@ -241,6 +247,7 @@ if not msgonly: msgboot.append("interval=%s" % interval) if len(msgboot) > 0: + if DBG: print "on boot" msgboot.append("name=%s" % iam) msgboot.append("time=%s" % time.time()) msgboot.append("acks=0") @@ -249,9 +256,11 @@ if len(msgboot) > 0: fail=0 for hb_host in hb_hosts: try: - if verbose: print "sock.send('%s', (%s, %s))" % (msg, hb_host, hb_port) + if DBG: print "sock.send('%s', (%s, %s))" % (msg, hb_host, hb_port) socksend(msg, (hb_host, hb_port)) except: + logm = '%s hbc died: \n%s' % ('socksend2', traceback.format_exc()) + if DBG: print logm fail=1 if fail: time.sleep(10) @@ -293,6 +302,7 @@ if fdaemon: working_directory='/tmp', umask=0o002, pidfile=pidfile, + initgroups=False, ) context.signal_map = { diff --git a/hbd b/hbd index 8926bf2..67411e4 100755 --- a/hbd +++ b/hbd @@ -2,7 +2,7 @@ # $Id: hbd,v 1.38 2013/07/14 02:25:05 andreas Exp $ # Wait for heartbeat messages and act on them (or their absence) # -VER = 1.62 +VER = 2.00 import time import os @@ -12,6 +12,7 @@ import socket import atexit import select import SocketServer +import BaseHTTPServer import getopt import signal import cPickle @@ -19,6 +20,8 @@ import smtplib import traceback import urllib import httplib +import threading +import Queue from subprocess import Popen, STDOUT, PIPE @@ -26,8 +29,8 @@ from subprocess import Popen, STDOUT, PIPE SEND_EMAIL=False SEND_PUSHOVER=True -False = 0 -True = 1 +DEBUG = 0 + LOGFILE = "/home/andreas/public_html/messages/andreas" PICKFILE = "/var/tmp/hbd.pick" AEMAIL = ["andreas@wrede.ca"] @@ -44,7 +47,7 @@ num = 0 PORT = 50003 TPORT = 50004 THOST = "" -DEBUG = False + verbose = False INTERVAL = 10 @@ -60,13 +63,10 @@ def handler(signum, frame): sig = signum if not running: if verbose: - print "NOT runing signal: %s running: %d" % (sig, running) - return -# signal.signal(sig, handler) + sys.stderr.write("NOT runing signal: %s running: %d" % (sig, running)) + sys.exit(2) if verbose: - print "signal: %s running: %s frame: %s" % (sig, running, frame) - running = False -# sys.exit(0) + sys.stderr.write("signal: %s running: %s frame: %s" % (sig, running, frame)) def shortname(name): @@ -78,6 +78,12 @@ class NullDevice: def write(self, s): pass +class Addr: + def __init__(self, a4, a6): + self.a4 = a4 + self.a6 = a6 + + class Host: up = "up" @@ -136,11 +142,11 @@ class Host: vs="" else: vs="-" - r+= "%s" % vs + r+= '%s' % vs return r else: return "(%s)" % (self.doesack) - return "N/A>" + return 'N/A>' # set new state, return number of secs in previous state def newstate(self, state, when=0): @@ -162,8 +168,7 @@ def email(s, msg): body = "To: %s\nFrom: %s\nSubject: %s\nDate: %s\n\n%s" % (toaddrs[0], fromaddr, subj, date, msg) try: server = smtplib.SMTP(SMTPSERVER) - if DEBUG: - server.set_debuglevel(1) + if DEBUG > 0: server.set_debuglevel(1) server.sendmail(fromaddr, toaddrs, body) except smtplib.SMTPRecipientsRefused, errs: log("cannot send email: %s\n" % (errs)) @@ -202,7 +207,17 @@ def nsupdate(hostname, newip): D['dnsttl'] = '5' D['newip'] = newip D['ts'] = time.strftime('%Y-%m-%d.%H:%M:%S', time.gmtime()) - nsup = """update delete %(fqdn)s A + if newip.find(":") > 0: + nsup = """update delete %(fqdn)s AAAA +update add %(fqdn)s %(dnsttl)s AAAA %(newip)s +update delete %(fqdn)s TXT +update add %(fqdn)s %(dnsttl)s TXT "Created: %(ts)s" +send +answer + +""" % D + else: + nsup = """update delete %(fqdn)s A update add %(fqdn)s %(dnsttl)s A %(newip)s update delete %(fqdn)s TXT update add %(fqdn)s %(dnsttl)s TXT "Created: %(ts)s" @@ -210,9 +225,10 @@ send answer """ % D -# log("DBG: nsup %s" % nsup) + + if DEBUG > 0: log("DBG: nsup %s" % nsup) cmd = ["/usr/local/bin/nsupdate", "-k", "/etc/dhcpc/K%(domain)s.+157+00000." % D, "-v"] -# log("DBG: cmd %s" % cmd) + if DEBUG > 0: log("DBG: cmd %s" % cmd) try: p = Popen(cmd, shell=False, bufsize=1, stdin=PIPE, stdout=PIPE, stderr=STDOUT) except OSError, e: @@ -261,9 +277,11 @@ def addhost(name, addr): # def on_exit(): - if DEBUG: - sys.stderr.write("on_exit\n") - logf.close() + if DEBUG > 0: sys.stderr.write("on_exit\n") + try: + logf.close() + except: + pass print "exit" @@ -289,7 +307,7 @@ def checkoverdue(): def log(m, service="heartbeat"): - if DEBUG: print "Log: %s" % m + if DEBUG > 0: print "Log: %s" % m msg = time.strftime("%b %d %H:%M:%S", time.localtime(time.time()))+": "+m+"\n" msgs.append(msg) if logfmt == "msg": @@ -301,6 +319,19 @@ def log(m, service="heartbeat"): pickleit() +def dnsupdatethread(): + while True: + name, addr = dnsQ.get() + m = "%s changed address to %s" % (name, addr) + err = nsupdate(name, addr) + if err: + m += ", DNS failed: %s" % err + email("error: nsupdate failed", m) + else: + m += ", DNS updated." + dnsQ.task_done() + log(m) + # # def fromaddr(name, addr, boot, interval, acks): @@ -319,14 +350,10 @@ def fromaddr(name, addr, boot, interval, acks): host.addr = addr htab[addr] = name m = "%s changed address to %s" % (host.name, addr) - if name in dyndnshosts: - err = nsupdate(name, addr) - if err: - m += ", DNS failed: %s" % err - email("error: nsupdate failed", m) - else: - m += ", DNS updated." - log(m) + if name in dyndnshosts and not ":" in addr: # don't try and cat ptr to IPv6 addr + dnsQ.put((name, addr)) + else: + log(m) if name in watchhosts: email("address change", m) pushover(m) @@ -350,7 +377,9 @@ def fromaddr(name, addr, boot, interval, acks): # def readsock(sock): global htab + if DEBUG > 3: sys.stderr.write("readsock recfrom start") data, addr = sock.recvfrom(1024) + if DEBUG > 2: sys.stderr.write("readsock = %s, %s\n" % (data,addr)) pairs = string.split(data, ';') boot = 0 shutdown = 0 @@ -434,20 +463,32 @@ def readsock(sock): log("%s command initiated" % name) try: ss=sock.sendto(rmsg, addr) - if DEBUG: - log("msg from %s,%s, sent %s bytes back" % (addr[0], addr[1], ss)) + if DEBUG > 2: print "msg from %s,%s, sent %s bytes back" % (addr[0], addr[1], ss) except: pass # # -class HtmlServer(SocketServer.TCPServer): - allow_reuse_address = True -# -# -class HtmlHandler(SocketServer.BaseRequestHandler): +#class HttpServer(BaseHTTPServer.HTTPServer): +class HttpServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): + allow_reuse_address = True + def threaded(): + pass +# +# +class HttpHandler(BaseHTTPServer.BaseHTTPRequestHandler): + + + def do_HEAD(self): + self.send_response(200) + self.send_header("Last-Modified", time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now))) +# self.send_header("Accept-Ranges","bytes") +# self.send_header("Connection","close") + self.send_header("Content-Type","text/html; charset = ISO-8859-1") + self.end_headers() + def buildhead(self, title="Heartbeat", refresh=None): res=[] @@ -458,12 +499,13 @@ class HtmlHandler(SocketServer.BaseRequestHandler): if refresh: res.append("\n" % refresh) res.append("") - res.append('') + res.append('') return res + def buildpage(self): res=self.buildhead(refresh=60) - res.append("

Heartbeat status

%s (%s)

" % (time.strftime("%H:%M:%S", time.localtime(now)), os.environ.get('TZ', 'CET-1CDT'))) + res.append("

Heartbeat status %s

%s (%s)

" % (VER, time.strftime("%H:%M:%S", time.localtime(now)), os.environ.get('TZ', 'CET-1CDT'))) res.append("") res.append("\n") hosts_sorted = hosts.keys() @@ -487,36 +529,26 @@ class HtmlHandler(SocketServer.BaseRequestHandler): res.append('

%s

' % (cause)) res.append('

%s

' % lcause) res.append('
') - res.append('
hbd (Unix) Server at %s Port %s
' % (hbd_host, hbd_port)) + res.append('
hbd (Unix) Server at %s:%s
' % (hbd_host, hbd_port)) res.append('') return res - def handle(self): - global sig, running - headers=[] - headers.append("Date: %s" % time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now))) - headers.append("Server: hbd") - headers.append("Last-Modified: %s" % time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now))) - headers.append("Accept-Ranges: bytes") - headers.append("Connection: close") - headers.append("Content-Type: text/html; charset = ISO-8859-1") - uri = '/unknown' - f = self.request.makefile() - while 1: - line = string.strip(f.readline()) - if len(line) == 0: - break - r = line.split() - if r[0] == "GET": - uri = r[1] - html = r[2] + def do_GET(self): + global sig + xsig = 0 + self.do_HEAD() + headers=[] + + if DEBUG > 2: sys.stderr.write("handle\n") + uri = self.path upar=string.split(uri,"?") if len(upar) == 1: uarg=[] else: uarg=string.split(upar[1],"&") + if DEBUG > 2: sys.stderr.write("handle = %s\n" % (uri)) code = 200 cause = "OK" if uri == "/": @@ -573,8 +605,7 @@ class HtmlHandler(SocketServer.BaseRequestHandler): elif upar[0] == "/r": # restart res=self.buildhead() res.append("restart request") - sig=signal.SIGHUP - running=False + xsig=signal.SIGHUP log("restart request") else: @@ -583,24 +614,67 @@ class HtmlHandler(SocketServer.BaseRequestHandler): res=self.builderror(code, cause, "The requested URL was not found on this server.") - self.request.send("HTTP/1.0 %s %s\r\n" % (code, cause)) + tosend = [] for h in headers: - self.request.send("%s\r\n" % h) - self.request.send("\r\n") + tosend.append("%s\r" % h) + tosend.append("\r") +# self.request.send("HTTP/1.0 %s %s\r\n" % (code, cause)) +# for h in headers: +# self.request.send("%s\r\n" % h) +# self.request.send("\r\n") - try: - self.request.send(string.join(res, "\n")) - except: - pass + tosend += res + self.wfile.write(string.join(tosend, "\n")) + if xsig: + sig = xsig + +def setrunning(new): + global running + if DEBUG > 0: sys.stderr.write("running is now = %s\n" % (new)) + running = new + + +def closeup(): + setrunning(False) + try: + sock.close() + except: + pass + try: + sock6.close() + except: + pass + if DEBUG > 0: sys.stderr.write("asking http server to stop\n") + + try: + serv.shutdown() + if DEBUG > 0: sys.stderr.write("http server stopped\n") + except Exception as e: + if DEBUG > 0: sys.stderr.write("http server did NOT stop: %s\n" % str(e)) + try: + serv.server_close() + except: + pass + + log("restarting") + try: + logf.close() + except: + pass + +# signal.signal(signal.SIGTERM, 0) + signal.signal(signal.SIGHUP, 0) + + +def restart(): + print "execv %s %s" % (sys.argv[0], [sys.argv[0]]+cmdargs) + os.execv(sys.argv[0], [sys.argv[0]]+cmdargs) + print "should not be here" def saveandrestart(): - sock.close() - sock6.close() -# serv.shutdown() #N.B. dont shutdown() as we don't use serv_forever - serv.server_close() - log("restarting") - os.execv(sys.argv[0], [sys.argv[0]]+cmdargs) + closeup() + restart() def pickleit(): @@ -643,8 +717,8 @@ for o, a in optlist: verbose = True cmdargs += [o] elif o == '-x': - DEBUG = True - + DEBUG += 1 + cmdargs += [o] if helpflag: print "hbc HeartBeatDaemon" @@ -655,7 +729,7 @@ if helpflag: print " -f run in foreground" print " -h this help" print " -v verbose" - print " -x debug" + print " -x increase debug lvl" print print """ config file can contain logfile = /var/log/heartbeat.log @@ -688,7 +762,7 @@ try: if verbose: print "notice: using config file %s" % configfile except: - print "warning: running without conifig file: %s" % configfile + print "warning: running without config file: %s" % configfile f = None if f: @@ -771,8 +845,12 @@ sock6.bind(("", hb_port)) ilist.append(sock6) -serv = HtmlServer((hbd_host, hbd_port), HtmlHandler) -ilist.append(serv.fileno()) +serv = HttpServer((hbd_host, hbd_port), HttpHandler) +servthread = threading.Thread(target=serv.serve_forever) +servthread.daemon = True +servthread.start() + +#ilist.append(serv.fileno()) if not forground: pid = os.fork() @@ -792,37 +870,49 @@ if not forground: os.setsid() os.umask(0) +dnsQ = Queue.Queue() +dnsT = threading.Thread(target=dnsupdatethread) +dnsT.daemon = True +dnsT.start() + running = True sig = 0 -signal.signal(signal.SIGTERM, handler) +#signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGHUP, handler) -next = int(now)+15 # 15 seconds time to settle after (re-)start -sleep = next - now -while running: +next = int(now)+15 # 15 seconds time to settle after (re-)start +sleep = 1 +firstcheck = int(now) + 15 - if DEBUG: - sys.stderr.write("about to sleep = %s\n" % (sleep)) +while running: + sr = None + if DEBUG > 2: sys.stderr.write("about to sleep = %s\n" % (sleep)) try: sr = select.select(ilist, [], [], sleep) now = time.time() except KeyboardInterrupt: - sys.exit(0) + sys.stderr.write("Keyboard Interrupt!\n") + running = False + closeup() + continue except select.error, value: if value[0] != 4: # interrupted system call - print select.error, value + sys.stderr.write("select err %s %s" % (select.error, value)) #raise os.error, value continue continue - except: + except Exception as e: + if DEBUG > 2: sys.stderr.write("select exception %s\n" % (str(e))) sys.exit(1) + if DEBUG > 2: sys.stderr.write("woke from sleep = %s (%s)\n" % (str(sr), str(ilist))) for fh in sr[0]: if fh in [sock, sock6]: readsock(fh) - elif fh == serv.fileno(): - serv.handle_request() +# elif fh == serv.fileno(): +# serv.handle_request() else: - print("what happend just now") + sys.stderr.write("what happend just now?\n") + if DEBUG > 2: sys.stderr.write("done handling, running is %s, sig is %s\n" % (running, sig)) # check hour/day/week for v in xrange(3): @@ -833,19 +923,21 @@ while running: for h in hosts.keys(): hosts[h].hdwcounts[v] = [hosts[h].doesack, hosts[h].upcount] - if now >= next: + if now >= next and now >= firstcheck: next = now+1 checkoverdue() sleep = next-now if sleep < 0: - sys.stderr.write("sleep is negaitive! %s next = %s\n" % (sleep, next)) + sys.stderr.write("sleep is negative! %s next = %s\n" % (sleep, next)) sleep = 0 - if DEBUG: - sys.stderr.write("sleep = %s next = %s\n" % (sleep, next)) + if DEBUG > 2: sys.stderr.write("sleep = %s next = %s\n" % (sleep, next)) + + if sig != 0: + setrunning(False) + if sig == signal.SIGHUP: - if DEBUG: - sys.stderr.write("signal 1 exit\n") + if DEBUG > 0: sys.stderr.write("signal 1 saveandrestart\n") saveandrestart()
HostStateHrDyWkIP AddrLast change