#! /bin/sh
. /usr/lib/innshellvars

##  Watch the state of the system relative to the news subsystem.
##  As controlled by the control file, when space or inodes are almost
##  exhausted, innd is throttled or paused, similarly if the load is
##  too high; when conditions revert to normal, innd is restarted.
##  No logging is done here, watch for syslog reports from innd.
##  Written by Mike Cooper <mcooper@usc.edu>.
##  Extensively modified by <kre@munnari.oz.au>.
##  Watch a log file and send mail when it gets new output by
##      Steve Groom <stevo@elroy.Jpl.Nasa.Gov>.
##  Steve's extensions merged in innwatch by
##      <Christophe.Wolfhugel@grasp.insa-lyon.fr>.

PROGNAME=innwatch
LOCK=${LOCKS}/LOCK.${PROGNAME}
DAILY=${LOCKS}/LOCK.news.daily
INNWATCHINITIALSLEEPTIME=1
CHILDPID=

##  Where to put the timestamp file (directory and filename).
TIMESTAMP=${LOCKS}/${PROGNAME}.time

##  Logfile to watch.  Comment out if no logwatch.
LOGFILE=${MOST_LOGS}/news.crit

##  Default value in case there is no definition in inn.conf.
: ${INNWATCHPAUSELOAD:=1500}
: ${INNWATCHHILOAD:=2000}
: ${INNWATCHLOLOAD:=1000}
: ${INNWATCHSPOOLSPACE:=25000}
: ${INNWATCHBATCHSPACE:=4000}
: ${INNWATCHLIBSPACE:=25000}
: ${INNWATCHSPOOLNODES:=200}
: ${INNWATCHSLEEPTIME:=600}

##  Parse JCL.
while [ $# -gt 0 ]; do
    case X"$1" in
    X-f)
        CTLWATCH=$2
        shift
        ;;
    X-f*)
        CTLWATCH=$(expr "$1" : '-f\(.*\)')
        ;;
    X-i)
        INNWATCHINITIALSLEEPTIME=$2
        shift
        ;;
    X-i*)
        INNWATCHINITIALSLEEPTIME=$(expr "$1" : '-i\(.*\)')
        ;;
    X-l)
        LOGFILE=$2
        shift
        ;;
    X-l*)
        LOGFILE=$(expr "$1" : '-l\(.*\)')
        ;;
    X-t)
        INNWATCHSLEEPTIME=$2
        shift
        ;;
    X-t*)
        INNWATCHSLEEPTIME=$(expr "$1" : '-t\(.*\)')
        ;;
    X--)
        shift
        break
        ;;
    X-*)
        echo "${PROGNAME}: Unknown flag $1" 1>&2
        exit 1
        ;;
    *)
        break
        ;;
    esac
    shift
done

##  Process arguments.
if [ $# -ne 0 ]; then
    echo "Usage: ${PROGNAME} [-f ctlfile] [-i seconds] [-l logfile] [-t seconds]" 1>&2
    exit 1
fi

##  Check the existence of the control file.
if [ ! -f "${CTLWATCH}" ]; then
    echo "Control file ${CTLWATCH} not found!" 1>&2
    exit 1
fi

trap '' 2

##  Anyone else there?
shlock -p $$ -f ${LOCK} || {
    echo "${PROGNAME}: [$$] locked by [$(cat ${LOCK})]"
    exit 0
}

trap 'rm -f ${LOCK} ${WATCHPID} ; test -n "${CHILDPID}" && kill ${CHILDPID} ; exit 1' 1 3 15
echo "$$" >${WATCHPID}

##  The reason why we turned innd off, and its, and our current state.
REASON=''
INND=''
STATE=''

trap '(
        echo "${PROGNAME} waiting for INND to start (pid: $$)"
        date
    ) >${INNWSTATUS}' 2

##  We need to remember the process ID of innd, in case one exits.
##  But we need to wait for innd to start before we can do that.
while
    PID=$(cat ${SERVERPID} 2>/dev/null)
    test -z "${PID}"
do
    sleep ${INNWATCHINITIALSLEEPTIME} &
    CHILDPID=$!
    wait
    CHILDPID=
done

trap '(
        if [ -z "${STATE}" ]; then
            echo "${PROGNAME} state RUN interval ${INNWATCHSLEEPTIME} pid $$"
        else
            echo "${PROGNAME} state ${STATE} interval ${INNWATCHSLEEPTIME} pid $$"
        fi
        if [ -z "${INND}" ]; then
            X=GO
        else
            X="${INND}"
        fi
        test -n "${REASON}" && X="${X}: ${REASON}"
        echo "INND state ${X}"
        date
    ) >${INNWSTATUS}' 2

cd ${SPOOL}

NEXTSLEEP=1
HASEXITED=false

while
    sleep ${NEXTSLEEP} &
    CHILDPID=$!
do
    wait
    CHILDPID=
    NEXTSLEEP=${INNWATCHSLEEPTIME}

    ##  If news.daily is running, idle: we don't want to change the
    ##  status of anything while news.daily may be depending on what we
    ##  have done.
    test -f "${DAILY}" && continue

    ##  Check to see if innd is running.
    ##  Notify the newsmaster if it has stopped or just restarted.
    if ctlinnd -s -t 120 mode 2>/dev/null; then
        ${HASEXITED} && {
            HASEXITED=false
            ${MAILCMD} -s "INND is now running" ${NEWSMASTER} </dev/null
        }
    else
        ${HASEXITED} || {
            HASEXITED=true
            ${MAILCMD} -s "INND is NOT running" ${NEWSMASTER} </dev/null
        }
        continue
    fi

    ##  If innd has exited & restarted, put the new one into the
    ##  same state the old one was in.
    nPID=$(cat ${SERVERPID} 2>/dev/null)
    test -n "${nPID}" -a "${PID}" -ne "${nPID}" && {
        test -n "${INND}" -a "${INND}" != go && ctlinnd -s "${INND}" "${REASON}"
        PID="${nPID}"
    }

    VALUE=0
    PREVEXP=''

    exec 3<&0
    exec 0<${CTLWATCH}

    LINE=0
    while read line; do
        LINE=$(expr ${LINE} + 1)
        test -z "$line" && continue

        ##  The first character on the line is the field delimiter,
        ##  except '#' which marks the line as a comment.
        delim=$(expr "${line}" : '\(.\).*')
        test "X${delim}" = 'X#' && continue

        ##  Parse the line into seven fields, and assign them to local vars.
        ##  You're welcome to work out what's going on with quoting in
        ##  the next few lines if you feel inclined.
        eval $(
            trap '' 2
            echo "${line}" \
                | ${SED} -e "s/'/'\"'\"'/g" \
                    -e "s/[ 	]*\\${delim}[ 	]*/\\${delim}/g" \
                | ${AWK} -F"${delim}" '{ print  "LAB='"'"'" $2 "'"'"'", \
                                                "CND='"'"'" $3 "'"'"'", \
                                                "EXP='"'"'" $4 "'"'"'", \
                                                "TST='"'"'" $5 "'"'"'", \
                                                "LIM=" $6,              \
                                                "CMD='"'"'" $7 "'"'"'", \
                                                "CMT='"'"'" $8 "'"'"'" }'
        )

        ##  If there's no label, the label is the line number.
        test -z "${LAB}" && LAB=${LINE}

        ##  Should we act on this line?  We will if one (or more) of the
        ##  specified conditions is satisfied.
        for X in a b; do # Meaningless trash because we have no goto.
            if [ -z "${CND}" ]; then
                X=-
            else
                X="${CND}"
            fi
            set -$- X ${X}
            shift
            for cnd; do
                case "${cnd}" in
                -)
                    test -n "${STATE}" -a "X${STATE}" != "X${LAB}" && continue
                    ;;
                +)
                    test -n "${STATE}" && continue
                    ;;
                '*') ;;
                -*)
                    test "X-${STATE}" = "X${cnd}" && continue
                    ;;
                *)
                    test "X${STATE}" != "X${cnd}" && continue
                    ;;
                esac
                break 2 # OK, continue with this line.
            done
            continue 2 # No, skip it.
        done

        ##  Evaluate the expression, if there is one, and if that works.
        if [ -z "${EXP}" -o "${EXP}" = "${PREVEXP}" ] \
            || {
                PREVEXP="${EXP}"
                VALUE=$(
                    trap '' 2
                    eval "${EXP}"
                )
            }; then
            ##  If innd is running, and test "succeeds", stop it.
            case "${CMD}" in
            throttle | pause)
                OK=n
                ;;
            *)
                OK=y
                ;;
            esac

            if [ \( -z "${STATE}" -o "${STATE}" != "${LAB}" -o "${OK}" = y \) \
                -a "${VALUE}" "-${TST}" "${LIM}" ]; then
                R="${CMT} [${PROGNAME}:${LAB}] ${VALUE} ${TST} ${LIM}"
                O=
                case "${CMD}" in
                throttle)
                    case "${STATE}" in
                    '' | go)
                        REASON="${R}"
                        ;;
                    *) ;;
                    esac
                    O="${LAB}"
                    ARG="${REASON}"
                    ;;
                pause)
                    O="${LAB}"
                    REASON="${R}"
                    ARG="${REASON}"
                    ;;
                shutdown)
                    ARG="${R}"
                    ;;
                flush)
                    ARG=''
                    O="${STATE}"
                    ARG="${REASON}"
                    ;;
                go)
                    ARG="${REASON}"
                    NEXTSLEEP=1
                    REASON=''
                    ;;
                exit)
                    exit 0
                    ;;
                *)
                    break
                    ;;
                esac

                ctlinnd -s "${CMD}" "${ARG}" && STATE="${O}" && INND="${CMD}"
                break

            ##  Otherwise, if innd is not running, and reverse test succeeds,
            ##  restart it.
            elif [ "${STATE}" = "${LAB}" -a \
                \( "${CMD}" = "throttle" -o "${CMD}" = pause \) -a \
                ! "${VALUE}" "-${TST}" "${LIM}" ]; then
                ctlinnd -s go "${REASON}"
                STATE=''
                REASON=''
                INND=''

                ##  If we have started innd, run all tests again quickly in
                ##  case there is some other condition that should stop it.
                NEXTSLEEP=1
                break
            fi
        fi

    done

    exec 0<&3
    exec 3<&-

    if [ -n "${LOGFILE}" -a -f "${LOGFILE}" ]; then
        if [ ! -f ${TIMESTAMP} ]; then
            DOIT=${LOGFILE}
        else
            # Use ls to print most recently modified file first.
            # If that's ${LOGFILE}, it has changed since the last pass.
            DOIT="$(ls -t ${TIMESTAMP} ${LOGFILE} | ${SED} -e 1q | grep ${LOGFILE})"
        fi

        # If the file has been modified more recently than the timestamp,
        # and the file has length greater than 0, send the warning.
        if [ -n "${DOIT}" -a -s ${LOGFILE} ]; then
            date >${TIMESTAMP}
            (
                ls -l ${LOGFILE}
                echo "-----"
                ctlinnd -t120 mode
                echo "-----"
                cat ${LOGFILE}
            ) 2>&1 \
                | sed -e 's/^~/~~/' \
                | ${MAILCMD} -s "${PROGNAME} warning: messages in ${LOGFILE}" \
                    ${NEWSMASTER}
        fi
    fi

done

rm -f ${LOCK}
