# an awk script 
# an NNTP log summary report generator
#
# NOTE: for systems that are not as yet using the new 4.3 BSD syslog
# (and therefore have nntp messages lumped with everything else), it
# would be best to invoke this script thusly:
#
#	egrep nntp syslog.old | awk -f nntp_awk > report_of_the_week
#
# because this script will include in the report all messages in the log
# that it does not recognize (on the assumption that they are errors to
# be dealt with by a human).
#
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
# May 17, 1986 - Norwegian Independence Day
#
# Recognize some new things - February 22, 1987
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
# fix "xmt is not an array" bug - March 11, 1987
# Change Elapsed/CPU fields to break out time values, HH:MM:SS
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
# Add reporting for newnews commands - August 27, 1987
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
# Add nntpxmit connection attempt counting/reporting - December 7, 1987
# Erik E. Fair <fair@ucbarpa.berkeley.edu>
#
BEGIN{
	readers = 0;
	transmit = 0;
	receive = 0;
	polled = 0;
}
### Skip stderr reports from rnews
{
	n = split($6, path, "/");
	if (path[n] == "rnews:") next;
	n = split($7, path, "/");
	if (path[n] == "rnews") next;
	host = $6;
}
$7 == "group" {
	readers = 1;
	ng[$8]++;
	next;
}
$7 == "ihave" {
	receive = 1;
	rec[host]++;
	if ($9 == "accepted") {
		rec_accept[host]++;
		if ($10 == "failed") rec_failed[host]++;
	} else if ($9 == "rejected") rec_refuse[host]++;
	next;
}
# this is from version 1.4 of nntpd
$7 == "ihave_stats" {
	receive = 1;
	rec[host] += $9 + $11 + $13;
	rec_accept[host] += $9;
	rec_refuse[host] += $11;
	rec_failed[host] += $13;
	next;
}
$7 == "connect" {
	systems[host]++;
	next;
}
# nntpxmit connection errors
# Ooooh! I *wish* awk had N dimensional arrays,
# so I wouldn't have to throw away the error message here!
$7 == "hello:" {
	conn[host]++;
	if ($8 == "Connection" && $9 == "refused")
		rmt_fail[host]++;
	else
		open_fail[host]++;
	next;
}
# we'll get stats from this, don't count conn[]
$7 == "xfer:" {
	open_fail[host]++;
# since these are expected to be few in number, we still print
# the exact error (no "next;" statement here).
}
$7 == "greeted" {
	conn[host]++;
	rmt_fail[host]++;
	next;
}
$7 == "host" && $8 == "unknown" {
	conn[host]++;
	ns_fail[host]++;
	next;
}
# nntpd connection abort - all "broken pipe" right now
$7 == "disconnect:" { next }
# syslogd shit
$7 == "repeated" { next }
# inews shit
$11 == "spooled" { next }
$7 == "exit" {
	if ($8 > 0) readers = 1;
	articles[host] += $8;
	groups[host] += $10;
	next;
}
$7 == "xmit" {
	xmt_cpu[host] += $9 + $11;
	xmt_ela[host] += $13;
	next;
}
$7 == "times" {
	cpu[host] += $9 + $11;
	ela[host] += $13;
	next;
}
$7 == "stats" {
	transmit = 1;
	conn[host]++;
	xmt[host] += $8;
	xmt_accept[host] += $10;
	xmt_refuse[host] += $12;
	xmt_failed[host] += $14;
	next;
}
#
#	For the Nth time, I wish awk had two dimensional associative
#	arrays. I assume that the last request is the same as all the
#	others in this section of logfile.
#
$7 == "newnews" {
	polled = 1;
	poll[host] ++;
	poll_asked[host] = $8;
	next;
}
$7 == "newnews_stats" {
	poll_offered[host] += $9;
	poll_took[host] += $11;
	next;
}
$7 == "post" {
	readers = 1;
	post[host]++;
	next;
}
$7 == "timeout" {
	timeout[host]++;
	timeouts = 1;
	next;
}
$7 == "unrecognized" {
	unknown[host] = 1;
	curious = 1;
}
### Print anything that we don't recognize in the report
{
	print;
}
END{
	printf("\n");
###############################################################################
### Article Exchange With Peers (other servers) Statistics                  ###
###############################################################################
	if (transmit || receive || polled)
		printf("NNTP peer article transfers\n\n");

	if (polled) for(s in poll) servers[s]++;
	if (receive) for(s in rec) servers[s]++;
	if (transmit) for(s in xmt) servers[s]++;

	if (receive) {
		printf("Article Reception (they contact us)\n");
		printf("System                  Offered  Took  Toss  Fail Toss   Elapsed       CPU  Pct\n");
		for(s in rec) {

			nrec += rec[s];
			nrec_accept += rec_accept[s];
			nrec_refuse += rec_refuse[s];
			nrec_failed += rec_failed[s];
			nrec_cpu += cpu[s];
			nrec_ela += ela[s];

			they_offered = rec[s];
			if (they_offered == 0) they_offered = 1;
			we_toss = (rec_refuse[s] / they_offered) * 100 + 0.5;

			e_hours      = ela[s] / 3600;
			e_sec        = ela[s] % 3600;
			e_min        = e_sec / 60;
			e_sec        %= 60;

			c_hours      = cpu[s] / 3600;
			c_sec        = cpu[s] % 3600;
			c_min        = c_sec / 60;
			c_sec        %= 60;

			tmp = ela[s];
			if (tmp == 0) tmp = 1;
			pct = ((cpu[s] / tmp) * 100.0 + 0.5);

			printf("%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n", s, rec[s], rec_accept[s], rec_refuse[s], rec_failed[s], we_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
		}

		e_hours      = nrec_ela / 3600;
		e_sec        = nrec_ela % 3600;
		e_min        = e_sec / 60;
		e_sec        %= 60;

		c_hours      = nrec_cpu / 3600;
		c_sec        = nrec_cpu % 3600;
		c_min        = c_sec / 60;
		c_sec        %= 60;

		they_offered = nrec;
		if (they_offered == 0) they_offered = 1;
		we_toss = (nrec_refuse / they_offered) * 100 + 0.5;

		if (nrec_ela == 0) nrec_ela = 1;
		pct = ((nrec_cpu / nrec_ela) * 100.0 + 0.5);

		printf("\n%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", "TOTALS", nrec, nrec_accept, nrec_refuse, nrec_failed, we_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
	}

###############################################################################
	if (polled) {
		printf("Article Transmission (they poll us)\n");
		printf("System                     Conn Offrd  Took   Elapsed       CPU  Pct  Groups\n");
		npoll = 0;
		npoll_offered = 0;
		npoll_took = 0;
		npoll_cpu = 0;
		npoll_ela = 0;

		for(s in poll) {
			npoll += poll[s];
			npoll_offered += poll_offered[s];
			npoll_took += poll_took[s];

			if (rec[s]) {
				printf("%-25s %5d %5d %5d  (see Article Reception)  %s\n", s, poll[s], poll_offered[s], poll_took[s], poll_asked[s]);
			} else {
				npoll_ela += ela[s];
				npoll_cpu += cpu[s];

				e_hours = ela[s] / 3600;
				e_sec   = ela[s] % 3600;
				e_min   = e_sec / 60;
				e_sec   %= 60;

				c_hours = cpu[s] / 3600;
				c_sec   = cpu[s] % 3600;
				c_min   = c_sec / 60;
				c_sec   %= 60;

				tmp = ela[s];
				if (tmp == 0) tmp = 1;
				pct = ((cpu[s] / tmp) * 100.0 + 0.5);

				printf("%-25s %5d %5d %5d %3d:%02d:%02d %3d:%02d:%02d %3d%%  %s\n", s, poll[s], poll_offered[s], poll_took[s], e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct, poll_asked[s]);
			}
		}
		printf("\n%-25s %5d %5d %5d", "TOTALS", npoll, npoll_offered, npoll_took);
		if (npoll_ela > 0 && npoll_cpu > 0) {

			e_hours = npoll_ela / 3600;
			e_sec   = npoll_ela % 3600;
			e_min   = e_sec / 60;
			e_sec   %= 60;

			c_hours = npoll_cpu / 3600;
			c_sec   = npoll_cpu % 3600;
			c_min   = c_sec / 60;
			c_sec   %= 60;

			tmp = npoll_ela;
			if (tmp == 0) tmp = 1;
			pct = ((npoll_cpu / tmp) * 100.0 + 0.5);

			printf(" %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
		} else
			printf("\n\n");
	}

###############################################################################
	if (transmit) {
		printf("Article Transmission (we contact them)\n");
		printf("System                    Offrd  Took  Toss  Fail  Pct   Elapsed       CPU  Pct\n");
		for(s in xmt) {
			we_offered = xmt[s];
			if (we_offered == 0) we_offered = 1;
			they_toss = (xmt_refuse[s] / we_offered) * 100 + 0.5;

			e_hours = xmt_ela[s] / 3600;
			e_sec   = xmt_ela[s] % 3600;
			e_min   = e_sec / 60;
			e_sec   %= 60;

			c_hours = xmt_cpu[s] / 3600;
			c_sec   = xmt_cpu[s] % 3600;
			c_min   = c_sec / 60;
			c_sec   %= 60;

			elapsed = xmt_ela[s];
			if (elapsed == 0) elapsed = 1;
			pct = ((xmt_cpu[s] / elapsed) * 100.0 + 0.5);

			printf("%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n", s, xmt[s], xmt_accept[s], xmt_refuse[s], xmt_failed[s], they_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);

			nxmt        += xmt[s];
			nxmt_accept += xmt_accept[s];
			nxmt_refuse += xmt_refuse[s];
			nxmt_failed += xmt_failed[s];
			nxmt_ela    += xmt_ela[s];
			nxmt_cpu    += xmt_cpu[s];
		}

		we_offered = nxmt;
		if (we_offered == 0) we_offered = 1;
		they_toss = (nxmt_refuse / we_offered) * 100 + 0.5;

		e_hours = nxmt_ela / 3600;
		e_sec   = nxmt_ela % 3600;
		e_min   = e_sec / 60;
		e_sec   %= 60;

		c_hours = nxmt_cpu / 3600;
		c_sec   = nxmt_cpu % 3600;
		c_min   = c_sec / 60;
		c_sec   %= 60;

		if (nxmt_ela == 0) nxmt_ela = 1;
		pct = ((nxmt_cpu / nxmt_ela) * 100.0 + 0.5);

		printf("\n%-25s %5d %5d %5d %5d %3d%% %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", "TOTALS", nxmt, nxmt_accept, nxmt_refuse, nxmt_failed, they_toss, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);

		printf("Transmission Connection Attempts         ------errors-------\n");
		printf("System                     Conn    OK    NS   Net   Rmt  Pct\n");
		for(s in xmt) {
			tot = conn[s];
			if (tot == 0) tot = 1;
			errs = rmt_fail[s] + ns_fail[s] + open_fail[s];
			ok = (conn[s] - errs);
			printf("%-25s %5d %5d %5d %5d %5d %3d%%\n", s, conn[s], ok, ns_fail[s], open_fail[s], rmt_fail[s], (100.0 * errs / tot + 0.5));
			ct_tot += conn[s];
			ct_ok  += ok;
			ct_ns  += ns_fail[s];
			ct_net += open_fail[s];
			ct_rmt += rmt_fail[s];
		}
		tot = ct_tot;
		if (tot == 0) tot = 1;
		errs = ct_ns + ct_net + ct_rmt;
		printf("\n%-25s %5d %5d %5d %5d %5d %3d%%\n\n", "TOTALS", ct_tot, ct_ok, ct_ns, ct_net, ct_rmt, (100.0 * errs / tot + 0.5));
	}

###############################################################################
### Article Readership Statistics                                           ###
###############################################################################

	if (readers) {
		printf("NNTP readership statistics\n");
		printf("System                     Conn Articles Groups Post   Elapsed       CPU  Pct\n");
		for(s in systems) {
###
### servers are different animals; they don't belong in this part of the report
###
			if (servers[s] > 0 && groups[s] == 0 && articles[s] == 0)
				continue;
###
### report the curious server pokers elsewhere
###
			if (groups[s] == 0 && articles[s] == 0 && post[s] == 0) {
				unknown[s] += systems[s];
				curious = 1;
				continue;
			}

			nconn += systems[s];
			nart += articles[s];
			ngrp += groups[s];
			npost += post[s];
			ncpu += cpu[s];
			nela += ela[s];

			e_hours      = ela[s] / 3600;
			e_sec        = ela[s] % 3600;
			e_min        = e_sec / 60;
			e_sec        %= 60;

			c_hours      = cpu[s] / 3600;
			c_sec        = cpu[s] % 3600;
			c_min        = c_sec / 60;
			c_sec        %= 60;

			elapsed = ela[s];
			if (elapsed == 0) elapsed = 1;
			pct = ((cpu[s] / elapsed) * 100 + 0.5);

			printf("%-25s %5d %8d %6d %4d %3d:%02d:%02d %3d:%02d:%02d %3d%%\n", s, systems[s], articles[s], groups[s], post[s], e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
		}

		e_hours      = nela / 3600;
		e_sec        = nela % 3600;
		e_min        = e_sec / 60;
		e_sec        %= 60;

		c_hours      = ncpu / 3600;
		c_sec        = ncpu % 3600;
		c_min        = c_sec / 60;
		c_sec        %= 60;

		if (nela == 0) nela = 1;
		pct = ((ncpu / nela) * 100 + 0.5);

		printf("\n%-25s %5d %8d %6d %4d %3d:%02d:%02d %3d:%02d:%02d %3d%%\n\n", "TOTALS", nconn, nart, ngrp, npost, e_hours, e_min, e_sec, c_hours, c_min, c_sec, pct);
	}

###############################################################################
	if (curious) {
		printf("Unknown NNTP server explorers\nSystem                     Conn\n");
		for(s in unknown) {
			printf("%-25s %5d\n", s, unknown[s]);
		}
		printf("\n");
	}
###############################################################################
	if (timeouts) {
		printf("Server timeouts\n");
		for(s in timeout) {
			printf("%-25s %5d\n", s, timeout[s]);
		}
		printf("\n");
	}
###############################################################################
	if (readers) {
		for(g in ng) {
			x = length(g);
			if (x > max) max = x;

			i = index(g, ".");
			if (i > 0) top = substr(g, 1, i - 1);
			else top = g;
			category[top] += ng[g];
		}
		fmt = sprintf("%%-%ds %%5d\n", max);

		printf("Newsgroup Request Counts (by category)\n");
		for(g in category) printf(fmt, g, category[g]);

		printf("\nNewsgroup Request Counts (by newsgroup)\n");
		for(g in ng) printf(fmt, g, ng[g]);
		printf("\n");
	}
}