#!/bin/ksh
#	@(#)	Handle hosts data
#..#	systems:	
#..#	sites:		
#..#	level:		
#..#	publish:	y
#..#	keys:		
#..#	date:		Tue Nov  2 12:40:32 GMT 1998
#..#	vers:		4.96	03/Aug/2006-15:45


[ $0 = ${0#/} ] && this=$(pwd -H)/$0 || this=$0
tag=${0##*/}
tm=/tmp/${tag}
tmp=${tm}$$
#rm -f ${tm}*

set -A AWK $(whence nawk)
[ ${#AWK[*]} -gt 0 -a "${AWK[0]}" != no ] && alias awk=${AWK[0]}

typeset dirn=X
unset user ogm datasets

usage="Usage: $tag {cat|get|put|diff} host1 [host2 ...]
       $tag {latest|largest|highest} host1 [host2 ...]
       $tag {sets|clean} host1 [host2 ...]
       $tag copyset set1 set2
       $tag -cron"

set -A UNAME $(uname -a)
UNAME[1]=${UNAME[1]%%.*}

case "$UNAME" in
HP-UX)	;;
SunOS)	alias remsh=rsh; alias bdf='df -k' ;;
Darwin)	;;
*)
	print -u2 "No command specials coded for $UNAME"
	exit ;;
esac

hostname=${UNAME[1]}

set -A ID $(id | tr "()=," "    ")

dataname=netdata
if [ -d $dataname -a -d $dataname/bin -a -d $dataname/data -a -f $dataname/..accessmethod ]; then
	dataroot=$PWD/$dataname
elif [ -d ../../bin -a -d ../../data -a -f ../../..accessmethod ]; then
	dataroot=../..
elif dataroot=\~/$dataname && dataroot="$(eval echo $dataroot)" && [ -d $dataroot ]; then
	:
else
	print -u2 "$usage"; print -u2 "$tag: can't locate dataroot"; exit 1
fi
dataroot="$(cd $dataroot || exit 1; /bin/pwd)" || exit 1
dataset=${PWD##*/}
family=${PWD%/*}; family=${family##*/}
# This assumes that if datadir is in a standard place on this host, it will
# be in the same location on all hosts; and if not there on this host it will
# be in the same place relative to this user's HOME on all other hosts.
dataloc=/var/adm
datadir=$dataloc/$dataname/$family/$dataset
[ -d $datadir ] || datadir=\~/$dataname/$family/$dataset

accessmethodfile=$PWD/../../..accessmethod

# ssh stuff. Only check ssh-agent has details if ssh is needed
[ -f ~/.ssh/agent_env ] && . ~/.ssh/agent_env >/dev/null
sshok=0
checkssh() {
	[ "$SSH_AGENT_PID" ] && kill -0 $SSH_AGENT_PID &&
		[ "$(ssh-add -l)" ] && sshok=1 || return 1
		return 0
}

## Some David Ledger standard functions

ask() {
	# Generic question asker
	fuseage='reply=$(ask [<prompt> [<default-ans> [<valid-list>]]])'
	typeset echo askans valid
	[ "${1-}" = "-e" ] && { echo=n; shift; }
	while true; do
		print -u2 -n "${1-Reply}${2+ [}${2-}${2+]}: "
		[ ${echo-y} = n ] && stty -echo
		read askans
		[ ${echo-y} = n ] && stty echo
		[ "x${askans-}" = "xq" ] && return 2
		[ $# -le 1 ] && { echo "${askans-}"; return 0; }
		[ "x${askans-}" = "x" -a "$2" ] && { echo "$2"; return 0; }
		[ $# -eq 2 ] && { echo "${askans-}"; return 0; }
		set -A valid $*; valid[0]=""; valid[1]=""
		set -A valid ${valid[*]}
		valid[0]=" ${valid[*]} "
		[ "${valid% ${askans} *}" != "$valid" ] && {
			echo "${askans-}"; return 0
		} || {
			print -u2 "Reply must be one of $valid"
                }
	done
}


sel() {
	# These names cannot be used as the array names as they are replaced
	typeset Reserved=" Reserved ITEMS ITEM i j "
	[ "${Reserved% $1 *}" = "$Reserved" ] || {
		print -u2 "Program error.  $1 is reserved"; exit; }
	typeset ITEMS
	typeset -i i=0 j
	j=$(eval echo '${#'$1'[*]}')
	eval $(echo set -A ITEMS Quit '"${'${1}'[@]}"')
	k=$(($j - 1))
	j=$(($j + 1))
	[ "${-%x*}" = "$-" ] && clear > /dev/tty 2>&1
	[ "$title" ] && print -u2 "$title"
	typeset i=0
	select ITEM in "${ITEMS[@]}"; do
		case "$REPLY" in
		1|[Qq])		echo "exit"; return 1 ;;
		!!)		exec $this $args ;;
		!set)		set 1>&2 ;;
		[+-][xv])
				set "$REPLY"
				echo "export FDBG=\"set $REPLY\";"
				;;
		[Bb])
			[ "${ITEMS[0]}" = Back ] && echo "idx=0" && return ;;
		[2-9]|[1-9][0-9])
			[ "$REPLY" -le $j ] &&
			echo "idx=$(($REPLY - 2))" && return ;;
		"")
			[ "$2" ] && echo "idx=$2" && break ;;
		*)
			i=0
			until [ "${ITEMS[$i]}" = "$REPLY" -o $i -gt $k ]; do
				i=$(($i + 1))
			done
			[ $i -le $k ] && echo "idx=$i" && break
		;;
	esac
	done
	return 0
}

prots2mode() {
	typeset -i m x
	typeset b s
	set -A b $(echo "x$1" | sed 's/./& /g')
	x=2; m=0
	while [ $x -le 10 ]; do
		[ "${b[$x]}" = "s" ] && s[$x]=1
		[ "${b[$x]}" = "S" ] && s[$x]=1 && b[$x]="-"
		[ "${b[$x]}" = "t" ] && s[$x]=1
		[ "${b[$x]}" = "T" ] && s[$x]=1 && b[$x]="-"
		((m *= 2))
		[ "${b[$x]}" = "-" ] || m=$(($m + 1))
		x=$(($x + 1))
	done
	printf "%d%03d\n" $((${s[4]-0} * 4 + ${s[7]-0} * 2 + ${s[10]-0})) $(echo 8o${m}p | dc)
}

mode2prot() {
	[ ${q-9} -eq 0 ] && set -x
	set -A perms "" x w wx r rx rw rwx
	typeset rest=$1 mode part prot=""
	case ${#rest} in
	4)
		set -A sid "" u+t g+s g+s,u+t u+s u+s,u+t u+s,g+s u+s,g+s,u+t
		mode=$rest
		rest=${mode#?}
		echo "$(mode2prot $rest),${sid[${mode%$rest}]}"
		;;
	3)
		typeset -i i=0
		set -A who u g o
		while [ "$rest" ]; do
			mode=$rest
			rest=${mode#?}
			part=${mode%$rest}
			prot="${prot-}${prot:+,}${who[$i]}=${perms[$part]:--r}"
			i=$(($i + 1))
		done
		echo "$prot"
		;;
	1)	echo ${perms[$rest]} ;;
	*)	print -u2 "$tag: Error calling mode2prot"; exit 1 ;;
	esac
}

askowner() {
	typeset owner=""
	[ "$owners" ] || {
		set -A owners $( {
			ls -ld ../*
			cat ../*/..access
			} |
			awk 'NF < 2 { ug[$0] = 1; next }
				{  ug[$3 ":" $4] = 1 }
				END { for (x in ug) { print x } }')
		owners[${#owners[*]}]="Other"
	}
	PS3="Who may access this saved dataset? Select by number: "
	eval $(sel owners || exit 0) || exit 0
	if [ "${owners[$idx]}" = "Other" ]; then
		while [ ! "$owner" ]; do
			owner="$(ask "Other <owner>:<group> " ||
				exit 1)" || exit 1
			[ "${owner%:??*}" = "$owner" ] && {
				[ "${owner#[Qq]}" = "$owner" ] || exit 1
				print -u2 "$tag: askowner: Please enter a username and group seperated by a ':'"
				owner=""
				continue
			}
			grep -q ^${owner%:*}: /etc/passwd || {
				print -u2 "User ${owner%:*} not known"
				owner=""; continue; }
			grep -q ^${owner#*:}: /etc/group || {
				print -u2 "Group ${owner#*:} not known"
				owner=""; continue; }

		done
	else
		owner="${owners[$idx]}"
	fi
	echo "$owner"
}

askmode() {
	typeset mode=""
	set -A modes 664 660 Other
	PS3="Select required protection for saved data "
	eval $(sel modes || exit 0) || exit 0
	case ${modes[$idx]} in
	+([0-7]))	mode=${modes[$idx]} ;;
	*)
		while [ ! "$mode" ]; do
			mode="$(ask "Other mode (Octal): " ||
				exit 1)" || exit 1
			[ "${mode#[Qq]}" = "$mode" ] || exit 1
			[ $(expr "$mode" : '[0-7][0-7][0-7]') -eq 3 ] ||
				mode=""
		done
		;;
	esac
	echo "$mode"
}

adddataset() {
	typeset family="" setname="" setpath="" method="" fpath=""
	typeset -u u
	typeset -l l
	while [ $# -gt 0 ]; do
		shift
	done
	[ "$family" ] || {
		set -A options $(ls ../..)
		PS3="
Select the data Family by number: "
		eval $(sel options || exit 1) || exit
		family=${options[$idx]}
	}
	while [ -z "$setpath" ]; do
		setname="$(ask "New dataset name" || exit 1)" || exit
		setpath=$dataroot/$family/$setname
		if [ -f $setpath -o -h $setpath ]; then
			print -u2 "$family/$setname exists"
			unset setpath
		fi
		if [ -d $setpath ]; then
			print -u2 "$family/$setname in use"
			yorn "Re-use" n || unset setpath
		fi
	done
	[ -d $setpath ] || mkdir $dataroot/$family/$setname || exit
	cd $dataroot/$family/$setname || exit
	set -A options "File" "Command" "Function"
	PS3="
Select the data gathering method by number.  Files are copied,
Commands are run on the remote host; Functions can do more complex things: "
	eval $(sel options || exit 1) || exit
	method=${options[$idx]}
	case "$method" in
	File)	fpath="$(ask "Pathname" || exit 1)" || exit
		echo "$fpath" > ..f
		unset options
		options="None"
		if [ -f $fpath ]; then
			set -A ogm x $(ls -ld $fpath)
			ogm[0]=${ogm[3]}
			ogm[2]="$(prots2mode ${ogm[1]})"
			ogm[1]=${ogm[4]}
			set -A ogm ${ogm[0]} ${ogm[1]} ${ogm[2]}
			options[1]="${ogm[*]} - as on this host"
		fi
		options[${#options[*]}]="Specify"
		PS3="
Select the owner:group mode of the remote data: "
		eval $(sel options || exit 1) || exit
		case ${options[$idx]} in
		None)	;;
		Specify)
			unset ogm
			ogm="$(ask "Owner" || exit 1)" || exit
			ogm[1]="$(ask "Group" || exit 1)" || exit
			ogm[2]="$(ask "Mode (octal)" || exit 1)" || exit
			printf "%s %s %s" ${ogm[*]} > ..ogm ;;
		*)	printf "%s %s %s" ${ogm[*]} > ..ogm ;;
		esac
		;;
	Command)
		touch $setpath/..c
		print "Please edit the command sequence into $setpath/..c"
		print
		yorn "Continue" y || exit ;;
	Function)
		if [ ! -f $setpath/..p ]; then
			cat <<-FrEd > $setpath/..p
			dataget() {
			:
			}
			#
			dataput() {
			:
			}
			FrEd
		fi
		print "Please edit the functions in $setpath/..p"
		print
		yorn "Continue" y || exit ;;
	*)	print -u2 "adddataset: method $method unknown"; exit ;;
	esac
	set -A triggers Done $(ls -d ../../*/*/.[!.]+([!.]) |
		awk '/.cron./ { next }; { sub(/^.*\//, ""); print }' |
		sort -u) "Cron"
	trigs="$(ls -d $setpath/.[!.]+([!.]) |
		sed 's!^.*/!!' |
		sort -u)"
	trigs=$trigs
	PS3="
Has triggers ${trigs:--None-}
Select required triggers by number. 'Done' (2) when finished: "
	while true; do
		typeset -i i=0
		while [ $i -lt ${#triggers[*]} ]; do
			[ -f $setpath/${triggers[$i]} ] && triggers[$i]=""
			i=$(($i + 1))
		done
		set -A triggers ${triggers[*]}
		eval $(sel triggers || exit 1) || exit
		[ ${triggers[$idx]} = Done ] && break
		[ ${triggers[$idx]} = Cron ] && {
			cron=""
			while [ -z "$cron" ]; do
			cron="$(ask "Enter 3-char weekday or date, 1-31")"
			u=${cron%${cron#?}}
			l=${cron#?}
			cron=$u$l
			case "$cron" in
			Mon|Tue|Wed|Thu|Fri|Sat|Sun)	;;
			*([0-9))
				[ $cron -gt 0 -a $cron -le 31 ] ||
					print -u2 "Invalid entry"; cron="" ;;
			*)	print -u2 "Invalid entry"; cron="" ;;
			esac
			done
			touch $setpath/.cron$cron
		}
		[ "$cron" ] || touch $setpath/${triggers[$idx]}
		trigs="$(ls -d $setpath/.[!.]+([!.]) |
			sed 's!^.*/!!' |
			sort -u)"
		trigs=$trigs
		PS3="
Has triggers ${trigs:--None-}
Select required triggers by number. 'Done' (2) when finished: "
	done
	unset PS3
	askmode > ..mode || exit
}

samefile() {
	[ ${q-9} -eq 0 ] && set -x
	[ $# -ne 2 ] && { print -u2 "samefile: needs 2 args; got $*"; exit 1; }
	[ -f $1 ] || return 0	# so we dont try the copy
	unset ino vol
	ls -i $1 $2 | while read ino[${#ino[*]}]; do
		:
	done
	bdf $1 $2 | while read vol[${#vol[*]}]; do
		:
	done
	[ "${ino[0]% *}" = "${ino[1]% *}" -a "${vol[1]%% *}" = "${vol[2]%% *}" ]
	return $?
}

# lookup array host:host
lookup() {
	typeset table
	typeset -i i=0
	[ ${numOfHosts-0} -eq 0 ] && loadAccessMethods
	eval table=\" \${${1}\[\*]} XX \"
	set -A table ${table% $2 *}
	echo ${#table[*]}
}
Olookup() {
	typeset table
	typeset -i i=0
	[ ${numOfHosts-0} -eq 0 ] && loadAccessMethods
	eval set -A table \${${1}\[\*]}
	while [ $i -lt ${#table[*]} ]; do
		[ "${table[$i]}" = "$2" ] && break
		i=$(($i + 1))
	done
	echo $i
}

loadAccessMethods() {
	[ -f $accessmethodfile ] || return
	eval "$(awk 'BEGIN { i = -1 }
		{ sub(/#.*/, ""); sub(/^[ 	]*/, "") }
		NF == 0 { next }
		{ connection = sprintf("%s>%s", $1, $2); from = $1 }
		(! (connection in connections)) {
			connections[connection]++
			i++
			printf("to[%d]=\"%s:%s\";\n", i, $1, $2)
		}
		{
			printf("%sto[%d]=\"%s\";\n", $3, i, $4)
		}
		END {
			print "numOfHosts=${#to[*]}"
		}
	' $accessmethodfile)"
}

getaccessmethod() {
	[ "${accessmethod-}" ] && echo $accessmethod && return
	[ -f $accessmethodfile ] && {
		accessmethod="$(cat $accessmethodfile)"
	} || {
		accessmethod=rsh
		print -u2 "No ${accessmethodfile}, using $accessmethod"
	}
	echo accessmethod=$accessmethod
	case "$UNAME:$accessmethod" in
	*:ssh)
		echo shellmethod=ssh
		echo copymethod=scp ;;
	HP-UX:rsh)
		echo shellmethod=remsh
		echo copymethod=rcp ;;
	sunOS:rsh)
		echo shellmethod=rsh
		echo copymethod=rcp ;;
	*:ftp)
		echo shellmethod=rsh
		echo copymethod=myftp ;;
	*local)
		echo "shellmethod=\"print -u2 Local acess only\""
		echo "copymethod=\"print -u2 Local acess only\""
		print -u2 "Local acess only" ;;
	esac
	return
}

checklocn() {
	[ -d ../../data/uname ] && return 0
	print -u2 "$usage"
	print -u2 "Must be run from a data directory"
	return 1
}

checkroot() {
	[ ! -f .root -o ${ID[1]} -eq 0 ]; return $?
}

checkput() {
	[ ! -f .putroot -o ${ID[1]} -eq 0 ]; return $?
}

okforHP() { [ ! -f .IRIX64 -a ! -f .SunOS -a ! -f .notHP-UX ]; }
okforIRIX64() { [ ! -f .HP-UX -a ! -f .SunOS -a ! -f .notIRIX64 ]; }
okforSunOS() { [ ! -f .IRIX64 -a ! -f .HP-UX -a ! -f .notSunOS ]; }

okforhost() {
	set -A uname $(cat ../uname/$1 || exit 1) || exit 1
	eval okfor${uname%[!a-zA-Z0-9]*}
	return $?
}

myftp() {
	typeset hst ftpcmd src="$1" dst="$2"
	hst="${src%:*}"
	[ "$hst" = "$src" ] && {
		[ "$hst" = "$dst" ] && { print -u2 "myftp: $*"; exit 1; }
		hst="${dst%:*}"
		dst="${dst#*:}"
		ftpcmd=put
	} || {
		src="${src#*:}"
		ftpcmd=get
	}
	src=${src#\$HOME/}
	dst=${dst#\$HOME/}
	ftp $hst <<- FrEd
	asc
	$ftpcmd $src $dst
	bye
	FrEd
}

shell() {
	[ ${q-9} -eq 0 ] && set -x
	typeset user=$1
	typeset host=${user#*@}
	user=${user%@$host}
	shift || return 1
	if [ $host = ${UNAME[1]} ]; then
		(eval "$@")
	else
		if true; then
		accidx=$(lookup to ${hostname}:$host)
		[ $accidx -ge $numOfHosts ] && {
				print -u2 "$tag: shell: $host access not known"
				exit 1
			}
		#eval shellto=\${conn${hostname}shell\[$accidx]}
		case "${shellto[$accidx]}" in
		ksh)	(eval "$@") ;;
		rsh)	remsh $host ${user+-l} ${user-} "$@" ;;
		telnet)
				echo "$@"
			case "$dirn" in
			get)
				telnet $host
				print -n "Reply: "
				read ans
				echo "$ans" > $host
				;;
			put)
				cat $host
				telnet $host
				;;
			*)	print -u2 "$tag: shell: dirn $dirn unknown" ;;
			esac
			;;
		ssh)
			[ $sshok ] || checkssh || return 1
			ssh $host "$@" ;;
		*)
			[ "$viax" ] ||
			    viax=$(lookup to "${hostname}:${shellto[$accidx]}")
			[ ${viax-$numOfHosts} -ge $numOfHosts ] && {
				print -u2 "$tag: shell: $host access ${shellto[$accidx]} not known"
				exit 1
			}
			shell $user@${shellto[$viax]}
			;;
		esac
		else
		eval $(getaccessmethod $hst)
		[ $accessmethod = local ] && {
			print -u2 "Local acess only"; return 1; }
		eval $shellmethod ${user-}$host \""\$@"\"
		fi
	fi
}

copy() {
	[ ${q-9} -eq 0 ] && set -x
	typeset host src dst srf dsf
	[ $# -lt 2 ] && return 1
	src=$1
	dst=$2
	case "${src}^$dst" in
	+([!:^]):+([!:^])^+([!:^]))	# host part before ^
		srf=${src#*:}
		dsf=$dst
		host=${src%:*}
		host=${host#*@}
		dirn=get
		;;
	+([!:^])^+([!:^]):+([!:^]))	# host part after ^
		srf=$src
		dsf=${dst#*:}
		host=${dst%:*}
		host=${host#*@}
		dirn=put
		;;
	*)
		print -u2 "Cannot work out what to copy"
		print -u2 "src = \"$src\"; dst = \"$dst\""
		;;
	esac
	if [ $host = ${UNAME[1]} ]; then
		eval samefile $srf $dsf || eval cp $srf $dsf
	elif [ $host != ${UNAME[1]} ]; then
		if true; then
		accidx=$(lookup to ${hostname}:$host)
		case "${copyto[$accidx]}" in
		cp)	eval cp "$@" ;;
		rcp)	rcp "$@" ;;
		ftp)	myftp "$@" ;;
		scp)	[ $sshok ] || checkssh || return 1; scp "$@" ;;
		*)
			via="${copyto[$accidx]}"
			[ "$viax" ] || viax=$(lookup to "${hostname}:$via")
			[ ${viax-$numOfHosts} -ge $numOfHosts ] && {
				print -u2 "$tag: shell: $host access ${copyto[$accidx]} not known"
				exit 1
			}
			eval ${copyto[$viax]} "$src" $via:$datadir/$host
			echo "On $via 'cd $datadir ; data put $host'"
			shell $user@$via
			;;
		esac
		else
		eval $(getaccessmethod $host)
		[ $accessmethod = local ] && {
			print -u2 "Local acess only"; return 1; }
		eval $copymethod "$src" "$dst"
		fi
	fi
}

protect() {
	[ ${q-9} -eq 0 ] && set -x
	typeset f ld ed=$PWD
	[ "$1" -a -d $1 ] || {
		print -u2 "usage"; print -u2 "$tag: $PWD: protect: arg?"
		return 1; }
	cd $1 || return 1
	checklocn 2>/dev/null || [ -f ..access -a -f ..mode ] || exit 1
	[ -f ..mode ] && mode=$(cat ..mode) || mode=660
	set -A mode $mode; [ ${#mode[*]} -gt 1 ] && echo $mode > ..mode
	[ $(($mode +  0)) = $mode ] && smode="$(mode2prot $mode)"
	{	[ ${q-9} -gt 5 ] && exec 1>/dev/null 2>&1
		chmod $smode,ug+rwx .
		chmod $smode,ug+rwX * .??*
		chmod ug+r .??*
		smode=$(mode2prot ${mode#??})
		chmod o=-r,o+${smode} . .??*
	}
	[ -f ..access ] && owner=$(cat ..access) || {
		set -A ld $PWD $(ls -ld .)
		owner=${ld[3]}:${ld[4]}; }
	{	[ ${q-9} -gt 5 ] && exec 1>/dev/null 2>&1
		chown $owner *
		chown 0:0 . .??*
	}
	cd $ed
}

while [ $# -gt 0 ]; do
	case "$1" in
	[+-][vx])	set "$1" ;;
	-vi)    lockvi $this; exit ;;
	-view)	view $this; exit ;;
	-about)
		echo "$tag was written by David Ledger"
		echo "david.ledger@ivdcs.co.uk"
		echo "Free to use at your own risk"
		echo "Copyright 2006"
		exit ;;
	-ver)
		ver=$(egrep '#..#[ 	]*vers:' $this)
		ver=${ver#"${ver%%[1-9]*}"}
		print ${ver%%[!0-9.]*}
		exit ;;
	-h)
		$this --h | ${PAGER-more}
		exit ;;
	--h)
		echo ""
		echo "$usage"
		echo ""
		echo "$tag gathers/distributes data from/to remote hosts."
		echo ""
		echo "Local copies are kept in files named after the host it"
		echo "came from, in a directory named after the data."
		echo ""
		echo "$tag is to be run only from such a dataset directory."
		echo ""
		echo "The nature of the data gathered/distributed depends on"
		echo "the contents of '..f', '..c', '..s' and/or '..p' files in"
		echo "the current directory."
		echo ""
		echo "$tag get <hostlist>	- gets data from hosts in list."
		echo "$tag put <hostlist>	- puts data to hosts in list."
		echo "$tag cat <hostlist>	- cats data from hosts in list."
		echo "$tag diff <hostlist>	- cats data from hosts in list"
		echo "			and for each one, does a 'diff'"
		echo "			between the data read and the"
		echo "			local copy."
		echo ""
		echo "The primary tasks above are run after any other tasks,"
		echo "and only one primary may be run per invocation. The next"
		echo "group of tasks below may be included in a task list that"
		echo "may end with a primary task. Such tasks will be run in"
		echo "order. This allows command lines such as:"
		echo "	$tag latest put *"
		echo "which will perform a '$tag latest *' and then a"
		echo "'$tag put *'"
		echo ""
		echo "$tag latest <hostlist>	- determines which of the local"
		echo "			copies of the data from the"
		echo "			hosts in the list is the latest"
		echo "			which is then copied to the"
		echo "			others ready for 'put'ting to"
		echo "			those other hosts."
		echo "$tag largest <hostlist>	- as 'latest' but uses the"
		echo "			largest."
		echo "$tag highest <hostlist>	- as 'latest' but uses the"
		echo "			embedded version number. Only"
		echo "			for scripts that return a"
		echo "			version number when asked. The"
		echo "			method of asking must be in a"
		echo "			'..v' script file."
		echo ""
		echo "$tag sets <hostlist>	- compares the data files for"
		echo "			the hosts in the list. Creates"
		echo "			a '.1' file listing the names"
		echo "			of all those that are identical"
		echo "			of 'type 1', a '.2' file"
		echo "			listing those of 'type 2' etc."
		echo "			Creates a hard link from one of"
		echo "			the files listed in '.1' to '1'"
		echo "			and from one of those listed in"
		echo "			'.2' to '2' etc. Simplifies"
		echo "			comparisons."
		echo ""
		echo "$tag clean		- removes all <n>, .<n> files"
		echo "			created by 'sets' and any other"
		echo "			spurious files."
		echo ""
		echo "The next group of tasks are macros that perform the same"
		echo "task in each of the selected datasets. These may not be"
		echo "stacked on the command line. They must still, however, be"
		echo "initiated from a dataset directory."
		echo ""
		echo "$tag -cron		- scans all dataset directories"
		echo "			for '.cron<day>' and '.cron<dayno>'"
		echo "			files where <day> and <dayno> are"
		echo "			today's day of the week and the date in"
		echo "			the month. '$tag' then 'cd's into each"
		echo "			such dataset directory in turn and runs"
		echo "			    $tag get <hosts>"
		echo "			where <hosts> may be specified on the"
		echo "			command line, but defaults to those"
		echo "			already present in each dataset visited"
		echo ""
		echo "$tag .<word> <task>	- scans all dataset directories"
		echo "			that contain the trigger file .<word>."
		echo "			'cd's into each dataset and runs the"
		echo "			first task only."
		echo ""
		echo "If '-cron' and '.<word>' are used together, only datasets"
		echo "which match both conditions are visited. This is done to"
		echo "allow a root crontab entry to run only in those datasets"
		echo "requiring root under cron."
		echo ""
		echo ""
		echo "Control files"
		echo ""
		echo "There are three types of control file, all with names"
		echo "starting with at least one '.' so that wildcards can be"
		echo "used freely when referring to the actual data and the"
		echo "dataset directories."
		echo ""
		echo "Global files live at the dataset root, two levels above"
		echo "the dataset directories. E.g. the ..accessmethod file"
		echo "describes how $tag can communicate with other hosts."
		echo ""
		echo ""
		echo "Run Control files are per dataset and define restrictions"
		echo "on if or how '$tag' is run for that dataset. For example:"
		echo "  .root		- signifies that only user root may"
		echo "		access the data, so if the current user isn't"
		echo "		root, ignore the current dataset for actions"
		echo "		requiring root access."
		echo "  .putroot	- as '.root', but only for 'put'."
		echo "  .HP-UX	- only access hosts known to be running HP-UX"
		echo "		(known from ../uname/<host>)."
		echo "		Similar controls named according to the output"
		echo "		of 'uname -s' for other systems may be used."
		echo "  .manual	- entries are maintained manually."
		echo "  .redirect	- data needs root access, but can be"
		echo "		handled by a proxy dataset on the host."
		echo "  .cron\*	- as described under '-cron' above."
		echo "  .fixed	- marks a dataset as for generating a fixed"
		echo "		list of hosts only."
		echo "  .xyz	- any otherwise unused name to group datasets"
		echo "		together for some purpost. Used with"
		echo "		'$tag .<word>' above."
		echo ""
		echo ""
		echo "Connection contol files are per dataset and define which"
		echo "data is transferred and how."
		echo ""
		echo "A ..f file may specify the pathname of the file to be"
		echo "gathered/distributed"
		echo ""
		echo "A ..s file may contain function definitions defining"
		echo "the functions 'datacat', 'dataget', and, optionally,"
		echo "'dataput', that can be used by '$tag' to perform the"
		echo "'cat', 'get' and 'put' (if required) tasks.  'put' is not"
		echo "always a valid concept."
		echo ""
		echo "A ..c file may specify a command to be run on the remote"
		echo "host to generate the required data."
		echo ""
		echo "A ..p file may specify a command to be run on the remote"
		echo "host to accept and handle the data to be sent."
		echo ""
		echo "The above three files are checked in the order '..f',"
		echo "'..s', then '..c' & '..p'. The first found is used."
		echo ""
		echo "The '-allhosts' flag is a synonim for all hosts with an"
		echo "entry in the ../uname dataset. The '-' is optional."
		echo ""
		echo "The '-alldata' flag causes $tag to visit all datasets and"
		echo "to run the current command there. The '-' is optional."
		echo ""
		echo ""
		echo "Housekeeping"
		echo ""
		echo "$tag add <newdatasetname>	- sets up a new dataset."
		echo "	Works ineractively. Asks if the new dataset is 'data'"
		echo "or not (determines use of the 'data' or 'bin' tree)."
		echo "Takes the user through the simpler configuration file"
		echo "generation."
		echo ""
		echo "$tag -add		- alternative method, less used."
		echo ""
		echo "$tag -export	- produces a cpio archive of an empty"
		echo "$tag tree plus all the control files. Can be used to copy"
		echo "a '$tag' system to another host. Has to be run from"
		echo "'$tag' data directory."
		echo ""
		echo "$tag -import	- unpacks a cpio achive produced by an"
		echo "$tag -export. Used to update work files from one '$tag'"
		echo "system to one existing on another host. Has to be run"
		echo "from a '$tag' data directory."
		echo ""
		exit ;;
		-hist*)
		$tag --hist | ${PAGER-more}
		exit ;;
		--hist)
		echo "'$tag' version 4"
		echo "- Rewritten to be driven by control files and triggers"
		echo "in the dataset directories rather than embedded in case"
		echo "statements within the script."
		echo "- Enhanced the other host access methods to include ssh"
		echo "and more complex two-hop methods."
		echo "- Introduced script management in '../bin/*' datasets."
		echo "- Multi-domains not included (but could be added)."
		echo "- Renamed as '$tag'"
		echo ""
		echo "'manage' version 3"
		echo "- Introduced trigger files."
		echo "- Introduced 'cron' initiation."
		echo "- Introduced multi-domain datasets to handle different."
		echo "groups of hosts for use when many datasets do not fit the"
		echo "normal 'keep it all the same' rule across groups of"
		echo "hosts."
		echo "- Enhanced the web view, especially in the region of"
		echo "'manual' datasets."
		echo "- Renamed as 'manage'"
		echo ""
		echo "'netdata' version 2"
		echo "- First integrated version."
		echo "- Gathered the core tasks and datasets from version 1"
		echo "into a single script with and 'add' option to edit new"
		echo "dataset information into a case statement within the"
		echo "script itself."
		echo "- Added a wide range of datasets."
		echo "- Added a cgi web view of the data."
		echo "- Named as 'manage'"
		echo ""
		echo "Version 1"
		echo "- Started as a collection of scripts to gather, fix, and"
		echo "push out passwd, group and hosts files at a site where"
		echo "all such data was unmanaged."
		echo "- Rewrote the scripts to be based on a common template."
		echo ""
		echo ""
		exit ;;
	-mode2prot)	mode2prot $2; exit ;;	# for debug
	-askowner)	askowner; exit ;;	# for debug
	-protect)	protect $2; exit ;;	# for debug
	-lookup)	shift && lookup to $*; exit ;;	# for debug
	-vars)
		echo "dataroot=$dataroot"
		echo "family=$family"
		echo "dataset=$dataset"
		echo "datadir=$datadir"
		;;
	-cron*)	task[${#task[*]}]="${1#-}" ;;
	-exp*)	task[${#task[*]}]=export ;;
	-imp*)	task[${#task[*]}]=import ;;
	-prot)	task=-prot ;;
	-n)	ovrwrt=0 ;;
	-q[0-9])
		export q=${1#-q}
		[ $q -le 1 ] && set -x
		[ $q -le 5 ] && unalias null
		;;
	-remsh)	accessmethod=rsh ;;
	-rsh)	accessmethod=rsh ;;
	-ssh)	accessmethod=ssh ;;
	-add)	adddataset "$@"; exit ;;
	--)     ;;
	?(-)alldata)	alldata=1 ;;
	?(-)allauto)	allauto=1 ;;
	?(-)allroot)	allroot=1 ;;
	?(-)allhos*)
		hosts=$(ls ../uname) ;;
	-[FMSTW][aehoru][deintu])
		export wday=${1#-} ;;
	-+([0-9]))
		export mday=${1#-} ;;
	-*)
		opt="$1"
		shift
		set -- -- $(getopt hvx $opt) $@ || exit
		;;
	add)	task=add ;;
	\.+([!./]))	datasets=$(echo ../../*/*/$1 | sed "s!/$1!!g") ;;
	+([0-9]))	sets[${#sets[*]}]=$1 ;;
	@(get|cat|put|diff|highest|largest|latest|clean|sets|copyset|redirect|check))
		dirn=$1
		task[${#task[*]}]="$1" ;;
	*@)
		user="$1" ;;
	me)	hosts="${hosts-}${hosts+ }$hostname" ;;
	*/*)
		h=${1##*/}
		shift || { print -u2 "$usage"; exit; }
		set x $h "$@"
		;;
	*)
		dsname=$1	# for add
		#hosts="${hosts-}${hosts+ }$(hst $1)"
		hosts="${hosts-}${hosts+ }${1##*/}"
		;;
	esac
	shift
done

tasks=" ${task[*]} "

checklocn || exit

if [ ${alldata-0} -eq 1 ]; then
	for d in ../*; do
		[ "$d" = "../*" ] && break
		cd $d || exit
		[ -f .fixed ] && continue
		[ -f .root -a ${ID[1]} -ne 0 ] && {
			print -u2 "Need root for ${PWD##*/}"; continue; }
		[ ${q-9} -lt 5 ] && print ${d##*/}
		$this -q${q-9} ${task?"$usage"} ${hosts?"$usage"}
	done
	exit
fi

if [ ${allauto-0} -eq 1 ]; then
	for d in ../*/.auto ../*/.derived; do
		[ "$d" = "../*/.auto" ] && break
		d=${d%/.*([a-z])}
		cd $d || exit
		[ ${q-9} -lt 5 ] && print ${d##*/}
		$this -q${q-9} ${task?"$usage"} ${hosts?"$usage"}
	done
	exit
fi

if [ ${allroot-0} -eq 1 ]; then
	for d in ../*/.root ../*/.derived; do
		[ "$d" = "../*/.root" ] && break
		d=${d%/.*([a-z])}
		cd $d || exit
		[ ${q-9} -lt 5 ] && print ${d##*/}
		$this -q${q-9} ${task?"$usage"} ${hosts?"$usage"}
	done
	exit
fi

if [ "$datasets" -a "${tasks% cron *}" = "$tasks" ]; then
	for d in $datasets; do
		cd $d || exit
		[ ${q-9} -lt 5 ] && print ${d##*/}
		$this -q${q-9} ${task?"$usage"} ${hosts?"$usage"}
	done
	exit
fi

if [ "$task" = "add" ]; then
	yorn "Item is data (rather than a script)" y && dstype=data || dstype=bin
	[ "$dsname" ] || dsname="$(ask "Dataset name" || return 1)" || exit
	[ -d ../../$dstype/$dsname ] && {
		yorn "Dataset exists. Modify" n || exit
	} || mkdir ../../$dstype/$dsname || exit
	cd ../../$dstype/$dsname || exit
	set -A methods "Pathname" "Command string" "Shell function"
	set -A methnam "..f" "..c" "..s" "..p"
	set -A methqst "Pathname of file" "first line (then edit ..c)" \
		"Edit ..s" "Edit ..p"
	[ $dstype = data ] && {
		eval $(sel methods || exit 1) || exit
		getmethod=${methods[$idx]}
		getmethnm=${methnam[$idx]}
		true
	} || { idx=0; }
	[ $idx -gt 1 ] && {
		cp ../../${methnam[$idx]} ${methnam[$idx]}
		true
	} || {
		[ -f ${methnam[$idx]} ] && dflt="$(cat ${methnam[$idx]})"
		mettxt="$(ask "Enter ${methqst[$idx]}" "$dflt" ||
			exit 1)" || exit 1
		echo "$mettxt" > ${methnam[$idx]}
	}
	owner="$(askowner || exit 1)" || exit 1
	echo "$owner" > ..access
	mode="$(askmode || exit 1)" || exit 1
	echo "$mode" > ..mode
	PS3="Select triggers till Done: "
	while true; do
		set -A triggers Done Other $(ls ../*/.[a-zA-Z]* |
			awk '
				{ sub(/^.*\//, ""); tr[$1] = 1 }
				END {
					while ("ls .[a-zA-Z]*" | getline > 0) {
						print | "cat 1>&2"
						if ($0 in tr) { delete tr[$0] }
					}
					for (x in tr) { print x }
				}
			')
		eval $(sel triggers || exit 1) || exit
		case ${triggers[$idx]} in
		Done)	break ;;
		Other)
			triggers="$(ask "Trigger" || exit 1)" || exit
			touch .${triggers##.}
			;;
		.*)	touch ${triggers[$idx]} ;;
		*)	echo "?? $idx ?? ${tr[$idx]} ??"; exit ;;
		esac
	done
	exit

elif [ "$task" = "export" ]; then
		cd $dataroot || exit
		[ -d ..bin ] || mkdir ..bin || exit
		cp $this ..bin/$tag
		[ -d bin/$tag ] || mkdir -p bin/$tag || exit
		cp $this bin/$tag/$(hostname)
		cd ..
		# pwd 1>&2
		if [ "$hosts" ]; then
			find ./$dataname \( -name '\.?*' -o -type l -o -name $tag -o -path '*/..prots/*' \) |
				cpio -oa > $tmp
			[ ${numOfHosts-0} -eq 0 ] && loadAccessMethods
			for h in $hosts; do
				[ $h = $hostname ] && continue
				shell $h PATH=\$PATH:./bin \; data -import < $tmp
			done
			rm $tmp
		else
			find ./$dataname \( -name '\.?*' -o -name $tag -o -path '*/..prots/*' \) | cpio -oa
		fi
		exit
elif [ "$task" = "import" ]; then
	cd $dataloc || exit 1
	[ -d $dataname ] || {
		print -u2 "$dataname not found. Please create before importing."
		exit; }
	cpio -ivd
fi

while [ "$task" ]; do
if [ "$task" = "clean" ]; then
	rm -f +([0-9]) .+([0-9])
	task=""; set -A task ${task[*]}
fi

if [ "$task" = "sets" ]; then
	rm -f +([0-9]) .+([0-9])
	cksum ${hosts=*} | awk '{
		key = $1 $2
		sum[key] = sprintf("%s %s", $3, (key in sum) ? sum[key] : "")
		if (! (key in pri)) { pri[key] = $3 }
	}
	END {
		i = 1
		for (x in sum) {
			print sum[x] > "." i
			cmnd = sprintf("ln %s %d;\n", pri[x], i)
			system(cmnd)
			i++
		}
	}'
	set -A sets [1-9]*
	[ ${#sets[*]} -eq 1 ] && [ "${sets%\[*}" = "$sets" ] && cp 1 ..master
	task=""; set -A task ${task[*]}
fi

if [ "$task" = "copyset" ]; then
	[ ${#sets[*]} -lt 2 ] && { print -u2 "$usage"; exit 1; }
	[ -f ${sets[0]} -a -f .${sets[1]} ] || { print -u2 "$usage"; exit 1; }
	#for s in $(cat $(eval echo .${sets[1]})); do
	for s in $(cat .${sets[0]}) $(cat .${sets[1]}); do
		cp ${sets[0]} $s
	done
	cat .${sets[1]} >> .${sets[0]}
	rm ${sets[1]} .${sets[1]}
	exit
	task=""; set -A task ${task[*]}
fi

if [ "$task" = "latest" ]; then
	set -A lst $(ls -t $hosts)
	latest=$lst
	for h in $hosts; do
		[ $h = $latest ] && continue
		cp $latest $h
	done
	cp $latest ..master
	task=""; set -A task clean ${task[*]}
fi

if [ "$task" = "highest" -a -x ./..v ]; then
	set -A lst $(for h in $hosts; do
		./..v $h
	done | sort -k 2.1nr -k 2.2nr | awk '{ print $1 }')
	highest=$lst
	for h in $hosts; do
		[ $h = $highest ] && continue
		cp $highest $h
	done
	cp $highest ..master
	task=""; set -A task clean ${task[*]}
fi

if [ "$task" = "largest" ]; then
	set -A lst $(ls -l $hosts | sort -k 5nr | awk '{ print $NF }')
	largest=$lst
	for h in $hosts; do
		[ $h = $largest ] && continue
		cp $largest $h
	done
	cp $largest ..master
	task=""; set -A task clean ${task[*]}
fi

if [ "$task" = "cron" ]; then
	[ "${-%x*}" = "$-" ] || flgs=-x
	dsd=" "
	: ${wday=$(date "+%a")}
	: ${mday=$(date "+%d")}
	for ds in $(eval echo $dataroot/*/*/.cron@(|${wday-}|${mday-0})); do
		ds=${ds%/*}
		[ -d "$ds" ] || { print -u2 "$tag: nothing to do today"; exit; }
		[ "${dsd% $ds }" = "$dsd" ] || continue
		dsa=${ds##*/}
		dsa=${ds%/*}; dsb=${dsa%/*}; dsa=${ds#$dsb}
		[ "$datasets" ] && [ "${datasets%$dsa *}" = "$datasets" ] &&
			continue
		[ ${q-9} -lt 5 ] && print ${ds##*/}
		(cd $ds || exit 1; $this ${flgs-} get ${hosts-\*})
		dsd="$dsd$ds "
	done
	exit
fi

if [ "$task" = "redirect" ]; then
	for h in $hosts; do
		[ ${ovrwrt-1} -ne 1 -a -f $h ] && {
			[ ${q-9} -lt 7 ] && print -u2 "Skipping $h"; continue; }
		[ $h = $hostname ] && continue	# Skip copy to self
		[ -f ../sshhosts/$h ] && {
			ssh $h ". ./.profile; cd $datadir; /usr/local/bin/data get me"
			$tag get $h
		} || print -u2 "$tag: can't ssh to $h. Do manually"
	done
fi

if [ "$task" = "-prot" ]; then
	protect .
	task=""; set -A task ${task[*]}
fi

if [ "$task" = "clean" ]; then
	rm -f +([0-9]) .+([0-9])
	task=""; set -A task ${task[*]}
fi

if [ "$task" = "get" ]; then
	break
fi
if [ "$task" = "put" ]; then
	break
fi
if [ "$task" = "cat" ]; then
	break
fi
if [ "$task" = "diff" ]; then
	break
fi

done

# Exit if all done
[ "$task" ] || exit
# Exit if the data here is manually maintained
[ -f .manual ] && exit
[ -f .fixed ] && exit

#checkroot || [ -r .redirect -a "$(cat .redirect)" != $hostname ] || 
checkroot || [ -f .redirect ] || {
	print "Need root for any transfers"; exit; }

[ "$task" = "put" ] && { [ -f .noput ] && {
		print "Can't put this dataset"; exit
	} || checkput || { print "Need root to put data"; exit; }; }

# Get access methods for hosts if not already got
[ ${numOfHosts-0} -eq 0 ] && loadAccessMethods

[ -f ..ogm ] && { set -A ogm $(cat ..ogm); }

if [ -f .redirect -a -r .redirect -a ${ID[1]} -ne 0 -a "$hosts" != $hostname ]
then
	dataget() { copy ${user-}${user+@}${1}:$datadir/$1 $1 </dev/null; }
	datacat() { shell ${user-}${user+@}${1} cat $datadir/$1 </dev/null; }
	dataput() {
		[ -f $1 ] || { print -u2 "$1 not here"; return 1; }
		copy $1 ${user-}${user+@}${1}:$datadir/$1 </dev/null
	}
elif [ -f ..f -a -r ..f ]; then
	set -A f $(cat ..f)
	dataget() { copy ${user-}${user+@}${1}:$f $1 </dev/null; }
	datacat() { shell ${user-}${user+@}${1} cat $f </dev/null; }
	metacat() { shell ${user-}${user+@}${1} ls -ld $f </dev/null; }
	dataput() {
		[ -f $1 ] || { print -u2 "$1 not here"; return 1; }
		[ -f ..ogm ] && { set -A ogm $(cat ..ogm); }
		for f in ${f[*]}; do
			copy $1 ${user-}${user+@}${1}:$f </dev/null
			[ ${#ogm[*]} -ge 3 ] && {
				case "${ogm[0]}:${ogm[1]}:${ogm[2]}" in
				=:=:=)	;;
				=:=:*)
				shell $1 chmod ${ogm[2]} $f  2>/dev/null;;
				=:+([!=]):=)
				shell $1 chgrp ${ogm[1]} $f  2>/dev/null;;
				=:+([!=]):+([!=]))
				shell $1 chmod ${ogm[2]} $f  2>/dev/null
				shell $1 chgrp ${ogm[1]} $f  2>/dev/null;;
				+([!=]):=:=)
				shell $1 chown ${ogm[0]} $f  2>/dev/null;;
				+([!=]):=:+([!=]))
				shell $1 chmod ${ogm[2]} $f  2>/dev/null
				shell $1 chown ${ogm[0]} $f  2>/dev/null;;
				+([!=]):+([!=]):=)
				shell $1 chgrp ${ogm[1]} $f  2>/dev/null
				shell $1 chown ${ogm[0]} $f  2>/dev/null;;
				*)
				shell $1 chmod ${ogm[2]} $f  2>/dev/null
				shell $1 chgrp ${ogm[1]} $f  2>/dev/null
				shell $1 chown ${ogm[0]} $f  2>/dev/null
				;;
				esac
				ogm[0]=""; ogm[1]=""; ogm[2]=""
				set -A ogm ${ogm[*]}
				}
		done
	}
elif [ -f ..s -a -r ..s ]; then
	. ./..s
elif [ -f ..c -a -r ..c ]; then
	cmd="$(cat ..c)"
	dataget() { shell $1 "$cmd" > $1 </dev/null; }
	datacat() { shell $1 $cmd </dev/null; }
elif [ -f ..derive -a -r ..derive ]; then
	. ./..derive $hosts
	exit
else
	print -u2 "Don't know how to $task ${PWD##*/}"
	exit 1
fi
if [ -f ..p -a -r ..p ]; then
	cmd="$(cat ..p)"
	dataput() { shell $1 $cmd < $1; }
fi
	
[ "$task" = check -a -f ..ogm ] && set -A ogm $(cat ..ogm) || unset ogm

for h in $hosts; do
	h=${h%%.*}
	okforhost $h || { print -u2 "Skipping $h"; continue; }
	[ ${ovrwrt-1} -ne 1 -a -f $h ] && {
		[ ${q-9} -lt 7 ] && print -u2 "Skipping $h"; continue; }
	[ ${q-9} -lt 7 ] && print $h
	unset viax accidx
	typeset -i accidx viax
	functions filter >/dev/null && {
		eval cat $h | filter | get $1
		true
	} || [ "$task" = check ] && {
		set -A lsr $h $(metacat $h)
		[ -f $h ] && {
			set -A lsl $h $(ls -ld $h)
			[ ${lsl[5]} = ${lsr[5]} ] || {
				print "$h size differs ${lsl[5]} <> ${lsr[5]}"
			}
		}
		[ "$ogm" ] && {
			[ "$ogm" = "=" -o "$ogm" = ${lsr[3]} ] || {
				print "$h owner differs $ogm <> ${lsr[3]}"
			}
			[ "${ogm[1]}" = "=" -o "${ogm[1]}" = ${lsr[4]} ] || {
				print "$h group differs ${ogm[1]} <> ${lsr[4]}"
			}
			mode=$(prots2mode ${lsr[1]})
			[ "${ogm[2]}" = "=" -o "${ogm[2]}" = ${mode#0} ] || {
				print "$h mode differs ${ogm[2]} <> $mode"
			}
		}
		true
	} || {
		[ "$task" = diff ] && {
			print -u2 $h
			[ -f $h ] && {
				datacat $h | diff - $h
				true
			} || datacat $h
			true
		}
	} || {
		eval data$task $h
		protect .
	}
done