#!/bin/bash

set -bm

MAXTRIES=3
MAXP=32

cmds=( )

for i in {1..512}; do
	sl=$((RANDOM % 10 + 3))
	eval "function randsleep${sl}() { (set -x; sleep $(( $sl + RANDOM % 5 - 2 ));); };"
	cmds+=( randsleep${sl} )
done

function runpartsp() {
	local cmdqueue=( )
	local pids=( )
	local start_ts=( )
	local tries=( )

	curts=$(date +%s)

	success_count=0
	failed_count=0

	function queue_command() {
		local cmd="${1}"
		local tries_made=${2}

		if [ ${tries_made} -lt ${MAXTRIES} ]; then
			local next_try=$(( ${2} + 1 ))
			cmdqueue+=( "${1}" )
			tries+=( ${next_try} )
			start_ts+=( 0 )
			printf "I [%d] attempt %d/%d queued #%d: '%s'\n" ${curts} ${next_try} ${MAXTRIES} ${#cmdqueue[@]} "${cmd}" >&2
		else
			printf "W [%d] tries exhaused: '%s'\n" ${curts} "${cmd}" >&2
			(( ++failed_count ))
		fi
	}

	for cmd in "${cmds[@]}"; do
		queue_command "${cmd}" 0
	done

	curcmd=0
	running_count=0
	slpid=0

	while [ ${running_count} -gt 0 -o ${curcmd} -lt ${#cmdqueue[@]} ]; do
		curts=$(date +%s)
		printf "I [%d] running jobs: %d, job queue: %d/%d\n" ${curts} ${running_count} ${curcmd} ${#cmdqueue[@]} >&2

		if [ ${running_count} -lt ${MAXP} -a ${curcmd} -lt ${#cmdqueue[@]} ]; then
			printf "I [%d] running: '%s'\n" ${curts} "${cmdqueue[$curcmd]}" >&2
			"${cmdqueue[$curcmd]}" &
			pid=$!
			pids+=($pid)
			printf "I [%d] [%d] process started\n" ${curts} ${pid} >&2
			start_ts[$curcmd]=${curts}
			(( ++curcmd ))
			(( ++running_count ))
		else
			sleep 999999 & 
			slpid=$!
			trap "trap -- \"\" SIGCHLD; kill $slpid;" SIGCHLD
			wait $slpid 2>/dev/null
			trap -- "" SIGCHLD

			curts=$(date +%s)

			for ((idx=0;idx<${#pids[@]};++idx)); do
				pid=${pids[$idx]}
				[ $pid -ne 0 ] || continue

				error="nan"
				if ! kill -0 ${pid} 2>/dev/null; then
					error=0
					wait ${pid} 2>/dev/null || error=$?
				fi

				if [ "$error" != "nan" ]; then
					printf "I [%d] [%d] finished with status %d\n" ${curts} ${pid} ${error}
					pids[$idx]=0
					(( running_count-- ))

					if [ "$error" = 0 ]; then
						(( ++success_count ))
					else
						queue_command "${cmdqueue[$idx]}" ${tries[$idx]}
					fi
				fi
			done

		fi
	done

	printf "I [%d] All done: success %d, failed %d\n" ${curts} ${success_count} ${failed_count}
}

runpartsp