#!/bin/bash
set -bm
MAXTRIES=3
MAXP=32
cmds=( )
for i in {1..512}; do
sl=$((RANDOM % 10 + 3))
eval "function randsleep${sl}() { (set -x; sleep $(( $sl + RANDOM % 5 - 2 ));); };"
cmds+=( randsleep${sl} )
done
function runpartsp() {
local cmdqueue=( )
local pids=( )
local start_ts=( )
local tries=( )
curts=$(date +%s)
success_count=0
failed_count=0
function queue_command() {
local cmd="${1}"
local tries_made=${2}
if [ ${tries_made} -lt ${MAXTRIES} ]; then
local next_try=$(( ${2} + 1 ))
cmdqueue+=( "${1}" )
tries+=( ${next_try} )
start_ts+=( 0 )
printf "I [%d] attempt %d/%d queued #%d: '%s'\n" ${curts} ${next_try} ${MAXTRIES} ${#cmdqueue[@]} "${cmd}" >&2
else
printf "W [%d] tries exhaused: '%s'\n" ${curts} "${cmd}" >&2
(( ++failed_count ))
fi
}
for cmd in "${cmds[@]}"; do
queue_command "${cmd}" 0
done
curcmd=0
running_count=0
slpid=0
while [ ${running_count} -gt 0 -o ${curcmd} -lt ${#cmdqueue[@]} ]; do
curts=$(date +%s)
printf "I [%d] running jobs: %d, job queue: %d/%d\n" ${curts} ${running_count} ${curcmd} ${#cmdqueue[@]} >&2
if [ ${running_count} -lt ${MAXP} -a ${curcmd} -lt ${#cmdqueue[@]} ]; then
printf "I [%d] running: '%s'\n" ${curts} "${cmdqueue[$curcmd]}" >&2
"${cmdqueue[$curcmd]}" &
pid=$!
pids+=($pid)
printf "I [%d] [%d] process started\n" ${curts} ${pid} >&2
start_ts[$curcmd]=${curts}
(( ++curcmd ))
(( ++running_count ))
else
sleep 999999 &
slpid=$!
trap "trap -- \"\" SIGCHLD; kill $slpid;" SIGCHLD
wait $slpid 2>/dev/null
trap -- "" SIGCHLD
curts=$(date +%s)
for ((idx=0;idx<${#pids[@]};++idx)); do
pid=${pids[$idx]}
[ $pid -ne 0 ] || continue
error="nan"
if ! kill -0 ${pid} 2>/dev/null; then
error=0
wait ${pid} 2>/dev/null || error=$?
fi
if [ "$error" != "nan" ]; then
printf "I [%d] [%d] finished with status %d\n" ${curts} ${pid} ${error}
pids[$idx]=0
(( running_count-- ))
if [ "$error" = 0 ]; then
(( ++success_count ))
else
queue_command "${cmdqueue[$idx]}" ${tries[$idx]}
fi
fi
done
fi
done
printf "I [%d] All done: success %d, failed %d\n" ${curts} ${success_count} ${failed_count}
}
runpartsp