sh run_jobs_in_parrallel.sh

Posted 2021-05-06

#!/bin/bash

if [ $# -lt 1 ]; then
  echo "USAGE: $(basename $0) [INFILE] [NUM_THREADS]"
  exit 0
fi

infile=$1
num_threads=${2:-"4"}

# Split file into pieces
mkdir -p tmp
num_lines=$(( $(cat $infile | wc -l) / $num_threads ))
tempdir=$(mktemp -d -p tmp/ --suffix="_$(basename $infile)")
mkdir -p $tempdir/logs
split -d -l $num_lines -a 2 $infile $tempdir/jobs_

for x in $(find $tempdir -maxdepth 1 -type f); do
  x=$(basename $x)
  echo "processing $x in $(hostname) ..."
  # python ../tools/stabilize_videos_many.py $tempdir/$x \
  sh $tempdir/$x \
    > >(tee -a $tempdir/logs/${x}.std) \
    2> >(tee -a $tempdir/logs/${x}.err >&2) &
done

wait

echo "Finished !"