sh bash wget - 在下载之前检查url文件是否存在

Posted 2021-05-07

#!/bin/bash

# simple function to check http response code before downloading a remote file
# example usage:
# if `validate_url $url >/dev/null`; then dosomething; else echo "does not exist"; fi




function validate_url(){
  if [[ `wget -S --spider $1  2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then 
        echo "URL exist $1" >> export_`date -I`.txt
   else
        echo "Error detected. $1" >> export_`date -I`.txt
  fi
}

echo "" > export_`date -I`.txt
for ((i = 1000000; i < 1000003; i++)); do
    validate_url "http://creativemedia3.rai.it/podcastcdn/raitre/ulisse/Ulisse_EP_Puntate/${i}_800.mp4"
done

#!/bin/bash

# simple function to check http response code before downloading a remote file
# example usage:
# if `validate_url $url >/dev/null`; then dosomething; else echo "does not exist"; fi

function validate_url () {
  if [[ `curl -s --head "$1" | head -n 1 | grep "HTTP/[1-3].[0-9] [23].."` ]]
  then
    # 0 = true
    return 0 
  else
    # 1 = false
    return 1
  fi 
}

start=2016-01-01
end=2016-12-31
   while [[ $start < $end ]] 
     do 
      dt=$(date -d "$start + 1 day" +"%Y-%m-%d");
      start=$(date -d "$start + 1 day" +"%Y-%m-%d");
      url=http://download.capital.it/podcast/whatever/2016/whatever-$dt.mp3;
      if validate_url $url; then wget $url -O ./mp3/whatever_${dt}.mp3; fi
     done

# http://download.capital.it/podcast/whatever/2016/whatever-2016-10-19.mp3
# http://cdn.flv.kataweb.it/deejay/audio/il_volo_del_mattino/$dt.mp3;