tim
/
webtoon-scraper.sh
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move user agent to variable

This commit is contained in:
timothy 2021-11-18 10:38:48 +10:30
parent 6cdd63ca6f
commit f86312ed0a
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
webtoon-scraper.sh
View File

@ -7,6 +7,8 @@
downloadtemp="$(mktemp)"
sortingtemp="$(mktemp)"
useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'
# Follow given url to final destination
url=$(curl -s -L -I -o /dev/null -w '%{url_effective}' "$1")
@ -29,7 +31,7 @@ echo -e "\nChapter URL: $url"
while read i; do
pictureno=$((pictureno+1))
imageurl=$(echo "$i" | cut -d'"' -f12 | cut -d'?' -f1) # Cut out the indervidual image urls
wget --no-verbose --tries=6 --user-agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36' --referer="http://www.webtoons.com/" "$imageurl" -O "$toonname/${toonname}_$(printf %03d $episodeno)/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" || echo "URL: $imageurl Name: $toonname/${toonname}_$(printf %03d $episodeno)/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" >> "$toonname/error.log"
wget --no-verbose --tries=6 --user-agent="$useragent" --referer="http://www.webtoons.com/" "$imageurl" -O "$toonname/${toonname}_$(printf %03d $episodeno)/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" || echo "URL: $imageurl Name: $toonname/${toonname}_$(printf %03d $episodeno)/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" >> "$toonname/error.log"
done < "$sortingtemp"
# Remove the last image if it is the webtoon logo