tim
/
webtoon-scraper.sh
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix _some_ of the mess

This commit is contained in:
timothy 2021-11-11 16:39:00 +10:30
parent da76dd91c4
commit 6cdd63ca6f
1 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
webtoon-scraper.sh
View File

@ -1,12 +1,11 @@
#!/bin/bash #!/bin/bash
# Dependencies: cut wget convert (imagemagic) openssl curl # Dependencies: cut wget curl printf
# "error.log" will list all download and convert errors # "error.log" will list all download and convert errors
# Set time file for sorting text # Set time file for sorting text
downloadtemp="/tmp/$(echo "$0" | rev | cut -d'/' -f1 | rev).$(openssl rand -hex 10).txt" downloadtemp="$(mktemp)"
sortingtemp="/tmp/$(echo "$0" | rev | cut -d'/' -f1 | rev).$(openssl rand -hex 10).txt" sortingtemp="$(mktemp)"
underscore="_" # Underscores are a vlaid character for variable names which messes up some of the naming. This is a dirty fix.
# Follow given url to final destination # Follow given url to final destination
url=$(curl -s -L -I -o /dev/null -w '%{url_effective}' "$1") url=$(curl -s -L -I -o /dev/null -w '%{url_effective}' "$1")
@ -15,7 +14,7 @@ toonname=$(echo $url | cut -d'/' -f6)
episodeno=$(echo $url | cut -d'=' -f3) episodeno=$(echo $url | cut -d'=' -f3)
eipsodename=$(echo $url | cut -d'/' -f7) eipsodename=$(echo $url | cut -d'/' -f7)
pictureno=0 pictureno=0
mkdir -p "$toonname/temp" mkdir -p "$toonname/${toonname}_$(printf %03d $episodeno)"
wget --quiet -O "$downloadtemp" "$url" wget --quiet -O "$downloadtemp" "$url"
if [ ! $? -eq 0 ]; then if [ ! $? -eq 0 ]; then
@ -30,14 +29,16 @@ echo -e "\nChapter URL: $url"
while read i; do while read i; do
pictureno=$((pictureno+1)) pictureno=$((pictureno+1))
imageurl=$(echo "$i" | cut -d'"' -f12 | cut -d'?' -f1) # Cut out the indervidual image urls imageurl=$(echo "$i" | cut -d'"' -f12 | cut -d'?' -f1) # Cut out the indervidual image urls
wget --no-verbose --tries=6 --user-agent='Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0' --referer="http://www.webtoons.com/" $imageurl -O "$toonname/temp/$toonname$underscore$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" || echo "URL: $imageurl Name:$toonname/temp/$toonname$underscore$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" >> "$toonname/error.log" wget --no-verbose --tries=6 --user-agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36' --referer="http://www.webtoons.com/" "$imageurl" -O "$toonname/${toonname}_$(printf %03d $episodeno)/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" || echo "URL: $imageurl Name: $toonname/${toonname}_$(printf %03d $episodeno)/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" >> "$toonname/error.log"
done < "$sortingtemp" done < "$sortingtemp"
# Remove the last image if it is the webtoon logo # Remove the last image if it is the webtoon logo
mv "$toonname/temp/$toonname$underscore$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" "$toonname/temp/logo/$toonname$underscore$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" #mv "$toonname/temp/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg" "$toonname/temp/logo/${toonname}_$(printf %03d $episodeno)_$(printf %03d $pictureno).jpg"
# Don't bother doint this, as this will often result in an image larger than image magic can handle if [ -f "$toonname/error.log" ]; then
#convert -quality 95 -append "$toonname/temp/$toonname$underscore$(printf %03d $episodeno)*.jpg" "$toonname/$toonname$underscore$(printf %03d $episodeno)_$eipsodename.jpg" || echo -e "convert.im6 error: Input: $toonname/temp/$toonname$underscore$(printf %03d $episodeno)\*.jpg\tOutput: $toonname/$toonname$underscore$(printf %03d $episodeno)_$eipsodename.jpg\tURL: $url" >> "$toonname/error.log" echo "Error log:"
cat "$toonname/error.log"
fi
# Remove temp file # Remove temp file
rm "$sortingtemp" "$downloadtemp" rm "$sortingtemp" "$downloadtemp"