#!/bin/bash - #=============================================================================== # # FILE: hstop40.sh # # USAGE: ./hstop40.sh # # DESCRIPTION: This script will scrape hardstyle top 40 from week 1 in 2018 to present day # # OPTIONS: --- # REQUIREMENTS: jq, pup (https://github.com/ericchiang/pup) and curl # BUGS: --- # NOTES: --- # AUTHOR: YOUR NAME (krage), # ORGANIZATION: # CREATED: 11/23/2018 22:19 # REVISION: --- #=============================================================================== of = "/tmp/output.csv" echo "YEAR;WEEK;RANK;ARTIST;TITLE;MIXEDBY" > $of start_year = "2012" stop_year = "$(date +%Y)" start_week = "1" stop_week = 52 while [ " $start_year " -le " $stop_year " ]; do # iterate years first while [ " $start_week " -le " $stop_week " ]; do # then iterate weeks data = "$(curl -s " https :// www.hardstyle.com / top40 / archive / $start_year / $start_week " | pup 'td[class=" text-1 "]' json{} | jq -r " . " |recode html)" # grab data specifics and convert to json # actual script here topp = 0 #jq starts at 0, while the rank stats at 1 listeplass = 1 while [ $topp -le 39 ]; do #then iterate listings tittel = "$(echo " $data " | jq -r " . [ $topp ] .children [ 0 ] .children [ 0 ] .text ")" mixedby = "$(echo " $data " | jq -r " . [ $topp ] .children [ 0 ] .children [ 1 ] .text ")" artist = "$(echo " $data " | jq -r " . [ $topp ] .children [ 0 ] .children [ 2 ] .text ")" if [ " $artist " = "null" ]; then artist = " $mixedby " mixedby = "none" echo " $start_year / $start_week . # $listeplass . $artist - $tittel " echo " $start_year ; $start_week ; $listeplass ; $artist ; $tittel ;original" >> $of else echo " $start_year / $start_week . # $listeplass . $artist - $tittel ( $mixedby )" echo " $start_year ; $start_week ; $listeplass ; $artist ; $tittel ; $mixedby " >> $of fi let listeplass = listeplass + 1 # for neat ouput let topp = topp + 1 #move on to next listing done let start_week = $start_week + 1 # Move on to next week if [ " $start_year " = " $stop_year " ]; then # stop before hitting present-day iteration as it will fail. if [ " $start_week " = "$(date +%W)" ]; then echo "END OF FILE" echo "END OF FILE" >> $of exit 0 fi fi done start_week = 1 # reset week counter let start_year = $start_year + 1 #Move on to next year done #test