shell-scripts

Shell scripts mentioned on the concealed.world website.
git clone git://git.concealed.world/shell-scripts
Log | Files | Refs | README | LICENSE

popularity (7013B)


      1 #!/bin/bash
      2 #
      3 # Check the number of GETs of each article, to ascertain their apparent
      4 # popularity, and output HTML graphic for web page display.
      5 #
      6 # DEPENDS: Bash, coreutils
      7 #
      8 #    Copyright (C) <2020>  <nixx@firemail.cc>
      9 #
     10 #    This program is free software: you can redistribute it and/or modify
     11 #    it under the terms of the GNU General Public License as published by
     12 #    the Free Software Foundation, either version 3 of the License, or
     13 #    (at your option) any later version.
     14 #
     15 #    This program is distributed in the hope that it will be useful,
     16 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
     17 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     18 #    GNU General Public License for more details.
     19 #
     20 #    You should have received a copy of the GNU General Public License
     21 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
     22 
     23 # NOT USED ANY MORE
     24 
     25 usage () {
     26 cat << EOF
     27 'popularity' - Check the number of GETs of each article, to ascertain 
     28 their apparent popularity, and output HTML graphic for web page display.
     29 
     30 Usage:
     31 	'-h' | '--help':
     32 		Display this message and exit.
     33 
     34 This script should be run automatically, and you should not need to 
     35 see this message.
     36 EOF
     37 }
     38 
     39 invalid () {
     40 	echo -e "\e[0;91mERROR: Invalid argument. See usage:\e[0m\n"
     41 	usage
     42 }
     43 
     44 getarticles () {
     45 	find "${1}Media/" -type f | grep -v '_index.php' > ${2}
     46 	find "${1}Technology/" -type f | grep -v '_index.php' >> ${2}
     47 	find "${1}News/" -type f | grep -v '_index.php' >> ${2}
     48 }
     49 
     50 main () {
     51 	home="/home/alarm"
     52 
     53 	allarticlesfile=$(mktemp)
     54 	prefix="/usr/share/nginx/html/"
     55 	
     56 	getarticles ${prefix} ${allarticlesfile}
     57 
     58 	filelength=$(wc -l ${allarticlesfile} | awk '{print $1}')
     59 	count=1
     60 
     61 	hostlog="/usr/share/nginx/logs/host.access.log"
     62 	lastcheck="${home}/.lastcheck-popularity"
     63 	locallog="${home}/.hostlog-popularity"
     64 	countfile="${home}/.popularity"
     65 	if [[ $(cat "${lastcheck}" | wc -L) == 0 ]]; then
     66 		cat "${hostlog}" | sed -r s/\x1b\[[^@-~]*[@-~]//g > "${locallog}"
     67 	else
     68 		if [[ $(grep -a "$(cat ${lastcheck})" "${hostlog}") ]]; then
     69 			grep -Pzo ".*$(cat ${lastcheck})(.*\n)*" "${hostlog}" | sed -r s/\x1b\[[^@-~]*[@-~]//g > "${locallog}"
     70 		else
     71 			exit 1
     72 		fi
     73 	fi
     74 
     75 	torhostlog="/usr/share/nginx/logs/host_tor.access.log"
     76 	torlastcheck="${home}/.lastcheck-popularity-tor"
     77 	torlocallog="${home}/.hostlog-popularity-tor"
     78 	torcountfile="${home}/.popularity-tor"
     79 	if [[ $(cat "${torlastcheck}" | wc -L) == 0 ]]; then
     80 		cat "${torhostlog}" | sed -r s/\x1b\[[^@-~]*[@-~]//g > "${torlocallog}"
     81 	else
     82 		if [[ $(grep -a "$(cat ${torlastcheck})" "${torhostlog}") ]]; then
     83 			grep -Pzo ".*$(cat ${torlastcheck})(.*\n)*" "${torhostlog}" | sed -r s/\x1b\[[^@-~]*[@-~]//g > "${torlocallog}"
     84 		else
     85 			exit 1
     86 		fi
     87 	fi
     88 
     89 	outputfile=$(mktemp)
     90 	awkprefix=$(echo ${prefix} | head -c -2)
     91 	[[ $(cat "${lastcheck}" | wc -L) == 0 || $(cat "${torlastcheck}" | wc -L) == 0 ]] && rm ${lastcheck} ${torlastcheck}
     92 
     93 	while [[ ${count} -le ${filelength} ]]; do
     94 		newcount=0
     95 		tornewcount=0
     96 		curarticle=$(head -n ${count} ${allarticlesfile} | tail -n 1 | awk -F "${awkprefix}" '{print $2}' | rev | sed 's/\//=tsop?php.xedni_\//' | rev)
     97 		echo -n ${curarticle} >> ${outputfile}
     98 		if [[ -z $(grep ${curarticle} ${countfile}) ]]; then
     99 			echo -n ${curarticle} >> ${countfile}
    100 			newcount=1
    101 		fi
    102 		if [[ -z $(grep ${curarticle} ${torcountfile}) ]]; then
    103 			echo -n ${curarticle} >> ${torcountfile}
    104 			tornewcount=1
    105 		fi
    106 
    107 		hitcount=$(grep -a "GET ${curarticle}" ${locallog} | wc -l)
    108 		if [[ ${newcount} == 1 ]]; then
    109 			addcount=${hitcount}
    110 			echo " ${addcount}" >> ${countfile}
    111 		else
    112 			addcount=$(grep ${curarticle} ${countfile} | awk '{print $2}')
    113 			addcount=$((addcount+hitcount))
    114 			delline=$(grep -n ${curarticle} ${countfile} | cut -d : -f 1)
    115 			sed -i ${delline}d ${countfile}
    116 			echo "${curarticle} ${addcount}" >> ${countfile}
    117 		fi
    118 		echo -n " ${addcount}" >> ${outputfile}
    119 
    120 		torhitcount=$(grep -a "GET ${curarticle}" ${torlocallog} | wc -l)
    121 		if [[ ${tornewcount} == 1 ]]; then
    122 			toraddcount=${torhitcount}
    123 			echo " ${torhitcount}" >> ${torcountfile}
    124 		else
    125 			toraddcount=$(grep ${curarticle} ${torcountfile} | awk '{print $2}')
    126 			toraddcount=$((toraddcount+torhitcount))
    127 			tordelline=$(grep -n ${curarticle} ${torcountfile} | cut -d : -f 1)
    128 			sed -i ${tordelline}d ${torcountfile}
    129 			echo "${curarticle} ${toraddcount}" >> ${torcountfile}
    130 		fi
    131 		echo -n " ${toraddcount}" >> ${outputfile}
    132 
    133 		totalhitcount=$((addcount+toraddcount))
    134 		echo " ${totalhitcount}" >> ${outputfile}
    135 
    136 		((count++))
    137 	done
    138 
    139 	sortedoutput=$(sort -k4 -nr ${outputfile})
    140 	sortedoutputfile=$(mktemp)
    141 	echo "${sortedoutput}" > ${sortedoutputfile}
    142 	finalhtml="/usr/share/nginx/html/scripts/popularity.php"
    143 	rm ${finalhtml}
    144 
    145 	count=1
    146 	longestbarlength=$(head -n 1 ${sortedoutputfile} | awk '{print $4}')
    147 	while [[ ${count} -le ${filelength} ]]; do
    148 		articlelink=$(head -n ${count} ${sortedoutputfile} | tail -n 1 | awk '{print $1}')
    149 		articletitle="${awkprefix}$(echo ${articlelink} | sed 's/_index.php?post=//g')"
    150 		articletitle=$(head -n 1 ${articletitle} | awk -F '<h1>' '{print $2}' | awk -F '</h1>' '{print $1}' | sed 's/\\//g')
    151 
    152 		barlength=$(head -n ${count} ${sortedoutputfile} | tail -n 1 | awk '{print $2}')
    153 		percbarlength=$((100 * ${barlength} / ${longestbarlength}))
    154 
    155 		torbarlength=$(head -n ${count} ${sortedoutputfile} | tail -n 1 | awk '{print $3}')
    156 		perctorbarlength=$((100 * ${torbarlength} / ${longestbarlength}))
    157 
    158 		echo "<p class='centre'>'<a href='${articlelink}'>${articletitle}</a>'</p>" >> ${finalhtml}
    159 		echo "<div style='width: 90%; margin-right: auto; margin-left: auto; margin-bottom: 10px; height: 20px; background: #222;'>" >> ${finalhtml}
    160 		echo "<div style='width: ${percbarlength}%; float: left; background: var(--main-col); height: 100%;'>" >> ${finalhtml}
    161 		echo "<p style='color: var(--bg-col); margin: 2px;'>${barlength}</p>" >> ${finalhtml}
    162 		echo "</div>" >> ${finalhtml}
    163 		echo "<div style='width: ${perctorbarlength}%; float: left; background: var(--fg-col); height: 100%'>" >> ${finalhtml}
    164 		echo "<p style='color: var(--bg-col); margin: 2px;'>${torbarlength}</p>" >> ${finalhtml}
    165 		echo "</div>" >> ${finalhtml}
    166 		echo "</div>" >> ${finalhtml}
    167 
    168 		((count++))
    169 	done
    170 
    171 	echo "<p>This data should be valid as of $(date +%Y.%m.%d).</p>" >> ${finalhtml}
    172 
    173 	checkvalue=$(tail -n 1 ${locallog})
    174 	torcheckvalue=$(tail -n 1 ${torlocallog})
    175 	[[ -z ${checkvalue} ]] && checkvalue=$(tail -n 2 ${locallog})
    176 	[[ -z ${torcheckvalue} ]] && torcheckvalue=$(tail -n 2 ${torlocallog})
    177 	checkvalue=$(echo "${checkvalue}" | cut -d \[ -f 2 | cut -d \  -f 1)
    178 	torcheckvalue=$(echo "${torcheckvalue}" | cut -d \[ -f 2 | cut -d \  -f 1)
    179 	echo "${checkvalue}" > ${lastcheck}
    180 	echo "${torcheckvalue}" > ${torlastcheck}
    181 
    182 	rm ${allarticlesfile} ${allarticlestorfile} ${outputfile} ${sortedoutputfile} ${locallog} ${torlocallog}
    183 }
    184 
    185 while [[ $(echo "${1}" | cut -c 1) == '-' ]]; do
    186 	case "$1" in
    187 		"-h"|"--help"	)
    188 			usage
    189 			exit 0
    190 			;;
    191 		*		)
    192 			invalid
    193 			exit 1
    194 			;;
    195 	esac
    196 done
    197 
    198 [[ $# == 0 ]] && main