#!/bin/bash # /usr/local/bin/blog # https://crystalfaeries.net/posix/bin/blog # celeste:crystalfaery BLOG 2020-08-13 18:35:02+00:00 # This is a blog maintenance wrapper around (the PERL program) "chronicle" # highly customized for https://crystalfaeries.net/ # # Other than the usual help and version options, we accept: # -e or --edit to request an interactive edit of the blog entry for today # -l or --links to request link validation at end of run ########################################################################## let help=12 # this line # - 1 # CONFIGURATION: HOST=$(my_twin -u) # HOST TO WHICH WE UPLOAD CONTENT let hostsleep=6 # number of seconds to sleep between ssh commands to HOST # PARSE OUR INVOCATION: cd -P $HOME # there's no place like Om umask 2 # NOT world writeable false # default to non interactive (e.g. executed as a cron job) let edit=$? # default to non interactive (e.g. executed as a cron job) false let links=$? # default to not checking links while [ $# -ne 0 ] do case $# in 0 | 1 | 2) case "${1}" in -h | --help) head -n ${help} $0 exit;; -v | --version) head -n ${help} $0 exit;; -e | --edit) true let edit=$? # user wants an interactive edit of the blog source shift;; -el | -le) true let edit=$? # user wants an interactive edit of the blog source let links=$? # user wants us to check our links shift;; -l | --links) true let links=$? # user wants us to check our links shift;; *) echo "$0 does not cognize ${@}" 1>&2 shift # pedantic exec $0 -h # cluestick ;; # nevermore esac;; *) echo "$0 does not take $# arguments" 1>&2 shift # pedantic exec $0 -h # cluestick ;; # nevermore esac done ####### BLOG BEGIN ####### cat $HOME/.chronicle.log > \ $HOME/.chronicle.log- \ 2>> $HOME/.chronicle.log || exit -1 # Error: 255 Require writeable log backup limits 2>> $HOME/.chronicle.log 1>&2 || exit -1 # Error: 255 Require writeable log file echo "`now` BLOG START" \ 2>> $HOME/.chronicle.log 1>&2 || exit -2 # Error: 254 Require "now" for time-tagging # Hebrew Calendar: echo 'HCAL BEGIN' 2>> $HOME/.chronicle.log 1>&2 /usr/bin/hcal -f -h -z -10 -l 22:12:45 -L -159:24:33 > $HOME/crystalfaeries.net/fae/hcal.html 2>> $HOME/.chronicle.log || exit -56 # Error: 200 # Empty the Trash echo 'CLEAR TRASH' 2>> $HOME/.chronicle.log 1>&2 rm -rf $HOME/.local/share/Trash/* $HOME/.Trash-${UID}/* 2>> $HOME/.chronicle.log 1>&2 echo 'PERMISSIONS BEGIN' 2>> $HOME/.chronicle.log 1>&2 # NOTE: Once this is run manually to authorize correct permissions, subsequent invocations by cron should work. # We only resort to sudo when necessary. # Goto the Blog Source directory where we edit most files of blog cd -P $HOME/crystalfaeries.net/src/ \ 2>> $HOME/.chronicle.log 1>&2 || exit -3 # Error: 253 Require blog source directory # NOTE: if we link posix/ to /usr/local, beware making /usr/local/bin/* UN-executable! # Force Permissions (private content is only available as file:///) # we try to be silent and not require sudo password, but will retry with it if necessary find $HOME/crystalfaeries.net/ -type d \ \! -perm 2775 \ \! -name fey \ \! -name galleries \ \! -name lost+found \ \! -name src \ -print -exec chmod 2775 {} \; \ 2>> $HOME/.chronicle.log 1>&2 || \ sudo find $HOME/crystalfaeries.net/ -type d \ \! -perm 2775 \ \! -name fey \ \! -name galleries \ \! -name lost+found \ \! -name src \ -print -exec chmod 2775 {} \; \ 2>> $HOME/.chronicle.log 1>&2 || exit -4 # Error: 252 Permissions Unsettable # index.html hides ALL private content from http:// # only access to private content is file:/// chmod 2771 $HOME/crystalfaeries.net/fay \ 2>> $HOME/.chronicle.log 1>&2 || \ sudo chmod 2771 $HOME/crystalfaeries.net/fay \ 2>> $HOME/.chronicle.log 1>&2 || exit -5 # Error: 251 Permissions Unsettable chmod 2770 $HOME/crystalfaeries.net/{fey,galleries,src} \ 2>> $HOME/.chronicle.log 1>&2 || \ sudo chmod 2770 $HOME/crystalfaeries.net/{fey,galleries,src} \ 2>> $HOME/.chronicle.log 1>&2 || exit -6 # Error: 250 Permissions Unsettable find $HOME/crystalfaeries.net/ -type f \! -perm 664 -print -exec chmod 664 {} \; \ 2>> $HOME/.chronicle.log 1>&2 || \ sudo find $HOME/crystalfaeries.net/ -type f \! -perm 664 -print -exec chmod 664 {} \; \ 2>> $HOME/.chronicle.log 1>&2 || exit -7 # Error: 249 Permissions Unsettable echo 'EXPIRE BEGIN' 2>> $HOME/.chronicle.log 1>&2 # EXPIRE Articles containing "Expiry:" entry for f in $(grep '^Expiry: ' $HOME/crystalfaeries.net/src/*.txt | sed 's/:.*$//') do if [ $(date +%s) -gt $(grep '^Expiry: ' ${f} | cut -f2-) ] then # this article is beyond its expiry timestamp echo "Expired: ${f}" >> $HOME/.chronicle.log 2>&1 delete ${f} >> $HOME/.chronicle.log 2>&1 else # this article is within its limited lifespan echo "Visible: ${f}" >> $HOME/.chronicle.log 2>&1 fi done # UNLESS THE USER ERRED in article creation, Expiry SHOULD post-date Publish date, RIGHT? # So above we only expired already published articles, not Future Publish articles we may now reveal. # This creates a minimum exposure of once for any article before it expires, per run of this script. # NOTE: This code assumes a daily run, and may miss prepublish files for days not run!!! ADD CODE! echo 'REVEAL BEGIN' 2>> $HOME/.chronicle.log 1>&2 # NOTE: User may PREVIEW hidden files with /usr/local/bin/preview in $HOME/crystalfaeries.net/fay/ # User may generate "tomorrow's" blog entry as a hidden file # which is not published by chronicle until first run on publish date: if [ -s .`date +'%Y-%m-%d'`.fae.txt ] then # do NOT overwrite the REVEALED file with a later-dated zero-size (deleted) UNrevealed file rsync -auvH .`date +'%Y-%m-%d'`.fae.txt \ `date +'%Y-%m-%d'`.fae.txt 2>> $HOME/.chronicle.log 1>&2 \ && delete .`date +'%Y-%m-%d'`.fae.txt 2>> $HOME/.chronicle.log 1>&2 \ && touch `date +'%Y-%m-%d'`.fae.txt 2>> $HOME/.chronicle.log 1>&2 \ && touch $HOME/crystalfaeries.net/src 2>> $HOME/.chronicle.log 1>&2 # else # ignore the deleted (zero size) file while it propogates to backups/library : fi # UNCOMMENT THIS SECTION TO DAILY GENERATE DEFAULT TEMPLATE ARTICLE if [ \! -f `date +'%Y-%m-%d'`.fae.txt ] then # generate default file if user did not pre-pare today's blog entry today -a 2>> $HOME/.chronicle.log || exit -10 # Error: 246 Default Template touch $HOME/crystalfaeries.net/src fi # PREVIEW future dated articles: firefox "${HOME}"/.rawdog/status-log.html < /dev/null >& /dev/null & # Make running before many previews disown %1 sleep 6 # not too long if Firefox already running, not too short if it is not. preview 2>> $HOME/.chronicle.log 1>&2 # Update Previews of Future Blog Entries # Did we have a -e or -el or -le argument to edit source before publishing? if [ ${edit} -eq 0 ] then # User wants an interactive edit session of blog source pushd $HOME 2>> $HOME/.chronicle.log 1>&2 # Go Home to be able to delete while editing today # edit today's entry before updating blog popd 2>> $HOME/.chronicle.log 1>&2 # Return to Blog Source Directory fi if [ $(date +%s -r $HOME/crystalfaeries.net/src) -gt $(date +%s -r $HOME/crystalfaeries.net/fae) ] then # source is newer than HTML (see /etc/chroniclerc for pre and post touch ops) echo "`now` CHRONICLE START" 2>> $HOME/.chronicle.log 1>&2 || exit -11 # Error: 245 Chronicle Error # chronicle --verbose 2>> $HOME/.chronicle.log 1>&2 chronicle 2>> $HOME/.chronicle.log 1>&2 let chroniclereturn=${?} if [[ ${chroniclereturn} -eq 0 ]] then echo "`now` CHRONICLE END" 2>> $HOME/.chronicle.log 1>&2 || exit -12 # Error: 244 Chronicle Error else echo "`now` CHRONICLE ERROR: ${chroniclereturn}" 2>> $HOME/.chronicle.log 1>&2 || exit -13 # Error: 243 Chronicle Error fi # replace the "No Tags" comment at end of each article with a horizontal line: echo 'INSERT buggi_cipher_001.png' 2>> $HOME/.chronicle.log 1>&2 sed -i 's/No tags/<\/a>/g' \ $HOME/crystalfaeries.net/fae/{*.html,archive/*/*/*.html} \ 2>> $HOME/.chronicle.log 1>&2 || exit -14 # Error: 242 Tags become Lines Error # create the crompressed sitemap file: echo 'GZIP SITEMAP' 2>> $HOME/.chronicle.log 1>&2 gzip -c $HOME/crystalfaeries.net/fae/sitemap.xml > $HOME/crystalfaeries.net/sitemap.xml.gz \ 2>> $HOME/.chronicle.log || exit -15 # Error: 241 SiteMap Error # generate the recent changes pages: # echo 'RECENTLY BOOKMARKED' 2>> $HOME/.chronicle.log 1>&2 # bookmarksbydate > $HOME/crystalfaeries.net/src/.bookmarksbydate.`/bin/hostname`.txt \ # 2>> $HOME/.chronicle.log 1>&2 || exit -16 # Error: 240 Recently BookMarked Error echo 'RECENTLY LINKED' 2>> $HOME/.chronicle.log 1>&2 recently_linked 2>> $HOME/.chronicle.log 1>&2 || exit -17 # Error: 239 Recently Linked Error echo 'RECENTLY UPDATED' 2>> $HOME/.chronicle.log 1>&2 recently_updated 2>> $HOME/.chronicle.log 1>&2 || exit -18 # Error: 238 Recently Updated Error fi cd -P $HOME/crystalfaeries.net/ 2>> $HOME/.chronicle.log 1>&2 || exit -45 # Error: 211 echo "CONTENTS" 2>> $HOME/.chronicle.log 1>&2 find . -maxdepth 1 -name '{.,}contents*' \! -name '.*.swp' -newer contents.html -exec blog_contents \; \ 2>> $HOME/.chronicle.log 1>&2 echo "FDEDUPE WEBSITE" 2>> $HOME/.chronicle.log 1>&2 #for i in {1..9} #do fdedupe -f 2>> $HOME/.chronicle.log 1>&2 # || exit -53 # Error: 203 #done # there's no place like AUM: cd -P $HOME 2>> $HOME/.chronicle.log 1>&2 echo "Download Website DISK USAGE" 2>> $HOME/.chronicle.log 1>&2 # DOWNLOAD rsync -auvH $(my_twin):~/crystalfaeries.net/.??* \ $HOME/crystalfaeries.net/ \ 2>> $HOME/.chronicle.log 1>&2 sleep ${hostsleep} echo "UPLOAD WEBSITE TO BLUEHOST SERVER" 2>> $HOME/.chronicle.log 1>&2 # UPLOAD rsync -auvH $HOME/crystalfaeries.net/.??* \ $(my_twin):~/crystalfaeries.net/ \ 2>> $HOME/.chronicle.log 1>&2 sleep ${hostsleep} rsync -auvH \ --exclude=audio \ --exclude=downloads \ --exclude=fay \ --exclude=fey \ --exclude=fey.crystalfaeries.net \ --exclude=galleries \ --exclude=posix \ --exclude=src \ --exclude=video \ $HOME/crystalfaeries.net/* \ $(my_twin):~/crystalfaeries.net/ \ 2>> $HOME/.chronicle.log 1>&2 sleep ${hostsleep} echo "UPLOAD VIDEOS TO BLUEHOST SERVER" 2>> $HOME/.chronicle.log 1>&2 # UPLOAD rsync -auvH --delete \ --exclude=audio \ --exclude=fay \ --exclude=fey \ --exclude=fey.crystalfaeries.net \ --exclude=galleries \ --exclude=src \ --exclude=video/flute/flute_fundamentals_hd-wvwrzo2_r_q.mp4 \ --exclude=video/devideo \ --exclude=video/dr_selfridge \ --exclude=video/flute \ --exclude=video/nina_perlove \ --exclude=video/ogv \ --exclude=video/security_now \ $HOME/crystalfaeries.net/* \ $(my_twin):~/crystalfaeries.net/ \ 2>> $HOME/.chronicle.log 1>&2 sleep ${hostsleep} # FROM .crontab.fey: echo 'KKCR Subscriptions' 2>> $HOME/.chronicle.log 1>&2 subscriptions > $HOME/crystalfaeries.net/audio/kkcr/subscriptions.html 2>> $HOME/.chronicle.log echo 'CALCULATE AUDIO FILE REFERENCES' 2>> $HOME/.chronicle.log 1>&2 # update our view of website's use of audio directory: # find all webpages, EXCEPT the igal2 IMAGE INDEX PAGES, # (yes, that means we fail to see audio links there!) # which reference our own website for audio files: find crystalfaeries.net -name '*.html' \ \! -name '[0-9].html' \ \! -name '[0-9][0-9].html' \ \! -name '[0-9][0-9][0-9].html' \ \! -name '[0-9][0-9][0-9][0-9].html' \ -exec grep -H 'href="/audio/' {} \; 2>/dev/null \ | grep -v 'href="/audio/"' \ | grep -v '/fae/' \ | sort -u > crystalfaeries.net/audio/.crystalfaeries.hrefs.audio.txt # ALL hrefs including # hash tags (relative addressing within contents of targeted file): sed 's/^.*href="//;s/".*$//' < crystalfaeries.net/audio/.crystalfaeries.hrefs.audio.txt \ | grep '#' | sort -u > crystalfaeries.net/audio/.crystalfaeries.hashs.audio.txt # we SHOULD utilize our list of hashtag references to VERIFY name hrefs have valid targets and vice versa: # ALL hrefs cleansed of # hash tags (relative addressing within contents of targeted file): sed 's/^.*href="//;s/".*$//;s/#.*$//' < crystalfaeries.net/audio/.crystalfaeries.hrefs.audio.txt \ | sort -u > crystalfaeries.net/audio/.crystalfaeries.dehashed.audio.txt #rm crystalfaeries.net/audio/.crystalfaeries.hrefs.audio.txt # make directory tree into which we can rsync individual files: grep -v '\/$' crystalfaeries.net/audio/.crystalfaeries.dehashed.audio.txt \ > crystalfaeries.net/audio/.crystalfaeries.files.audio.txt sed 's/\/[^\/]*$/\//g' < crystalfaeries.net/audio/.crystalfaeries.files.audio.txt \ > crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt grep '\/index.html$' crystalfaeries.net/audio/.crystalfaeries.dehashed.audio.txt \ | sed 's/index.html$//g' >> crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt grep '\/$' crystalfaeries.net/audio/.crystalfaeries.dehashed.audio.txt \ >> crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt #rm crystalfaeries.net/audio/.crystalfaeries.dehashed.audio.txt sed 's/\/$//g' < crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt | sort -u \ > /tmp/tmp.$$.txt mv /tmp/tmp.$$.txt crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt echo 'BUILD AUDIO DIRectory TREE' 2>> $HOME/.chronicle.log 1>&2 rsync -auvH ~/crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt \ $(my_twin):~/crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt for d in $(cat ~/crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt) do ls "${d}/HEADER.html" 2>/dev/null >> crystalfaeries.net/audio/.crystalfaeries.files.audio.txt # files list ls "${d}/index.html" 2>/dev/null >> crystalfaeries.net/audio/.crystalfaeries.files.audio.txt # files list mkdir -p crystalfaeries.net/"${d}" # local dir done sleep ${hostsleep} # Host rate-limits ssh connections ssh -t $(my_twin) "for d in $(cat ~/crystalfaeries.net/audio/.crystalfaeries.dirs.audio.txt);do mkdir -p ~/crystalfaeries.net${d};done" sleep ${hostsleep} # Host rate-limits ssh connections sort -u > /tmp/tmp.$$.txt < crystalfaeries.net/audio/.crystalfaeries.files.audio.txt # remove duplicates mv /tmp/tmp.$$.txt crystalfaeries.net/audio/.crystalfaeries.files.audio.txt # upload just audio files referenced in website: echo 'UPLOAD REFERENCED AUDIO FILES TO SERVER' 2>> $HOME/.chronicle.log 1>&2 # NOT YET DELETING for f in $(cat crystalfaeries.net/audio/.crystalfaeries.files.audio.txt) do for removable in /media/celeste/* do echo "rsync -auvH ${removable}/${f} crystalfaeries.net/${f}" 2>> $HOME/.chronicle.log 1>&2 # file RESTORATION rsync -auvH ${removable}/"${f}" crystalfaeries.net/"${f}" 2>> $HOME/.chronicle.log 1>&2 # File RESTORATION done echo "rsync -auvH $HOME/crystalfaeries.net/${f} \ $(my_twin):~/crystalfaeries.net/${f}" 2>> $HOME/.chronicle.log 1>&2 rsync -auvH $HOME/crystalfaeries.net/"${f}" \ $(my_twin):~/crystalfaeries.net/"${f}" 2>> $HOME/.chronicle.log 1>&2 # UPLOAD to SERVER sleep ${hostsleep} # Host rate-limits ssh connections done # upload WITH DELETING calibre: echo 'UPLOAD CALIBRE' 2>> $HOME/.chronicle.log 1>&2 echo "rsync -auvH --delete $HOME/crystalfaeries.net/calibre $(my_twin):~/crystalfaeries.net/" 2>> $HOME/.chronicle.log 1>&2 rsync -auvH --delete $HOME/crystalfaeries.net/calibre $(my_twin):~/crystalfaeries.net/ 2>> $HOME/.chronicle.log 1>&2 # UPLOAD to SERVER sleep ${hostsleep} # Host rate-limits ssh connections echo 'CLEAR SWAP FILES and EMPTY FILES a week old' 2>> $HOME/.chronicle.log 1>&2 find crystalfaeries.net/ -iname '.*.swp' -mtime +7 -print -exec rm {} \; \ 2>> $HOME/.chronicle.log 1>&2 find crystalfaeries.net/ -type f -size 0 -mtime +7 -print -exec rm {} \; \ 2>> $HOME/.chronicle.log 1>&2 echo 'EXPUNGE DELETED files + TOUCH OLD files' 2>> $HOME/.chronicle.log 1>&2 find $HOME/crystalfaeries.net/ \( \ -type f \ -mtime +96 \ -size 0 \ -name '.??*' \ -exec rm {} \; \ \) -o \( \ -type f \ -mtime +96 \ \! -size 0 \ \! -name .directory \ \! -name .indextemplate2.html \ \! -name .name_tidy \ \! -name .slidetemplate2.html \ \! -name '.thumb_*' \ \! -name .tile.png \ -exec touch {} \; \ \) # pau for now touch $HOME/crystalfaeries.net/ 2>> $HOME/.chronicle.log 1>&2 # website update complete, yes? # RAWDOG update of just our own website: echo 'RAWDOG BEGIN' 2>> $HOME/.chronicle.log 1>&2 # now that what it sees is updated /usr/bin/rawdog -f https://crystalfaeries.net/fae/index.rss \ 2>> $HOME/.rawdog/log.txt 1>&2 if [ -s $HOME/.rawdog/log.txt ] then mail -s rawdog `/usr/bin/whoami` < $HOME/.rawdog/log.txt && \ cp /dev/null $HOME/.rawdog/log.txt # mail any errors to user fi 2>> $HOME/.chronicle.log & # in the background while we proceed... # now it's time to validate our links if [ ${links} -eq 0 ] then # User wants links validated echo "`now` LINK_LINT BEGIN" 2>> $HOME/.chronicle.log 1>&2 link_lint 2>> $HOME/.chronicle.log 1>&2 echo "`now` LINK_LINT END $?" 2>> $HOME/.chronicle.log 1>&2 echo "################################################################################################" \ 2>> $HOME/.chronicle.log 1>&2 fi # done updating blog echo "`now` BLOG END $?" 2>> $HOME/.chronicle.log 1>&2 limits 2>> $HOME/.chronicle.log 1>&2 exit $? # Pau