diff --git a/etc/gsl/gsl.conf b/etc/gsl/gsl.conf index 049ed11..3e7f9e7 100644 --- a/etc/gsl/gsl.conf +++ b/etc/gsl/gsl.conf @@ -32,6 +32,7 @@ gsl_file_logs="$gsl_dir_logs/gsl.log" gsl_file_db_domains="$gsl_dir_db/domains.db" gsl_file_db_posts="$gsl_dir_db/posts.db" gsl_file_db_stats="$gsl_dir_db/stats.db" +gsl_file_db_files="$gsl_dir_db/files.db" # Registred Authors filename gsl_filename_auth="authors.db" diff --git a/usr/local/bin/gsl b/usr/local/bin/gsl index 584f58c..237a8a8 100755 --- a/usr/local/bin/gsl +++ b/usr/local/bin/gsl @@ -66,7 +66,7 @@ case "$1" in ;; esac ;; - + new|-N) gsl__check_source "$gsl_dir_scripts/gsl__new_website" || exit 1 gsl__logs_print \ @@ -75,9 +75,9 @@ case "$1" in "Process" \ "New Website" \ "$gsl_dir_scripts/gsl__new_website" - + gsl__new_website - + gsl__logs_print \ "$gsl_log_i" \ "Stopping" \ @@ -90,7 +90,7 @@ case "$1" in && cd "$gsl_dir_user_posts" \ && gsl author add ;; - + log|-L) [[ -z `grep '[^[:space:]]' "$gsl_file_logs" 2>/dev/null` ]] \ && echo "# No logs to show. File is empty." \ @@ -121,7 +121,7 @@ case "$1" in gsl__logs_show ;; - + check|-C) gsl__find_domain # || exit 1 gsl__check_source "$gsl_dir_scripts/gsl__auth_manager" || exit 1 @@ -140,23 +140,16 @@ case "$1" in [[ "$2" ]] && gsl__check_opt_post "$2" ;; esac - + gsl__logs_print \ "$gsl_log_i" \ "Starting" \ "Process" \ "Check Post for domain $gsl_find_domain" \ "${PWD}" - - gsl__logs_print \ - "$gsl_log_i" \ - "Found" \ - "Posts" \ - "$gsl_nbr_posts" \ - "${PWD}" - + gsl__loop_posts check - + gsl__logs_print \ "$gsl_log_i" \ "Stopping" \ @@ -164,7 +157,31 @@ case "$1" in "Check Post for domain $gsl_find_domain" \ "${PWD}" ;; - + + make|-M) + gsl__find_domain # || exit 1 + gsl__check_source "$gsl_dir_scripts/gsl__post_makers" || exit 1 + gsl__check_source "$gsl_dir_scripts/gsl__post_manager" || exit 1 + gsl__check_nbr_posts + gsl__check_source "$gsl_dir_scripts/gsl__db_manager" || exit 1 + + gsl__logs_print \ + "$gsl_log_i" \ + "Starting" \ + "Process" \ + "Make Post for domain $gsl_find_domain" \ + "${PWD}" + + gsl__loop_posts make + + gsl__logs_print \ + "$gsl_log_i" \ + "Stopping" \ + "Process" \ + "Make Post for domain $gsl_find_domain" \ + "${PWD}" + ;; + readme) if [[ -f "/var/lib/gsl/README.md" ]];then clear @@ -173,10 +190,10 @@ case "$1" in echo "! Missing file: /var/lib/gsl/README.md" fi ;; - + help|--help|-h) gsl__check_file "$gsl_dir_helps/gsl_help" || exit 1 - + clear case "$2" in "") diff --git a/var/lib/gsl/scripts/gsl__db_manager b/var/lib/gsl/scripts/gsl__db_manager index 7947fc4..445dbdf 100644 --- a/var/lib/gsl/scripts/gsl__db_manager +++ b/var/lib/gsl/scripts/gsl__db_manager @@ -15,23 +15,50 @@ grep "$gsl_post" \ } #======================================================================= -# Check Post exists and get line NBR in DB | $1 DB file +# From checkers (bcodes,images,files): Write DB #======================================================================= -gsl__db_post_exists() { +gsl__db_line_file() { +gsl__db_exists "$gsl_file_db_files" "$gsl_post_header_field_2" +gsl_db_line_file="$gsl_post_header_field_2:$gsl_file_csum" +if [[ "$gsl_db_file_hash" ]];then + sed -i "${gsl_db_file_line_nbr}s,.*,$gsl_db_line_file," \ + "$gsl_file_db_files" +else + echo "$gsl_post_header_field_2:$gsl_file_csum" \ + >> "$gsl_file_db_files" +fi +} +#======================================================================= +# Check Post exists and get line NBR in DB | $1 DB file $2 TERM +#======================================================================= +gsl__db_exists() { case "$1" in "$gsl_file_db_posts") gsl_db_post_exists=` grep -n "$gsl_post" "$1"` - gsl_db_post_line_nbr=` - awk -F":" '{print $1}' <<< "$gsl_db_post_exists"` - gsl_db_post_status=` - awk -F"|" '{print $4}' <<< "$gsl_db_post_exists"` + if [[ "$gsl_db_post_exists" ]];then + gsl_db_post_line_nbr=` + awk -F":" '{print $1}' <<< "$gsl_db_post_exists"` + gsl_db_post_status=` + awk -F"|" '{print $5}' <<< "$gsl_db_post_exists"` + fi ;; "$gsl_file_db_stats") gsl_db_stat_exists=` grep -n "$gsl_post" "$1"` - gsl_db_stat_line_nbr=` - awk -F":" '{print $1}' <<< "$gsl_db_stat_exists"` + if [[ "$gsl_db_stat_exists" ]];then + gsl_db_stat_line_nbr=` + awk -F":" '{print $1}' <<< "$gsl_db_stat_exists"` + fi + ;; + "$gsl_file_db_files") + gsl_db_file_exists=`grep -n "$2" "$1"` + if [[ "$gsl_db_file_exists" ]];then + gsl_db_file_line_nbr=` + awk -F: '{print $1}' <<< "$gsl_db_file_exists"` + gsl_db_file_hash=` + awk -F: '{print $3}' <<< "$gsl_db_file_exists"` + fi ;; esac } @@ -45,23 +72,23 @@ case "$gsl_process" in #------------------------------------------------------------------- # Check if post exists in DB stats.db and get line #------------------------------------------------------------------- - if ! [[ "$gsl_checker_err" ]];then - gsl_db_stat_line=` - printf '%s%s%s%s%s%s%s%s%s%s\n' \ - "$gsl_post|" \ - "$gsl_post_stat_w|" \ - "$gsl_post_stat_p|" \ - "$gsl_post_stat_abbr|" \ - "$gsl_post_stat_link|" \ - "$gsl_post_stat_image|" \ - "$gsl_post_stat_file|" \ - "$gsl_post_stat_bq|" \ - "$gsl_post_stat_bcode|" \ - "${gsl_post_stat_mdb// /|}|"` + gsl_db_stat_line=` + printf '%s%s%s%s%s%s%s%s%s%s\n' \ + "$gsl_post|" \ + "$gsl_post_stat_w|" \ + "$gsl_post_stat_p|" \ + "$gsl_post_stat_abbr|" \ + "$gsl_post_stat_link|" \ + "$gsl_post_stat_image|" \ + "$gsl_post_stat_file|" \ + "$gsl_post_stat_bq|" \ + "$gsl_post_stat_bcode|" \ + "${gsl_post_stat_mdb// /|}|"` - gsl__db_post_exists "$gsl_file_db_stats" + if ! [[ "$gsl_checker_err" ]];then + gsl__db_exists "$gsl_file_db_stats" if [[ "$gsl_db_stat_line_nbr" ]];then - sed -i "${gsl_db_stat_line_nbr}s/.*/$gsl_db_stat_line/" \ + sed -i "${gsl_db_stat_line_nbr}s,.*,$gsl_db_stat_line," \ "$gsl_file_db_stats" && \ gsl__logs_print \ "$gsl_log_w" \ @@ -83,28 +110,32 @@ case "$gsl_process" in #------------------------------------------------------------------- # Check if post exists in DB posts.db and get line / status #------------------------------------------------------------------- - gsl__db_post_exists "$gsl_file_db_posts" - [[ "$gsl_checker_err" ]] && \ - [[ "$gsl_db_post_line_nbr" ]] \ - && sed -i "${gsl_db_post_line_nbr}d" "$gsl_file_db_posts" \ - && gsl__logs_print \ + gsl__db_exists "$gsl_file_db_posts" + if [[ "$gsl_checker_err" ]] && [[ "$gsl_db_post_line_nbr" ]];then + sed -i "${gsl_db_post_line_nbr}d" "$gsl_file_db_posts" \ + gsl__logs_print \ "$gsl_log_w" \ "DB" \ "Post" \ "Line:$gsl_db_post_line_nbr removed. Error in $gsl_post" \ - "$gsl_file_db_posts" \ - && return + "$gsl_file_db_posts" + return + fi + + echo -e "\nD> Pass next DB" gsl_db_post_line=` - printf '%s%s%s%s%s\n' \ + printf '%s%s%s%s%s%s%s\n' \ "$gsl_post_type|" \ "$gsl_post|" \ "$gsl_post_hash|" \ "$gsl_post_size|" \ - "$gsl_process|"` + "$gsl_process|" \ + "${PWD}/$gsl_post|"\ + "$gsl_db_post_files"` if [[ "$gsl_db_post_exists" ]];then - sed -i "${gsl_db_post_line_nbr}s/.*/$gsl_db_post_line/" \ + sed -i "${gsl_db_post_line_nbr}s,.*,$gsl_db_post_line," \ "$gsl_file_db_posts" && \ gsl__logs_print \ "$gsl_log_w" \ diff --git a/var/lib/gsl/scripts/gsl__post_checkers b/var/lib/gsl/scripts/gsl__post_checkers index cb67a19..69be69a 100644 --- a/var/lib/gsl/scripts/gsl__post_checkers +++ b/var/lib/gsl/scripts/gsl__post_checkers @@ -10,7 +10,6 @@ gsl__post_all_checkers() { clear unset gsl_check_done -gsl_process="Checked" echo -ne "Searching for #1..." gsl__post_check_h1 || return @@ -41,12 +40,6 @@ echo -ne "\r\033[2K" # End of checkers : show logs for war and err gsl__db_line_post - -[[ "$gsl_checker_war" ]] \ - && gsl log -s -w - -[[ "$gsl_checker_err" ]] \ - && gsl log -s -e } #======================================================================= @@ -445,6 +438,13 @@ do "Not found: $gsl_post_header_field_2" \ "$gsl_dir_domain_files/$gsl_post_header_field_2" gsl_checker_err=true + else + # Register file to DB with hash + gsl_file_csum=` + cksum "$gsl_dir_domain_files/$gsl_post_header_field_2" \ + | awk '{print $1}'` + gsl_db_post_files+="$gsl_post_header_field_2|" + gsl__db_line_file fi # Find if ref is in CONTENT @@ -485,6 +485,13 @@ do "Not found: $gsl_post_header_field_2" \ "$gsl_dir_domain_files/$gsl_post_header_field_2" gsl_checker_err=true + else + # Register file to DB with hash + gsl_file_csum=` + cksum "$gsl_dir_domain_files/$gsl_post_header_field_2" \ + | awk '{print $1}'` + gsl_db_post_files+="$gsl_post_header_field_2|" + gsl__db_line_file fi # Find if ref is in CONTENT @@ -530,6 +537,15 @@ do "Not found: $gsl_post_header_field_2" \ "$gsl_dir_domain_files/$gsl_post_header_field_2" gsl_checker_err=true + else + # Register file to DB with hash + gsl_file_csum=` + cksum "$gsl_dir_domain_files/$gsl_post_header_field_2" \ + | awk '{print $1}'` + gsl_db_post_files+="$gsl_post_header_field_2|" + + gsl__db_exists "$gsl_file_db_files" "$gsl_post_header_field_2" + gsl__db_line_file fi # Find if ref is in CONTENT diff --git a/var/lib/gsl/scripts/gsl__post_makers b/var/lib/gsl/scripts/gsl__post_makers index 37b8d90..92bef3d 100644 --- a/var/lib/gsl/scripts/gsl__post_makers +++ b/var/lib/gsl/scripts/gsl__post_makers @@ -3,3 +3,256 @@ # Folder: /var/lib/gsl/scripts # By echolib # License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 + +#======================================================================= +# --------- +# MAIN CALL +# --------- +#======================================================================= + +#======================================================================= +# From: gsl__loop_posts | Convert Post from modules to HTML +#======================================================================= +gsl__post_all_makers() { +clear +unset gsl_check_done +gsl__prepare_makers + +echo "# Great, let's begin with Block Code..." +gsl__check_bcodes +gsl__check_cites + +cat "$gsl_uri_tmp_post" +} + +#======================================================================= +# --------------------------------------------- +# Tools and prepapre tmp file from CONTENT Post +# --------------------------------------------- +#======================================================================= + +#======================================================================= +# Get line nbr from TMP CONTENT Post... $1: TERM | $2: File +#======================================================================= +gsl__get_mark_content_line() { +awk -v s="$1" \ + '$0 ~ s {print NR}' \ + "$2" || return +} + +#======================================================================= +# Get NEEDED infos, and create TMP file from CONTENT +#======================================================================= +gsl__prepare_makers() { +# Getting NEEDED infos +gsl_post_begin=` +grep -n "#1" $gsl_post \ + | head -1 \ + | awk -F: '{print $1}'` + +source "$gsl_this_conf_domain/$gsl_find_domain.conf" + +# Create TMP file from CONTENT +gsl_dir_tmp="/tmp/gsl" +gsl_file_tmp_post_content="post_content" +gsl_uri_tmp_post="$gsl_dir_tmp/$gsl_file_tmp_post_content" + +mkdir -p "$gsl_dir_tmp" + +echo "" > "$gsl_uri_tmp_post" +awk -v l="$gsl_post_begin" \ + 'NR >= l' \ + "$gsl_post" \ + > "$gsl_uri_tmp_post" +} + +#======================================================================= +# --------------------------------- +# Check if Convert is needed and do +# --------------------------------- +#======================================================================= + +#======================================================================= +# Find BLOCK-CODES +#======================================================================= +gsl__check_bcodes() { +gsl_lines_nbr=(` +gsl__get_mark_content_line \ + "$gsl_mark_code" \ + "$gsl_uri_tmp_post"`) + +! [[ "$gsl_lines_nbr" ]] \ + && return + +for i in `seq 1 ${#gsl_lines_nbr[@]}` +do + gsl__post_maker_bcode +done +} + +#======================================================================= +# Find BLOCKQUOTES +#======================================================================= +gsl__check_cites() { +gsl_lines_nbr=(` +gsl__get_mark_content_line \ + "$gsl_mark_blockquote" \ + "$gsl_uri_tmp_post"`) + +! [[ "$gsl_lines_nbr" ]] \ + && return + +echo "BQ: ${gsl_lines_nbr[@]}" + +for i in `seq 1 ${#gsl_lines_nbr[@]}` +do + if [[ "$gsl_cite_start" ]];then + unset gsl_cite_start + echo "> BQ C:${gsl_lines_nbr[$((i-1))]}" + gsl_cite_line_stop=${gsl_lines_nbr[(($i-1))]} + gsl__post_maker_cite + else + gsl_cite_start=true + echo "> BQ O:${gsl_lines_nbr[(($i-1))]}" + gsl_cite_line_start=${gsl_lines_nbr[(($i-1))]} + fi +done +} + +#======================================================================= +# ----------------------------------------------- +# Convert To HTML in duplicated CONTENT Post File +# ----------------------------------------------- +#======================================================================= + +#======================================================================= +# Replace gsl_mark_code +#======================================================================= +gsl__post_maker_bcode() { + +# Get mark line nbr +gsl_line_nbr=` +gsl__get_mark_content_line \ + "$gsl_mark_code" \ + "$gsl_uri_tmp_post" | head -1` + +#--------- +# Convert +#--------- +gsl_css_class="${gsl_site_css}_block-code" +gsl_header_content_line=`gsl__get_header "$gsl_marker_code" "$gsl_post"` +gsl_file_tmp_code=`mktemp` + +# Set gsl_post_header_field_1 gsl_post_header_field_2 +gsl__get_header_fields "$gsl_marker_code" + +printf '%s %s\n%b\n%s\n' \ + "
" \
+       "$(cat -n "$gsl_dir_domain_files/$gsl_post_header_field_2")" \
+       "
" \ + > "$gsl_file_tmp_code" + +sed -i "${gsl_line_nbr}r $gsl_file_tmp_code" "$gsl_uri_tmp_post" +sed -i "${gsl_line_nbr}d" "$gsl_uri_tmp_post" +rm -f "$gsl_file_tmp_code" +} + +#======================================================================= +# Convert & replace this blockquote +#======================================================================= +gsl__post_maker_cite() { +echo "Block: $gsl_cite_line_start > $gsl_cite_line_stop" + +# Put Blockquote in a file and get DATAS +gsl_file_tmp_cite=`mktemp` +# Put Blockquote CONTENT only in this file +gsl_file_tmp_cite_content=`mktemp` +# Put HTML blockquote in this file +gsl_file_tmp_cite_html=`mktemp` + +awk -v o="$gsl_cite_line_start" -v c="$gsl_cite_line_stop" \ + 'NR >= o && NR <= c' \ + "$gsl_uri_tmp_post" \ + > "$gsl_file_tmp_cite" + +while read -r "gsl_content_line" +do + case "$gsl_content_line" in + ---*) + continue + ;; + _cite*) + gsl_bq_author=`awk -F" : " '{print $2}' <<< "$gsl_content_line"` + ;; + _book*) + gsl_bq_book=`awk -F" : " '{print $2}' <<< "$gsl_content_line"` + ;; + _lang*) + gsl_bq_lang=`awk -F" : " '{print $2}' <<< "$gsl_content_line"` + ;; + _year*) + gsl_bq_year=`awk -F" : " '{print $2}' <<< "$gsl_content_line"` + ;; + _link*) + gsl_bq_link=`awk -F" : " '{print $2}' <<< "$gsl_content_line"` + ;; + *) + echo " $gsl_content_line" >> "$gsl_file_tmp_cite_content" + ;; + esac +done < <(cat "$gsl_file_tmp_cite") +#echo "D> Auteur: $gsl_bq_author" +#echo "D> Book : $gsl_bq_book" +#echo "D> lang : $gsl_bq_lang" +#echo "D> year : $gsl_bq_year" +#echo "D> link : $gsl_bq_link" + +rm -f "$gsl_file_tmp_cite" + +# Convert DATAS and put HTML code in a file +gsl_file_tmp_cite=`mktemp` +if [[ "$gsl_bq_author" ]];then + gsl_css_class="class=\"${gsl_site_css}_quote\"" + +# In a function... +[[ "$gsl_bq_lang" ]] \ + && gsl_html_cite_lang=" lang=\"$gsl_bq_lang\"" + +[[ "$gsl_bq_link" ]] \ + && gsl_html_cite_link=" cite=\"$gsl_bq_link\"" + +cat << EOCITE > "$gsl_file_tmp_cite_html" +
+
+$(cat "$gsl_file_tmp_cite_content") +
+
+ + $gsl_bq_author + +
+
+EOCITE + +# Insert blockquote file +sed -i "${gsl_cite_line_stop}r $gsl_file_tmp_cite_html" "$gsl_uri_tmp_post" +# Remove useless lines +gsl_cite_total_lines=$(( $gsl_cite_line_stop - gsl_cite_line_start )) +for i in `seq 0 $gsl_cite_total_lines` +do + sed -i "${gsl_cite_line_start}d" "$gsl_uri_tmp_post" +done + +# No more BLOCKQUOTE CONTENT needed +rm -f "$gsl_file_tmp_cite_content" + +# No more html blockquote file needed +rm -f "$gsl_file_tmp_cite_html" +else + gsl_css_class="${gsl_site_css}_simple-quote" +fi + +# No more DATAS needed for next if any +unset ${!gsl_bq_@} +} diff --git a/var/lib/gsl/scripts/gsl__post_manager b/var/lib/gsl/scripts/gsl__post_manager index d41b1ba..7d657f2 100644 --- a/var/lib/gsl/scripts/gsl__post_manager +++ b/var/lib/gsl/scripts/gsl__post_manager @@ -27,6 +27,13 @@ gsl_nbr_posts=`ls -1 *.gsl 2>/dev/null | wc -l` # Read from Post list #======================================================================= gsl__loop_posts() { +gsl__logs_print \ + "$gsl_log_i" \ + "Found" \ + "Posts" \ + "$gsl_nbr_posts" \ + "${PWD}" + for gsl_post in `ls -1 *.gsl 2>/dev/null` do # Check specific asked Post @@ -64,10 +71,11 @@ do make) gsl_process="Made" - gsl__db_post_exists "$gsl_file_db_posts" + gsl__db_exists "$gsl_file_db_posts" case "$gsl_db_post_status" in Checked) - true # gsl__post_makers + gsl__post_compare_hash || continue + gsl__post_all_makers ;; Made) @@ -84,6 +92,12 @@ do esac done +[[ "$gsl_checker_war" ]] \ + && gsl log -s -w + +[[ "$gsl_checker_err" ]] \ + && gsl log -s -e + [[ "$gsl_check_done" ]] \ && echo "# Check already done" } @@ -102,16 +116,62 @@ gsl_post_size=`awk '{print $2}' <<< "$gsl_post_csum"` #======================================================================= gsl__post_compare_hash() { gsl__db_post_hash -if [[ "$gsl_db_post_hash" == "$gsl_post_hash" ]];then - gsl__logs_print \ - "$gsl_log_i" \ - "Post" \ - "Hash" \ - "$gsl_post already Checked from $gsl_post_hash" \ - "$gsl_file_db_posts" - gsl_check_done=true - return 1 -fi + +case "$gsl_process" in + Checked) + if [[ "$gsl_db_post_hash" == "$gsl_post_hash" ]];then + gsl__logs_print \ + "$gsl_log_i" \ + "Post" \ + "Hash" \ + "$gsl_post already Checked from $gsl_post_hash" \ + "$gsl_file_db_posts" + gsl_check_done=true + return 1 + fi + ;; + + Made) + if ! [[ "$gsl_db_post_hash" == "$gsl_post_hash" ]];then + gsl__logs_print \ + "$gsl_log_e" \ + "Post" \ + "Hash" \ + "$gsl_post has changed. Check it again" \ + "$gsl_file_db_posts" + unset gsl_check_done + gsl_checker_err=true + return 1 + fi + + # Compare external content hash + while read "gsl_content_filename" + do + #unset gsl_db_file_hash gsl_file_hash + ! [[ "$gsl_content_filename" ]] && continue + gsl_db_file_hash=` + grep "$gsl_content_filename" "$gsl_file_db_files" \ + | awk -F: '{print $2}'` + gsl_file_hash=` + cksum "$gsl_dir_domain_files/$gsl_content_filename" \ + | awk '{print $1}'` + + if ! [[ "$gsl_db_file_hash" == "$gsl_file_hash" ]];then + gsl__logs_print \ + "$gsl_log_e" \ + "File" \ + "Hash" \ + "$gsl_content_filename has changed. $gsl_db_file_hash $gsl_file_hash" \ + "$gsl_file_db_files" + unset gsl_check_done + gsl_checker_err=true + return 1 + fi + done < <(grep "$gsl_post" "$gsl_file_db_posts" \ + | awk -F"|" \ + '{for(i=7;i<=NF;i++){print $i}}') + ;; +esac } #----------------------------------------------------------------------- @@ -135,8 +195,6 @@ gsl__get_header() { awk -F"$1" -v marker="$1" -v l="$gsl_post_begin" \ 'NR < l && $0 ~ marker {print $2}' \ "$2" 2>/dev/null - -# '{if ($0 ~ marker) print $2}' \ } #======================================================================= diff --git a/var/lib/gsl/scripts/gsl__tools b/var/lib/gsl/scripts/gsl__tools index cb4fabe..f40da9e 100644 --- a/var/lib/gsl/scripts/gsl__tools +++ b/var/lib/gsl/scripts/gsl__tools @@ -44,6 +44,7 @@ done for gsl_file in "$gsl_file_logs" \ "$gsl_file_db_posts" \ "$gsl_file_db_stats" \ + "$gsl_file_db_files" \ "$gsl_file_db_domains" do gsl__check_file "$gsl_file" && touch "$gsl_file"