From f9bdad2da2017d762f9f965cdf025e18545f68a8 Mon Sep 17 00:00:00 2001 From: Cyrille L Date: Tue, 1 Mar 2022 18:39:46 +0100 Subject: [PATCH] Updated: New DB System --- var/lib/gsl/scripts/gsl__db_manager | 233 +++++++++---------------- var/lib/gsl/scripts/gsl__do_commons | 8 +- var/lib/gsl/scripts/gsl__page_creator | 3 +- var/lib/gsl/scripts/gsl__post_checkers | 57 +++--- var/lib/gsl/scripts/gsl__post_manager | 79 ++------- var/lib/gsl/scripts/gsl__tools | 15 +- 6 files changed, 144 insertions(+), 251 deletions(-) diff --git a/var/lib/gsl/scripts/gsl__db_manager b/var/lib/gsl/scripts/gsl__db_manager index 8c69f08..e9f0fab 100644 --- a/var/lib/gsl/scripts/gsl__db_manager +++ b/var/lib/gsl/scripts/gsl__db_manager @@ -4,160 +4,101 @@ # By echolib # License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 -#======================================================================= -# Get Post Hash FROM DB posts.db -#======================================================================= -gsl__db_post_hash() { -gsl_db_post_hash=` -grep "$gsl_post" \ - "$gsl_file_db_posts" \ - | awk -F"|" '{print $3}'` -} -#======================================================================= -# From checkers (bcodes,images,files): Write DB -#======================================================================= -gsl__db_line_file() { -gsl__db_exists "$gsl_file_db_files" "$gsl_post_hf_2" -gsl_db_line_file="$gsl_post_hf_2:$gsl_file_csum" - -if [[ "$gsl_db_file_hash" ]];then - - sed -i "${gsl_db_file_line_nbr}s,.*,$gsl_db_line_file," \ - "$gsl_file_db_files" - -else - - echo "$gsl_post_hf_2:$gsl_file_csum" \ - >> "$gsl_file_db_files" +#====================================================================== +# Get Post Hash & Status From Database +#====================================================================== +gsl__db_get_post_datas() { +gsl_db_post_hash=0 +if [[ -f "$gsl_file_db_post" ]];then + + gsl_db_post_hash=` + awk -F: 'NR == 1 {print $3}' \ + "$gsl_file_db_post"` + + gsl_db_post_status=` + awk -F: 'NR == 2 {print $2}' \ + "$gsl_file_db_post"` fi } -#======================================================================= -# Check Post exists and get line NBR in DB | $1 DB file $2 TERM -#======================================================================= -gsl__db_exists() { -case "$1" in - "$gsl_file_db_posts") - gsl_db_post_exists=` - grep -n "$gsl_post" "$1"` - if [[ "$gsl_db_post_exists" ]];then - gsl_db_post_line_nbr=` - awk -F":" '{print $1}' <<< "$gsl_db_post_exists"` - gsl_db_post_status=` - awk -F"|" '{print $5}' <<< "$gsl_db_post_exists"` - fi - ;; - "$gsl_file_db_stats") - gsl_db_stat_exists=` - grep -n "$gsl_post" "$1"` - if [[ "$gsl_db_stat_exists" ]];then - gsl_db_stat_line_nbr=` - awk -F":" '{print $1}' <<< "$gsl_db_stat_exists"` - fi - ;; - "$gsl_file_db_files") - gsl_db_file_exists=`grep -n "$2" "$1"` - if [[ "$gsl_db_file_exists" ]];then - gsl_db_file_line_nbr=` - awk -F: '{print $1}' <<< "$gsl_db_file_exists"` - gsl_db_file_hash=` - awk -F: '{print $3}' <<< "$gsl_db_file_exists"` - fi - ;; -esac + +#====================================================================== +# Get Post Hash and Compare from Database +#====================================================================== +gsl__db_compare_post_hash() { +! (( $gsl_post_hash == $gsl_db_post_hash )) \ + && gsl__logs_print \ + "$gsl_log_w" \ + "Post" \ + "Hash" \ + "$gsl_post Changed ($gsl_post_hash)" \ + "${PWD}/$gsl_post" \ + && gsl_checker_war=true \ + && return } -#======================================================================= -# From gsl__loop_posts: Write DB -#======================================================================= -gsl__db_line_post() { +#====================================================================== +# Set Post Status in Database according to proceess +#====================================================================== +gsl__db_set_post_status() { case "$gsl_process" in Checked) - #------------------------------------------------------------------- - # Check if post exists in DB stats.db and get line - #------------------------------------------------------------------- - gsl_db_stat_line=` - printf '%s%s%s%s%s%s%s%s%s%s\n' \ - "$gsl_post|" \ - "$gsl_post_stat_w|" \ - "$gsl_post_stat_p|" \ - "$gsl_post_stat_abbr|" \ - "$gsl_post_stat_link|" \ - "$gsl_post_stat_image|" \ - "$gsl_post_stat_file|" \ - "$gsl_post_stat_bq|" \ - "$gsl_post_stat_bcode|" \ - "${gsl_post_stat_mdb// /|}|"` - - if ! [[ "$gsl_checker_err" ]];then - gsl__db_exists "$gsl_file_db_stats" - if [[ "$gsl_db_stat_line_nbr" ]];then - sed -i "${gsl_db_stat_line_nbr}s,.*,$gsl_db_stat_line," \ - "$gsl_file_db_stats" && \ - gsl__logs_print \ - "$gsl_log_w" \ - "DB" \ - "Stat" \ - "Line:$gsl_db_stat_line_nbr $gsl_post" \ - "$gsl_file_db_stats" - else - echo "$gsl_db_stat_line" >> "$gsl_file_db_stats" && \ - gsl__logs_print \ - "$gsl_log_w" \ - "DB" \ - "Stat" \ - "Line:NEW $gsl_db_stat_line" \ - "$gsl_file_db_stats" - fi - fi - - #------------------------------------------------------------------- - # Check if post exists in DB posts.db and get line / status - #------------------------------------------------------------------- - gsl__db_exists "$gsl_file_db_posts" - if [[ "$gsl_checker_err" ]] && [[ "$gsl_db_post_line_nbr" ]];then - sed -i "${gsl_db_post_line_nbr}d" "$gsl_file_db_posts" && \ - gsl__logs_print \ - "$gsl_log_w" \ - "DB" \ - "Post" \ - "Line:$gsl_db_post_line_nbr removed. Error in $gsl_post" \ - "$gsl_file_db_posts" - return - fi - - gsl_db_post_line=` - printf '%s%s%s%s%s%s%s\n' \ - "$gsl_post_type|" \ - "$gsl_post|" \ - "$gsl_post_hash|" \ - "$gsl_post_size|" \ - "$gsl_process|" \ - "${PWD}/$gsl_post|"\ - "$gsl_db_post_files"` - - if [[ "$gsl_db_post_exists" ]];then - sed -i "${gsl_db_post_line_nbr}s,.*,$gsl_db_post_line," \ - "$gsl_file_db_posts" && \ - gsl__logs_print \ - "$gsl_log_w" \ - "DB" \ - "Post" \ - "Line:REPLACED - $gsl_post - Hash: $gsl_post_hash" \ - "$gsl_file_db_posts" - gsl_checker_war=true - - else - echo "$gsl_db_post_line" \ - >> "$gsl_file_db_posts" && \ - gsl__logs_print \ - "$gsl_log_i" \ - "DB" \ - "Post" \ - "Line:NEW - $gsl_post - Hash: $gsl_post_hash" \ - "$gsl_file_db_posts" - fi + case "$gsl_db_post_status" in + Checked) + gsl__db_compare_post_hash || return + gsl__db_create_post + ;; + Made) + return + ;; + Online) + return + ;; + *) + gsl__db_create_post + ;; + esac ;; esac } + +#====================================================================== +# Create DB File from Data Post +#====================================================================== +gsl__db_create_post() { +touch "$gsl_file_db_post" + +# Main Print +printf '%s\n%s\n%s\n' \ + "Post:${PWD}/$gsl_post:$gsl_post_hash:$gsl_post_size" \ + "Status:$gsl_process" \ + "`cat "$gsl_db_tmp"`" \ + > "$gsl_file_db_post" + +# Stats Print +printf '\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \ + "Stats" \ + "Author:$gsl_header_author" \ + "Date:$gsl_header_date" \ + "Words:$gsl_stat_words" \ + "Parag:$gsl_stat_p" \ + "Links:$gsl_stat_link:$gsl_stat_links" \ + "Strongs:$gsl_stat_strongs" \ + "Bolds:$gsl_stat_bolds"\ + "Italics:$gsl_stat_italics" \ + "Abbrs:$gsl_stat_abbr:$gsl_stat_abbrs" \ + "Images:$gsl_stat_image" \ + "Files:$gsl_stat_file:$gsl_count_files" \ + "Cites:$gsl_stat_bq" \ + "ICodes:$gsl_stat_icode" \ + "FCodes:$gsl_stat_fcode" \ + >> "$gsl_file_db_post" + +gsl__logs_print \ + "$gsl_log_w" \ + "DB" \ + "Create" \ + "Datas from $gsl_post" \ + "$gsl_file_db_post" +} diff --git a/var/lib/gsl/scripts/gsl__do_commons b/var/lib/gsl/scripts/gsl__do_commons index ebdf22d..ae33083 100644 --- a/var/lib/gsl/scripts/gsl__do_commons +++ b/var/lib/gsl/scripts/gsl__do_commons @@ -23,12 +23,6 @@ gsl_header_tags=`gsl__get_header "$gsl_marker_tags" "$gsl_post"` gsl__do_header() { unset ${!gsl_post_hf@} -# Stats -gsl_stat_link=0 -gsl_stat_abbr=0 -gsl_stat_file=0 -gsl_stat_fcode=0 - while read -r "gsl_header_content_line" do gsl__get_header_fields "$2" @@ -179,7 +173,7 @@ do case "$1" in "stats") - ((gsl_stat_icodes++)) + ((gsl_stat_icode++)) ;; "make") echo -ne "\r\033[2K: Convrting Inline-Code... $icode" diff --git a/var/lib/gsl/scripts/gsl__page_creator b/var/lib/gsl/scripts/gsl__page_creator index 16976ef..9863b42 100644 --- a/var/lib/gsl/scripts/gsl__page_creator +++ b/var/lib/gsl/scripts/gsl__page_creator @@ -18,6 +18,7 @@ gsl__page_prepare_datas gsl__check_templates gsl__html_meta_head gsl__page_create +cat -n "$gsl_srv_wip_page" } #----------------------------------------------------------------------- @@ -58,7 +59,7 @@ gsl_uri_site_logo="$gsl_uri/templates/$gsl_site_logo" #----------------------------------------------------------------------- gsl__check_srv_files() { if [[ -f "$1" ]];then - echo "! $1" + echo "! Replaced Page: $1" fi } diff --git a/var/lib/gsl/scripts/gsl__post_checkers b/var/lib/gsl/scripts/gsl__post_checkers index bdcb446..516a3dc 100644 --- a/var/lib/gsl/scripts/gsl__post_checkers +++ b/var/lib/gsl/scripts/gsl__post_checkers @@ -10,9 +10,17 @@ # -------------------------------------------------- #---------------------------------------------------------------------- gsl__all_checkers() { -clear unset gsl_check_done +# Stats +gsl_db_tmp=`mktemp` +gsl_stat_link=0 +gsl_stat_abbr=0 +gsl_stat_file=0 +gsl_stat_image=0 +gsl_stat_icode=0 +gsl_stat_fcode=0 + gsl__check_h1 || return gsl__post_content_only @@ -36,8 +44,9 @@ rm -f "$gsl_tmp_post" echo -ne "\r\033[2K" -# End of checkers : show logs for war and err -gsl__db_line_post +gsl__db_set_post_status +rm -f "$gsl_db_tmp" + } #---------------------------------------------------------------------- @@ -194,20 +203,20 @@ gsl__get_content_line "_$gsl_post_hf_1" "$gsl_post" | wc -l if (( "$gsl_count_links" > 0 ));then # Check URL - gsl_url_online=` + gsl_url_status=` curl -o /dev/null --silent --head --write-out \ '%{http_code}' \ "$gsl_post_hf_2" 2>/dev/null` - if (( $gsl_url_online == 200 )) || \ - (( $gsl_url_online == 301 )) || \ - (( $gsl_url_online == 302 ));then + if (( $gsl_url_status == 200 )) || \ + (( $gsl_url_status == 301 )) || \ + (( $gsl_url_status == 302 ));then gsl__logs_print \ "$gsl_log_i" \ "Post" \ "$gsl_log_h_link" \ - "Online:$gsl_url_online ${gsl_post_hf_2:0:40}..." \ + "Online:$gsl_url_status ${gsl_post_hf_2:0:40}..." \ "${PWD}/$gsl_post" else @@ -216,7 +225,7 @@ if (( "$gsl_count_links" > 0 ));then "$gsl_log_e" \ "Post" \ "$gsl_log_h_link" \ - "Offline:$gsl_url_online ${gsl_post_hf_2:0:40}..." \ + "Offline:$gsl_url_status ${gsl_post_hf_2:0:40}..." \ "${PWD}/$gsl_post" gsl_checker_err=true return @@ -301,8 +310,10 @@ else gsl_file_csum=` cksum "$gsl_dir_domain_files/$gsl_post_hf_2" \ | awk '{print $1}'` - gsl_db_post_files+="$gsl_post_hf_2|" - gsl__db_line_file + + printf '%s\n' \ + "File:$gsl_dir_domain_images/$gsl_post_hf_2:$gsl_file_csum" \ + >> "$gsl_db_tmp" fi @@ -341,34 +352,36 @@ gsl__check_header_field3 "Post" "$gsl_log_h_image" "$gsl_marker_image" \ || return # File exists in folder -if ! [[ -f "$gsl_dir_domain_files/$gsl_post_hf_2" ]];then +if ! [[ -f "$gsl_dir_domain_images/$gsl_post_hf_2" ]];then gsl__logs_print \ "$gsl_log_e" \ "Post" \ "$gsl_log_h_image" \ "Not found: $gsl_post_hf_2" \ - "$gsl_dir_domain_files/$gsl_post_hf_2" + "$gsl_dir_domain_images/$gsl_post_hf_2" gsl_checker_err=true else # Register file to DB with hash gsl_file_csum=` - cksum "$gsl_dir_domain_files/$gsl_post_hf_2" \ + cksum "$gsl_dir_domain_images/$gsl_post_hf_2" \ | awk '{print $1}'` - gsl_db_post_files+="$gsl_post_hf_2|" - gsl__db_line_file + + printf '%s\n' \ + "Image:$gsl_dir_domain_images/$gsl_post_hf_2:$gsl_file_csum" \ + >> "$gsl_db_tmp" fi # Exists in Content -gsl_count_files=` +gsl_count_images=` gsl__get_content_line "__image:$gsl_post_hf_1" "$gsl_post" | wc -l` -if (( $gsl_count_files > 0 ));then +if (( $gsl_count_images > 0 ));then - ((gsl_stat_file++)) + ((gsl_stat_image++)) else @@ -412,8 +425,10 @@ else gsl_file_csum=` cksum "$gsl_dir_domain_files/$gsl_post_hf_2" \ | awk '{print $1}'` - gsl_db_post_files+="$gsl_post_hf_2|" - gsl__db_line_file + + printf '%s\n' \ + "FCode:$gsl_dir_domain_filess/$gsl_post_hf_2:$gsl_file_csum" \ + >> "$gsl_db_tmp" fi diff --git a/var/lib/gsl/scripts/gsl__post_manager b/var/lib/gsl/scripts/gsl__post_manager index 656d576..44dfa08 100644 --- a/var/lib/gsl/scripts/gsl__post_manager +++ b/var/lib/gsl/scripts/gsl__post_manager @@ -19,13 +19,15 @@ gsl__logs_print \ for gsl_post in `ls -1 *.gsl 2>/dev/null` do + # Check specific asked Post if [[ "$gsl_this_post" ]];then ! [[ "$gsl_post" == "$gsl_this_post" ]] && continue fi + gsl__get_sum "$gsl_post" + # Post too small - gsl__post_hash_size if (( "$gsl_post_size" <= $gsl_post_min_size ));then gsl__logs_print \ "$gsl_log_e" \ @@ -35,12 +37,16 @@ do "${PWD}/$gsl_post" continue fi - + + # Set DB Post file + gsl_file_db_post="$gsl_dir_db/$gsl_post.db" + gsl__db_get_post_datas + # From COMMAND [OPT] case "$1" in check) if ! [[ "$gsl_force_check" ]];then - gsl__post_compare_hash || continue + gsl__db_compare_post_hash || continue fi gsl__logs_print \ @@ -55,10 +61,10 @@ do make) gsl_process="Made" - gsl__db_exists "$gsl_file_db_posts" + gsl__db_compare_post_hash && continue + case "$gsl_db_post_status" in Checked) - gsl__post_compare_hash || continue gsl__all_makers ;; @@ -76,6 +82,7 @@ do esac done +# End of checkers : show logs for war and err [[ "$gsl_checker_war" ]] \ && gsl log -s -w @@ -111,68 +118,6 @@ gsl_nbr_posts=`ls -1 *.gsl 2>/dev/null | wc -l` && exit 1 } -#====================================================================== -# Check if Post has same Hash -#====================================================================== -gsl__post_compare_hash() { -gsl__db_post_hash - -case "$gsl_process" in - Checked) - if [[ "$gsl_db_post_hash" == "$gsl_post_hash" ]];then - gsl__logs_print \ - "$gsl_log_i" \ - "Post" \ - "Hash" \ - "$gsl_post already Checked from $gsl_post_hash" \ - "$gsl_file_db_posts" - gsl_check_done=true - return 1 - fi - ;; - - Made) - if ! [[ "$gsl_db_post_hash" == "$gsl_post_hash" ]];then - gsl__logs_print \ - "$gsl_log_e" \ - "Post" \ - "Hash" \ - "$gsl_post has changed. Check it again" \ - "$gsl_file_db_posts" - unset gsl_check_done - gsl_checker_err=true - return 1 - fi - - # Compare external content hash - while read "gsl_content_filename" - do - #unset gsl_db_file_hash gsl_file_hash - ! [[ "$gsl_content_filename" ]] && continue - gsl_db_file_hash=` - grep "$gsl_content_filename" "$gsl_file_db_files" \ - | awk -F: '{print $2}'` - gsl_file_hash=` - cksum "$gsl_dir_domain_files/$gsl_content_filename" \ - | awk '{print $1}'` - - if ! [[ "$gsl_db_file_hash" == "$gsl_file_hash" ]];then - gsl__logs_print \ - "$gsl_log_w" \ - "File" \ - "Hash" \ - "$gsl_content_filename has changed. $gsl_db_file_hash $gsl_file_hash" \ - "$gsl_file_db_files" - gsl_checker_war=true - return 1 - fi - done < <(grep "$gsl_post" "$gsl_file_db_posts" \ - | awk -F"|" \ - '{for(i=7;i<=NF;i++){print $i}}') - ;; -esac -} - #====================================================================== # Create TMP file > Get Content Post Only (after #1) #====================================================================== diff --git a/var/lib/gsl/scripts/gsl__tools b/var/lib/gsl/scripts/gsl__tools index 340dc79..fde3ff4 100644 --- a/var/lib/gsl/scripts/gsl__tools +++ b/var/lib/gsl/scripts/gsl__tools @@ -42,9 +42,6 @@ done # Files for gsl_file in "$gsl_file_logs" \ - "$gsl_file_db_posts" \ - "$gsl_file_db_stats" \ - "$gsl_file_db_files" \ "$gsl_file_db_domains" do gsl__check_file "$gsl_file" && touch "$gsl_file" @@ -116,12 +113,12 @@ done #----------------------------------------------------------------------- #======================================================================= -# Get Post Hash +# Get Size & cksum Hash from File... $1 hash/size $2: File #======================================================================= -gsl__post_hash_size() { -gsl_post_csum=`cksum "$gsl_post"` -gsl_post_hash=`awk '{print $1}' <<< "$gsl_post_csum"` -gsl_post_size=`awk '{print $2}' <<< "$gsl_post_csum"` +gsl__get_sum() { +gsl_post_sum=`cksum "$1"` +gsl_post_hash=`awk '{print $1}' <<< "$gsl_post_sum"` +gsl_post_size=`awk '{print $2}' <<< "$gsl_post_sum"` } #======================================================================= @@ -248,7 +245,7 @@ fi # Check OPTIONAL Field 3 from HEADER #======================================================================= gsl__check_header_field3() { -if ! [[ "$gsl_post_header_field_3" ]];then +if ! [[ "$gsl_post_hf_3" ]];then case "$3" in "$gsl_marker_image") gsl__logs_print \