From c924dad232d235cd18abd98e00a4a518c0500a48 Mon Sep 17 00:00:00 2001 From: Cyrille L Date: Sun, 17 Apr 2022 17:08:06 +0200 Subject: [PATCH] Fixes + statoolinfos gen stats with gsl stats --- control | 2 +- etc/gsl/gsl.conf | 1 - usr/local/bin/gsl | 11 ++- var/lib/gsl/scripts/gsl__db_manager | 36 ++++---- var/lib/gsl/scripts/gsl__do_commons | 2 +- var/lib/gsl/scripts/gsl__post_checkers | 65 +++++++------- var/lib/gsl/scripts/gsl__post_manager | 10 +-- var/lib/gsl/scripts/gsl__stats | 113 +++++++++++++++++++++++++ var/lib/gsl/scripts/gsl__tools | 6 +- 9 files changed, 184 insertions(+), 62 deletions(-) create mode 100644 var/lib/gsl/scripts/gsl__stats diff --git a/control b/control index 47df0c1..4bd55e3 100644 --- a/control +++ b/control @@ -1,5 +1,5 @@ Package: egsl -Version: 0.0.34 +Version: 0.0.35 Section: custom Priority: optional Architecture: all diff --git a/etc/gsl/gsl.conf b/etc/gsl/gsl.conf index 9d6f74f..913cc94 100644 --- a/etc/gsl/gsl.conf +++ b/etc/gsl/gsl.conf @@ -33,7 +33,6 @@ gsl_file_logs="$gsl_dir_logs/gsl.log" # Management gsl_file_db_domains="$gsl_dir_db/domains.db" gsl_file_db_posts="$gsl_dir_db/posts.db" -gsl_file_db_stats="$gsl_dir_db/stats.db" gsl_file_db_files="$gsl_dir_db/files.db" # Registred Authors filename diff --git a/usr/local/bin/gsl b/usr/local/bin/gsl index 7138188..1838450 100755 --- a/usr/local/bin/gsl +++ b/usr/local/bin/gsl @@ -1,5 +1,5 @@ #!/bin/bash -# Version: 0.0.34 +# Version: 0.0.35 # file: gsl # Folder: /usr/local/bin # By echolib @@ -433,6 +433,15 @@ case "$1" in gsl__sidebar $1 $2 $3 $4 ;; + stats) + echo "! Working on it..." + gsl__if_file "$gsl_dir_scripts/gsl__stats" source + gsl_file_sti="$gsl_site_server/$gsl_find_domain/gsl-sti.properties" + + gsl__get_stats + exit + ;; + *) gsl help exit diff --git a/var/lib/gsl/scripts/gsl__db_manager b/var/lib/gsl/scripts/gsl__db_manager index 3acdc19..566dc48 100644 --- a/var/lib/gsl/scripts/gsl__db_manager +++ b/var/lib/gsl/scripts/gsl__db_manager @@ -32,7 +32,7 @@ else gsl__logs_print \ "$gsl_log_w" \ "DB" \ - "Post" \ + "File" \ "Missing for $gsl_post. Check it first" \ "$gsl_file_db_post" gsl_checker_war=true @@ -51,7 +51,7 @@ if (( $gsl_post_hash == $gsl_db_post_hash ));then gsl__logs_print \ "$gsl_log_i" \ - "Post" \ + "File" \ "Hash" \ "$gsl_post checked with hash:$gsl_db_post_hash" \ "${PWD}/$gsl_post" @@ -63,7 +63,7 @@ else 0) gsl__logs_print \ "$gsl_log_w" \ - "Post" \ + "File" \ "New" \ "$gsl_post not yet checked" \ "${PWD}/$gsl_post" @@ -72,7 +72,7 @@ else *) gsl__logs_print \ "$gsl_log_w" \ - "Post" \ + "File" \ "Hash" \ "$gsl_post Changed ($gsl_post_hash) !" \ "${PWD}/$gsl_post" @@ -90,11 +90,15 @@ gsl__db_set_post_status() { case "$1" in chk) case "$gsl_db_post_status" in - ""|chk) - [[ "$gsl_post_new_hash" ]] || return # Do Nothing if same Hash - gsl__db_create_post "$1" + "") + gsl_db_post_status=$1 ;; esac + + if ! [[ "$gsl_force_check" ]];then + [[ "$gsl_post_new_hash" ]] || return # Do Nothing if same Hash + fi + gsl__db_create_post "$gsl_db_post_status" ;; wip) @@ -104,10 +108,10 @@ case "$1" in gsl__logs_print \ "$gsl_log_w" \ "DB" \ - "Post" \ + "File" \ "Status:$1 ; $gsl_post" \ "$gsl_file_db_post" - gsl_checker_war=true + gsl_checker_war=true ;; esac ;; @@ -119,10 +123,10 @@ case "$1" in gsl__logs_print \ "$gsl_log_w" \ "DB" \ - "Post" \ + "File" \ "Status:$1 ; $gsl_post" \ "$gsl_file_db_post" - gsl_checker_war=true + gsl_checker_war=true ;; esac ;; @@ -139,8 +143,8 @@ if [[ "$gsl_checker_err" ]];then && gsl__logs_print \ "$gsl_log_w" \ "DB" \ - "Delete" \ - "file ; error in $gsl_post" \ + "File" \ + "Remove ; error in $gsl_post" \ "$gsl_file_db_post" return fi @@ -149,7 +153,7 @@ touch "$gsl_file_db_post" # Main Print printf '%s\n%s\n%s\n' \ - "Post:${PWD}/$gsl_post:$gsl_post_hash:$gsl_post_size" \ + "$gsl_post_type:${PWD}/$gsl_post:$gsl_post_hash:$gsl_post_size" \ "Status:$1" \ "`cat "$gsl_db_tmp"`" \ > "$gsl_file_db_post" @@ -176,7 +180,7 @@ printf '\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \ gsl__logs_print \ "$gsl_log_w" \ "DB" \ - "Create" \ - "Datas from $gsl_post (status: $1)" \ + "File" \ + "Create datas from $gsl_post (status: $1)" \ "$gsl_file_db_post" } diff --git a/var/lib/gsl/scripts/gsl__do_commons b/var/lib/gsl/scripts/gsl__do_commons index f180335..6bcbe53 100644 --- a/var/lib/gsl/scripts/gsl__do_commons +++ b/var/lib/gsl/scripts/gsl__do_commons @@ -153,7 +153,7 @@ do | grep -o "[^[:space:]]*$italic[^[:space:]]*"` case "$this_word" in - "_http"*) + "_http"*|"_xmpp"*|"_ftp"*) continue ;; esac diff --git a/var/lib/gsl/scripts/gsl__post_checkers b/var/lib/gsl/scripts/gsl__post_checkers index 9b2553e..d4bdf31 100644 --- a/var/lib/gsl/scripts/gsl__post_checkers +++ b/var/lib/gsl/scripts/gsl__post_checkers @@ -25,7 +25,6 @@ gsl_db_tmp=`mktemp` gsl__check_h1 || return gsl__post_content_only -gsl__get_needed_headers gsl__check_type gsl__check_headers gsl__check_paragraphs "$gsl_tmp_post" @@ -58,7 +57,7 @@ rm -f "$gsl_db_tmp" # Check Post for begining content #====================================================================== gsl__check_h1() { -echo -ne ": Searching for #1..." +echo -ne ": Searching for begining #1..." gsl_post_begin=` grep -n "#1" $gsl_post \ @@ -68,7 +67,7 @@ grep -n "#1" $gsl_post \ if ! [[ "$gsl_post_begin" ]];then gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_h1" \ "Missing: #1 TITLE" \ "${PWD}/$gsl_post" @@ -86,7 +85,7 @@ case "$gsl_post_type" in page|post) gsl__logs_print \ "$gsl_log_i" \ - "Post" \ + "Content" \ "Set as" \ "$gsl_post_type: $gsl_post" \ "${PWD}/$gsl_post" @@ -94,10 +93,10 @@ case "$gsl_post_type" in *) gsl_post_type="post" sed -i "1i$gsl_marker_type$gsl_post_type" \ - "$gsl_post" && \ + "$gsl_post" && \ gsl__logs_print \ "$gsl_log_w" \ - "Post" \ + "Content" \ "Set as" \ "Default: $gsl_post_type for $gsl_post" \ "${PWD}/$gsl_post" @@ -128,7 +127,7 @@ if (( "$gsl_post_p_open" == 0 ));then gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_loc_c_p" \ "Missing: ( and ) at begining lines" \ "${PWD}/$gsl_post" @@ -145,7 +144,7 @@ else gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_p" \ "Mismatch: (=$gsl_post_p_open ; )=$gsl_post_p_close" \ "${PWD}/$gsl_post" @@ -177,7 +176,7 @@ else gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_cite" \ "Mismatch: --- not paired" \ "${PWD}/$gsl_post" @@ -191,17 +190,17 @@ fi gsl__check_link() { # Get & Check Header CONTENT gsl__check_header_fields \ - "Post" "$gsl_log_h_link" "$gsl_log_act_link" \ + "Content" "$gsl_log_h_link" "$gsl_log_act_link" \ || return echo -ne "\r\033[2K: Searching for Links... $gsl_post_hf_1" # No ALT TEXT ? -gsl__check_header_field3 "Post" "$gsl_log_h_link" "$gsl_marker_link" +gsl__check_header_field3 "Content" "$gsl_log_h_link" "$gsl_marker_link" gsl_count_links=` -gsl__get_content_line "_$gsl_post_hf_1" "$gsl_post" | wc -l -` +gsl__get_content_line "_$gsl_post_hf_1" "$gsl_post" | wc -l` + if (( "$gsl_count_links" > 0 ));then # Check URL @@ -214,7 +213,7 @@ if (( "$gsl_count_links" > 0 ));then 1*|2*|3*) gsl__logs_print \ "$gsl_log_i" \ - "Post" \ + "Content" \ "$gsl_log_h_link" \ "Online:$gsl_url_status ${gsl_post_hf_2:0:40}..." \ "${PWD}/$gsl_post" @@ -222,7 +221,7 @@ if (( "$gsl_count_links" > 0 ));then ""|*) gsl__logs_print \ "$gsl_log_w" \ - "Post" \ + "Content" \ "$gsl_log_h_link" \ "Offline:$gsl_url_status ${gsl_post_hf_2:0:40}..." \ "${PWD}/$gsl_post" @@ -242,7 +241,7 @@ else gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_link" \ "Unused: _$gsl_post_hf_1" \ "${PWD}/$gsl_post" @@ -256,7 +255,7 @@ fi #====================================================================== gsl__check_abbr() { gsl__check_header_fields \ - "Post" "$gsl_log_h_abbr" "$gsl_log_act_abbr" \ + "Content" "$gsl_log_h_abbr" "$gsl_log_act_abbr" \ || return echo -ne "\r\033[2K: Searching for ABBRs... $gsl_post_hf_1" @@ -274,7 +273,7 @@ else # Missing CONTENT gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_abbr" \ "Missing: $gsl_post_header_field_1" \ "${PWD}/$gsl_post" @@ -288,19 +287,19 @@ fi #====================================================================== gsl__check_file() { gsl__check_header_fields \ - "Post" "$gsl_log_h_file" "$gsl_log_act_file" \ + "Content" "$gsl_log_h_file" "$gsl_log_act_file" \ || return echo -ne "\r\033[2K: Searching for Content Files... $gsl_post_hf_1" -gsl__check_header_field3 "Post" "$gsl_log_h_file" "$gsl_marker_file" +gsl__check_header_field3 "Content" "$gsl_log_h_file" "$gsl_marker_file" # File exists in folder if ! [[ -f "$gsl_dir_domain_files/$gsl_post_hf_2" ]];then gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "Header $gsl_marker_file" \ "Not found: $gsl_post_hf_2" \ "$gsl_dir_domain_files/$gsl_post_hf_2" @@ -331,7 +330,7 @@ else gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_file" \ "Missing: __$gsl_post_hf_1" \ "${PWD}/$gsl_post" @@ -345,12 +344,12 @@ fi #====================================================================== gsl__check_image() { gsl__check_header_fields \ - "Post" "$gsl_log_h_image" "$gsl_log_act_image" \ + "Content" "$gsl_log_h_image" "$gsl_log_act_image" \ || return echo -ne "\r\033[2K: Searching for Images... $gsl_post_hf_1" -gsl__check_header_field3 "Post" "$gsl_log_h_image" "$gsl_marker_image" \ +gsl__check_header_field3 "Content" "$gsl_log_h_image" "$gsl_marker_image" \ || return # File exists in folder @@ -358,7 +357,7 @@ if ! [[ -f "$gsl_dir_domain_images/$gsl_post_hf_2" ]];then gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_h_image" \ "Not found: $gsl_post_hf_2" \ "$gsl_dir_domain_images/$gsl_post_hf_2" @@ -389,7 +388,7 @@ else gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_image" \ "Missing: _image:$gsl_post_hf_1" \ "${PWD}/$gsl_post" @@ -404,19 +403,19 @@ fi gsl__check_fcode() { gsl__check_header_fields \ - "Post" "$gsl_log_h_code" "$gsl_log_act_code" \ + "Content" "$gsl_log_h_code" "$gsl_log_act_code" \ || return echo -ne "\r\033[2K: Searching for File Codes... $gsl_post_hf_1" -gsl__check_header_field3 "Post" "$gsl_log_h_code" "$gsl_marker_code" \ +gsl__check_header_field3 "Content" "$gsl_log_h_code" "$gsl_marker_code" \ # File exists in folder if ! [[ -f "$gsl_dir_domain_files/$gsl_post_hf_2" ]];then gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_h_code" \ "Not found: $gsl_post_hf_2" \ "$gsl_dir_domain_files/$gsl_post_hf_2" @@ -447,7 +446,7 @@ else gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "$gsl_log_c_code" \ "Missing: _code:$gsl_post_hf_1" \ "${PWD}/$gsl_post" @@ -528,7 +527,7 @@ if ! [[ "$gsl_author_err" ]] && \ gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "Header $gsl_marker_author" \ "$gsl_header_author not registred for domain $gsl_find_domain" \ "${PWD}/$gsl_post" @@ -542,7 +541,7 @@ if ! [[ "$gsl_date_err" ]] && \ gsl__logs_print \ "$gsl_log_e" \ - "Post" \ + "Content" \ "Header $gsl_marker_date" \ "$gsl_header_date not YYYY-MM-DD" \ "${PWD}/$gsl_post" @@ -562,7 +561,7 @@ if ! [[ "$gsl_slug_err" ]];then "$gsl_post" && \ gsl__logs_print \ "$gsl_log_w" \ - "Post" \ + "Content" \ "Header $gsl_marker_slug" \ "Changed: $gsl_header_slug_test" \ "${PWD}/$gsl_post" diff --git a/var/lib/gsl/scripts/gsl__post_manager b/var/lib/gsl/scripts/gsl__post_manager index 8b3dfa6..48def21 100644 --- a/var/lib/gsl/scripts/gsl__post_manager +++ b/var/lib/gsl/scripts/gsl__post_manager @@ -14,7 +14,7 @@ gsl__check_nbr_posts (( $gsl_nbr_posts == 0 )) \ && gsl__logs_print \ "$gsl_log_i" \ - "Posts" \ + "Files" \ "Not found" \ "$gsl_nbr_posts post. Nothing to do in" \ "${PWD}" \ @@ -22,9 +22,9 @@ gsl__check_nbr_posts gsl__logs_print \ "$gsl_log_i" \ - "Posts" \ + "Files" \ "Found" \ - "$gsl_nbr_posts posts in" \ + "$gsl_nbr_posts" \ "${PWD}" for gsl_post in `ls -1 *.gsl 2>/dev/null` @@ -42,7 +42,7 @@ do if (( "$gsl_post_size" <= $gsl_post_min_size ));then gsl__logs_print \ "$gsl_log_w" \ - "Post" \ + "File" \ "Size" \ "$gsl_post - Too small $gsl_post_size <= $gsl_post_min_size" \ "${PWD}/$gsl_post" @@ -94,7 +94,7 @@ do gsl__logs_print \ "$gsl_log_i" \ "Reading" \ - "Post" \ + "File" \ "$gsl_post" \ "${PWD}" gsl__all_checkers diff --git a/var/lib/gsl/scripts/gsl__stats b/var/lib/gsl/scripts/gsl__stats new file mode 100644 index 0000000..5baf3ac --- /dev/null +++ b/var/lib/gsl/scripts/gsl__stats @@ -0,0 +1,113 @@ +#!/bin/bash +# file: gsl__stats +# Folder: /var/lib/gsl/scripts +# By echolib +# License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 + +gsl__get_stats() { +gsl_stats_article_all=0 # All non error + +# Status +gsl_stats_articles_www=0 # All www +gsl_stats_articles_pages_www=0 # All page www +gsl_stats_articles_posts_www=0 # All page www + +gsl_stats_articles_wip=0 # All wip +gsl_stats_articles_pages_wip=0 +gsl_stats_articles_posts_wip=0 + +gsl_stats_articles_chk=0 # All wip +gsl_stats_articles_pages_chk=0 +gsl_stats_articles_posts_chk=0 + +# Author +gsl_auth_max=0 +while read -r "N" "AUTHOR" +do + gsl_auth[$N]="$AUTHOR" + gsl_this_auth[$N]=0 + ((gsl_auth_max++)) +done < <(cat -n "$gsl_this_conf_domain/$gsl_filename_auth") + +# Counting +while read -r "file_gsl_db" +do + ((gsl_stats_article_all++)) + gsl_file_uri_db="$gsl_dir_db/$file_gsl_db" + + # Author + gsl_auth_name=`grep "Author:" $gsl_file_uri_db | awk -F: '{print $2}'` + for a in `seq 1 $gsl_auth_max` + do + [[ ${gsl_auth[$a]} == "$gsl_auth_name" ]] \ + && ((gsl_this_auth[$a]++)) + done + + # Online + if [[ `grep "Status:www" $gsl_file_uri_db` ]];then + ((gsl_stats_articles_www++)) + + [[ `grep "page:" $gsl_file_uri_db` ]] \ + && ((gsl_stats_articles_pages_www++)) + + [[ `grep "post:" $gsl_file_uri_db` ]] \ + && ((gsl_stats_articles_posts_www++)) + + # WIP + elif [[ `grep "Status:wip" $gsl_file_uri_db` ]];then + ((gsl_stats_articles_wip++)) + + [[ `grep "page:" $gsl_file_uri_db` ]] \ + && ((gsl_stats_articles_pages_wip++)) + + [[ `grep "post:" $gsl_file_uri_db` ]] \ + && ((gsl_stats_articles_posts_wip++)) + + # CHK + elif [[ `grep "Status:chk" $gsl_file_uri_db` ]];then + ((gsl_stats_articles_chk++)) + + [[ `grep "page:" $gsl_file_uri_db` ]] \ + && ((gsl_stats_articles_pages_chk++)) + + [[ `grep "post:" $gsl_file_uri_db` ]] \ + && ((gsl_stats_articles_posts_chk++)) + fi + +done < <(ls -1 "$gsl_dir_db" | grep ".gsl.db") + +# Creating statoolinfos file +#gsl.articles=nombre d'articles +#gsl.articles.pages=nombre d'articles type page +#gsl.articles.posts=nombre d'articles type page +#gsl.authors=nombre d'auteurs + +printf '%s\n%s\n%s\n%s\n' \ + "gsl.articles=$gsl_stats_articles_www" \ + "gsl_articles_pages=$gsl_stats_articles_pages_www" \ + "gsl_articles_posts=$gsl_stats_articles_posts_www" \ + "gsl.authors=$gsl_auth_max" \ + > "$gsl_file_sti" && \ +echo "> Génération des stats (Statoolinfos): $gsl_file_sti" + +# Show stats +echo -e "# Total (no error): $gsl_stats_article_all" +for a in `seq 1 $gsl_auth_max` +do + echo "- ${gsl_auth[$a]}: ${gsl_this_auth[a]}" +done + +echo +printf '%s\n%s%s%s\n%s%s%s\n%s%s%s\n' \ + "CHK|WIP|WWW" \ + "All: $gsl_stats_articles_chk|" \ + "All: $gsl_stats_articles_wip|" \ + "All: $gsl_stats_articles_www" \ + "Pages: $gsl_stats_articles_pages_chk|" \ + "Pages: $gsl_stats_articles_pages_wip|" \ + "Pages: $gsl_stats_articles_pages_www" \ + "Posts: $gsl_stats_articles_posts_chk|" \ + "Posts: $gsl_stats_articles_posts_wip|" \ + "Posts: $gsl_stats_articles_posts_wip" \ + | column -t -s'|' +} diff --git a/var/lib/gsl/scripts/gsl__tools b/var/lib/gsl/scripts/gsl__tools index d1c3f58..9d50c1c 100644 --- a/var/lib/gsl/scripts/gsl__tools +++ b/var/lib/gsl/scripts/gsl__tools @@ -125,9 +125,7 @@ awk -F"$1" -v marker="$1" -v l="$gsl_post_begin" \ # Get line nbr from CONTENT Post... $1: TERM | $2: File #======================================================================= gsl__get_content_line() { -awk -v l="$gsl_post_begin" -v s="$1" \ - 'NR >= l && $0 ~ s {print NR}' \ - "$2" || return +grep -n "$1" "$2" | awk -F: '{print $1}' || return } #======================================================================= @@ -260,5 +258,5 @@ printf '\n%s\n%s\n%s\n%s\n' \ "D> 2. '$2'" \ "D> 3. '$3'" \ "D> 4. '$4'" -sleep 1 +sleep .5 }