Updated: New DB System

This commit is contained in:
Cyrille L 2022-03-01 18:39:46 +01:00
parent 5c40f613d4
commit f9bdad2da2
6 changed files with 144 additions and 251 deletions

View File

@ -4,160 +4,101 @@
# By echolib
# License: GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007
#=======================================================================
# Get Post Hash FROM DB posts.db
#=======================================================================
gsl__db_post_hash() {
gsl_db_post_hash=`
grep "$gsl_post" \
"$gsl_file_db_posts" \
| awk -F"|" '{print $3}'`
}
#=======================================================================
# From checkers (bcodes,images,files): Write DB
#=======================================================================
gsl__db_line_file() {
gsl__db_exists "$gsl_file_db_files" "$gsl_post_hf_2"
gsl_db_line_file="$gsl_post_hf_2:$gsl_file_csum"
#======================================================================
# Get Post Hash & Status From Database
#======================================================================
gsl__db_get_post_datas() {
gsl_db_post_hash=0
if [[ -f "$gsl_file_db_post" ]];then
if [[ "$gsl_db_file_hash" ]];then
gsl_db_post_hash=`
awk -F: 'NR == 1 {print $3}' \
"$gsl_file_db_post"`
sed -i "${gsl_db_file_line_nbr}s,.*,$gsl_db_line_file," \
"$gsl_file_db_files"
else
echo "$gsl_post_hf_2:$gsl_file_csum" \
>> "$gsl_file_db_files"
gsl_db_post_status=`
awk -F: 'NR == 2 {print $2}' \
"$gsl_file_db_post"`
fi
}
#=======================================================================
# Check Post exists and get line NBR in DB | $1 DB file $2 TERM
#=======================================================================
gsl__db_exists() {
case "$1" in
"$gsl_file_db_posts")
gsl_db_post_exists=`
grep -n "$gsl_post" "$1"`
if [[ "$gsl_db_post_exists" ]];then
gsl_db_post_line_nbr=`
awk -F":" '{print $1}' <<< "$gsl_db_post_exists"`
gsl_db_post_status=`
awk -F"|" '{print $5}' <<< "$gsl_db_post_exists"`
fi
;;
"$gsl_file_db_stats")
gsl_db_stat_exists=`
grep -n "$gsl_post" "$1"`
if [[ "$gsl_db_stat_exists" ]];then
gsl_db_stat_line_nbr=`
awk -F":" '{print $1}' <<< "$gsl_db_stat_exists"`
fi
;;
"$gsl_file_db_files")
gsl_db_file_exists=`grep -n "$2" "$1"`
if [[ "$gsl_db_file_exists" ]];then
gsl_db_file_line_nbr=`
awk -F: '{print $1}' <<< "$gsl_db_file_exists"`
gsl_db_file_hash=`
awk -F: '{print $3}' <<< "$gsl_db_file_exists"`
fi
;;
esac
#======================================================================
# Get Post Hash and Compare from Database
#======================================================================
gsl__db_compare_post_hash() {
! (( $gsl_post_hash == $gsl_db_post_hash )) \
&& gsl__logs_print \
"$gsl_log_w" \
"Post" \
"Hash" \
"$gsl_post Changed ($gsl_post_hash)" \
"${PWD}/$gsl_post" \
&& gsl_checker_war=true \
&& return
}
#=======================================================================
# From gsl__loop_posts: Write DB
#=======================================================================
gsl__db_line_post() {
#======================================================================
# Set Post Status in Database according to proceess
#======================================================================
gsl__db_set_post_status() {
case "$gsl_process" in
Checked)
#-------------------------------------------------------------------
# Check if post exists in DB stats.db and get line
#-------------------------------------------------------------------
gsl_db_stat_line=`
printf '%s%s%s%s%s%s%s%s%s%s\n' \
"$gsl_post|" \
"$gsl_post_stat_w|" \
"$gsl_post_stat_p|" \
"$gsl_post_stat_abbr|" \
"$gsl_post_stat_link|" \
"$gsl_post_stat_image|" \
"$gsl_post_stat_file|" \
"$gsl_post_stat_bq|" \
"$gsl_post_stat_bcode|" \
"${gsl_post_stat_mdb// /|}|"`
if ! [[ "$gsl_checker_err" ]];then
gsl__db_exists "$gsl_file_db_stats"
if [[ "$gsl_db_stat_line_nbr" ]];then
sed -i "${gsl_db_stat_line_nbr}s,.*,$gsl_db_stat_line," \
"$gsl_file_db_stats" && \
gsl__logs_print \
"$gsl_log_w" \
"DB" \
"Stat" \
"Line:$gsl_db_stat_line_nbr $gsl_post" \
"$gsl_file_db_stats"
else
echo "$gsl_db_stat_line" >> "$gsl_file_db_stats" && \
gsl__logs_print \
"$gsl_log_w" \
"DB" \
"Stat" \
"Line:NEW $gsl_db_stat_line" \
"$gsl_file_db_stats"
fi
fi
#-------------------------------------------------------------------
# Check if post exists in DB posts.db and get line / status
#-------------------------------------------------------------------
gsl__db_exists "$gsl_file_db_posts"
if [[ "$gsl_checker_err" ]] && [[ "$gsl_db_post_line_nbr" ]];then
sed -i "${gsl_db_post_line_nbr}d" "$gsl_file_db_posts" && \
gsl__logs_print \
"$gsl_log_w" \
"DB" \
"Post" \
"Line:$gsl_db_post_line_nbr removed. Error in $gsl_post" \
"$gsl_file_db_posts"
return
fi
gsl_db_post_line=`
printf '%s%s%s%s%s%s%s\n' \
"$gsl_post_type|" \
"$gsl_post|" \
"$gsl_post_hash|" \
"$gsl_post_size|" \
"$gsl_process|" \
"${PWD}/$gsl_post|"\
"$gsl_db_post_files"`
if [[ "$gsl_db_post_exists" ]];then
sed -i "${gsl_db_post_line_nbr}s,.*,$gsl_db_post_line," \
"$gsl_file_db_posts" && \
gsl__logs_print \
"$gsl_log_w" \
"DB" \
"Post" \
"Line:REPLACED - $gsl_post - Hash: $gsl_post_hash" \
"$gsl_file_db_posts"
gsl_checker_war=true
else
echo "$gsl_db_post_line" \
>> "$gsl_file_db_posts" && \
gsl__logs_print \
"$gsl_log_i" \
"DB" \
"Post" \
"Line:NEW - $gsl_post - Hash: $gsl_post_hash" \
"$gsl_file_db_posts"
fi
case "$gsl_db_post_status" in
Checked)
gsl__db_compare_post_hash || return
gsl__db_create_post
;;
Made)
return
;;
Online)
return
;;
*)
gsl__db_create_post
;;
esac
;;
esac
}
#======================================================================
# Create DB File from Data Post
#======================================================================
gsl__db_create_post() {
touch "$gsl_file_db_post"
# Main Print
printf '%s\n%s\n%s\n' \
"Post:${PWD}/$gsl_post:$gsl_post_hash:$gsl_post_size" \
"Status:$gsl_process" \
"`cat "$gsl_db_tmp"`" \
> "$gsl_file_db_post"
# Stats Print
printf '\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \
"Stats" \
"Author:$gsl_header_author" \
"Date:$gsl_header_date" \
"Words:$gsl_stat_words" \
"Parag:$gsl_stat_p" \
"Links:$gsl_stat_link:$gsl_stat_links" \
"Strongs:$gsl_stat_strongs" \
"Bolds:$gsl_stat_bolds"\
"Italics:$gsl_stat_italics" \
"Abbrs:$gsl_stat_abbr:$gsl_stat_abbrs" \
"Images:$gsl_stat_image" \
"Files:$gsl_stat_file:$gsl_count_files" \
"Cites:$gsl_stat_bq" \
"ICodes:$gsl_stat_icode" \
"FCodes:$gsl_stat_fcode" \
>> "$gsl_file_db_post"
gsl__logs_print \
"$gsl_log_w" \
"DB" \
"Create" \
"Datas from $gsl_post" \
"$gsl_file_db_post"
}

View File

@ -23,12 +23,6 @@ gsl_header_tags=`gsl__get_header "$gsl_marker_tags" "$gsl_post"`
gsl__do_header() {
unset ${!gsl_post_hf@}
# Stats
gsl_stat_link=0
gsl_stat_abbr=0
gsl_stat_file=0
gsl_stat_fcode=0
while read -r "gsl_header_content_line"
do
gsl__get_header_fields "$2"
@ -179,7 +173,7 @@ do
case "$1" in
"stats")
((gsl_stat_icodes++))
((gsl_stat_icode++))
;;
"make")
echo -ne "\r\033[2K: Convrting Inline-Code... $icode"

View File

@ -18,6 +18,7 @@ gsl__page_prepare_datas
gsl__check_templates
gsl__html_meta_head
gsl__page_create
cat -n "$gsl_srv_wip_page"
}
#-----------------------------------------------------------------------
@ -58,7 +59,7 @@ gsl_uri_site_logo="$gsl_uri/templates/$gsl_site_logo"
#-----------------------------------------------------------------------
gsl__check_srv_files() {
if [[ -f "$1" ]];then
echo "! $1"
echo "! Replaced Page: $1"
fi
}

View File

@ -10,9 +10,17 @@
# --------------------------------------------------
#----------------------------------------------------------------------
gsl__all_checkers() {
clear
unset gsl_check_done
# Stats
gsl_db_tmp=`mktemp`
gsl_stat_link=0
gsl_stat_abbr=0
gsl_stat_file=0
gsl_stat_image=0
gsl_stat_icode=0
gsl_stat_fcode=0
gsl__check_h1 || return
gsl__post_content_only
@ -36,8 +44,9 @@ rm -f "$gsl_tmp_post"
echo -ne "\r\033[2K"
# End of checkers : show logs for war and err
gsl__db_line_post
gsl__db_set_post_status
rm -f "$gsl_db_tmp"
}
#----------------------------------------------------------------------
@ -194,20 +203,20 @@ gsl__get_content_line "_$gsl_post_hf_1" "$gsl_post" | wc -l
if (( "$gsl_count_links" > 0 ));then
# Check URL
gsl_url_online=`
gsl_url_status=`
curl -o /dev/null --silent --head --write-out \
'%{http_code}' \
"$gsl_post_hf_2" 2>/dev/null`
if (( $gsl_url_online == 200 )) || \
(( $gsl_url_online == 301 )) || \
(( $gsl_url_online == 302 ));then
if (( $gsl_url_status == 200 )) || \
(( $gsl_url_status == 301 )) || \
(( $gsl_url_status == 302 ));then
gsl__logs_print \
"$gsl_log_i" \
"Post" \
"$gsl_log_h_link" \
"Online:$gsl_url_online ${gsl_post_hf_2:0:40}..." \
"Online:$gsl_url_status ${gsl_post_hf_2:0:40}..." \
"${PWD}/$gsl_post"
else
@ -216,7 +225,7 @@ if (( "$gsl_count_links" > 0 ));then
"$gsl_log_e" \
"Post" \
"$gsl_log_h_link" \
"Offline:$gsl_url_online ${gsl_post_hf_2:0:40}..." \
"Offline:$gsl_url_status ${gsl_post_hf_2:0:40}..." \
"${PWD}/$gsl_post"
gsl_checker_err=true
return
@ -301,8 +310,10 @@ else
gsl_file_csum=`
cksum "$gsl_dir_domain_files/$gsl_post_hf_2" \
| awk '{print $1}'`
gsl_db_post_files+="$gsl_post_hf_2|"
gsl__db_line_file
printf '%s\n' \
"File:$gsl_dir_domain_images/$gsl_post_hf_2:$gsl_file_csum" \
>> "$gsl_db_tmp"
fi
@ -341,34 +352,36 @@ gsl__check_header_field3 "Post" "$gsl_log_h_image" "$gsl_marker_image" \
|| return
# File exists in folder
if ! [[ -f "$gsl_dir_domain_files/$gsl_post_hf_2" ]];then
if ! [[ -f "$gsl_dir_domain_images/$gsl_post_hf_2" ]];then
gsl__logs_print \
"$gsl_log_e" \
"Post" \
"$gsl_log_h_image" \
"Not found: $gsl_post_hf_2" \
"$gsl_dir_domain_files/$gsl_post_hf_2"
"$gsl_dir_domain_images/$gsl_post_hf_2"
gsl_checker_err=true
else
# Register file to DB with hash
gsl_file_csum=`
cksum "$gsl_dir_domain_files/$gsl_post_hf_2" \
cksum "$gsl_dir_domain_images/$gsl_post_hf_2" \
| awk '{print $1}'`
gsl_db_post_files+="$gsl_post_hf_2|"
gsl__db_line_file
printf '%s\n' \
"Image:$gsl_dir_domain_images/$gsl_post_hf_2:$gsl_file_csum" \
>> "$gsl_db_tmp"
fi
# Exists in Content
gsl_count_files=`
gsl_count_images=`
gsl__get_content_line "__image:$gsl_post_hf_1" "$gsl_post" | wc -l`
if (( $gsl_count_files > 0 ));then
if (( $gsl_count_images > 0 ));then
((gsl_stat_file++))
((gsl_stat_image++))
else
@ -412,8 +425,10 @@ else
gsl_file_csum=`
cksum "$gsl_dir_domain_files/$gsl_post_hf_2" \
| awk '{print $1}'`
gsl_db_post_files+="$gsl_post_hf_2|"
gsl__db_line_file
printf '%s\n' \
"FCode:$gsl_dir_domain_filess/$gsl_post_hf_2:$gsl_file_csum" \
>> "$gsl_db_tmp"
fi

View File

@ -19,13 +19,15 @@ gsl__logs_print \
for gsl_post in `ls -1 *.gsl 2>/dev/null`
do
# Check specific asked Post
if [[ "$gsl_this_post" ]];then
! [[ "$gsl_post" == "$gsl_this_post" ]] && continue
fi
gsl__get_sum "$gsl_post"
# Post too small
gsl__post_hash_size
if (( "$gsl_post_size" <= $gsl_post_min_size ));then
gsl__logs_print \
"$gsl_log_e" \
@ -36,11 +38,15 @@ do
continue
fi
# Set DB Post file
gsl_file_db_post="$gsl_dir_db/$gsl_post.db"
gsl__db_get_post_datas
# From COMMAND [OPT]
case "$1" in
check)
if ! [[ "$gsl_force_check" ]];then
gsl__post_compare_hash || continue
gsl__db_compare_post_hash || continue
fi
gsl__logs_print \
@ -55,10 +61,10 @@ do
make)
gsl_process="Made"
gsl__db_exists "$gsl_file_db_posts"
gsl__db_compare_post_hash && continue
case "$gsl_db_post_status" in
Checked)
gsl__post_compare_hash || continue
gsl__all_makers
;;
@ -76,6 +82,7 @@ do
esac
done
# End of checkers : show logs for war and err
[[ "$gsl_checker_war" ]] \
&& gsl log -s -w
@ -111,68 +118,6 @@ gsl_nbr_posts=`ls -1 *.gsl 2>/dev/null | wc -l`
&& exit 1
}
#======================================================================
# Check if Post has same Hash
#======================================================================
gsl__post_compare_hash() {
gsl__db_post_hash
case "$gsl_process" in
Checked)
if [[ "$gsl_db_post_hash" == "$gsl_post_hash" ]];then
gsl__logs_print \
"$gsl_log_i" \
"Post" \
"Hash" \
"$gsl_post already Checked from $gsl_post_hash" \
"$gsl_file_db_posts"
gsl_check_done=true
return 1
fi
;;
Made)
if ! [[ "$gsl_db_post_hash" == "$gsl_post_hash" ]];then
gsl__logs_print \
"$gsl_log_e" \
"Post" \
"Hash" \
"$gsl_post has changed. Check it again" \
"$gsl_file_db_posts"
unset gsl_check_done
gsl_checker_err=true
return 1
fi
# Compare external content hash
while read "gsl_content_filename"
do
#unset gsl_db_file_hash gsl_file_hash
! [[ "$gsl_content_filename" ]] && continue
gsl_db_file_hash=`
grep "$gsl_content_filename" "$gsl_file_db_files" \
| awk -F: '{print $2}'`
gsl_file_hash=`
cksum "$gsl_dir_domain_files/$gsl_content_filename" \
| awk '{print $1}'`
if ! [[ "$gsl_db_file_hash" == "$gsl_file_hash" ]];then
gsl__logs_print \
"$gsl_log_w" \
"File" \
"Hash" \
"$gsl_content_filename has changed. $gsl_db_file_hash $gsl_file_hash" \
"$gsl_file_db_files"
gsl_checker_war=true
return 1
fi
done < <(grep "$gsl_post" "$gsl_file_db_posts" \
| awk -F"|" \
'{for(i=7;i<=NF;i++){print $i}}')
;;
esac
}
#======================================================================
# Create TMP file > Get Content Post Only (after #1)
#======================================================================

View File

@ -42,9 +42,6 @@ done
# Files
for gsl_file in "$gsl_file_logs" \
"$gsl_file_db_posts" \
"$gsl_file_db_stats" \
"$gsl_file_db_files" \
"$gsl_file_db_domains"
do
gsl__check_file "$gsl_file" && touch "$gsl_file"
@ -116,12 +113,12 @@ done
#-----------------------------------------------------------------------
#=======================================================================
# Get Post Hash
# Get Size & cksum Hash from File... $1 hash/size $2: File
#=======================================================================
gsl__post_hash_size() {
gsl_post_csum=`cksum "$gsl_post"`
gsl_post_hash=`awk '{print $1}' <<< "$gsl_post_csum"`
gsl_post_size=`awk '{print $2}' <<< "$gsl_post_csum"`
gsl__get_sum() {
gsl_post_sum=`cksum "$1"`
gsl_post_hash=`awk '{print $1}' <<< "$gsl_post_sum"`
gsl_post_size=`awk '{print $2}' <<< "$gsl_post_sum"`
}
#=======================================================================
@ -248,7 +245,7 @@ fi
# Check OPTIONAL Field 3 from HEADER
#=======================================================================
gsl__check_header_field3() {
if ! [[ "$gsl_post_header_field_3" ]];then
if ! [[ "$gsl_post_hf_3" ]];then
case "$3" in
"$gsl_marker_image")
gsl__logs_print \