From eda5d7f6fcdbb0f405231eb88e28dbbcb945d710 Mon Sep 17 00:00:00 2001
From: Cyrille L <echolib@a-lec.org>
Date: Fri, 17 Mar 2023 15:25:35 +0100
Subject: [PATCH] arguments "all", "again", "newer" with wip/publish. Invert
 item in RSS (newest checked first. Lots of changes)

---
 src/usr/bin/tyto                    |  1 -
 src/var/lib/tyto/program/args.py    | 10 ++++++
 src/var/lib/tyto/program/check.py   | 28 +++++++--------
 src/var/lib/tyto/program/db.py      | 33 +++++++++++++++---
 src/var/lib/tyto/program/domain.py  | 22 ++++++++++++
 src/var/lib/tyto/program/html.py    | 25 ++++++++++++-
 src/var/lib/tyto/program/infos.py   |  4 ++-
 src/var/lib/tyto/program/publish.py | 54 +++++++++++++----------------
 src/var/lib/tyto/program/rss.py     | 13 ++++---
 src/var/lib/tyto/program/stats.py   |  3 +-
 src/var/lib/tyto/program/tyto.py    | 45 +++++++++++++++++++++---
 src/var/lib/tyto/program/wip.py     | 41 ++++++++++++----------
 12 files changed, 198 insertions(+), 81 deletions(-)

diff --git a/src/usr/bin/tyto b/src/usr/bin/tyto
index d5dfb5b..646a9de 100755
--- a/src/usr/bin/tyto
+++ b/src/usr/bin/tyto
@@ -44,7 +44,6 @@ target = args.set_target()
 # Command start argument
 import check, domain, html, new, publish, show, wip, infos
 actions = {
-           'all'      : wip.manage_wip,
            'check'    : check.manage_check,
            'help'     : infos.tyto,
            'edit'     : show.manage_show,
diff --git a/src/var/lib/tyto/program/args.py b/src/var/lib/tyto/program/args.py
index e00e599..b37813d 100644
--- a/src/var/lib/tyto/program/args.py
+++ b/src/var/lib/tyto/program/args.py
@@ -64,6 +64,8 @@ pass_db = \
 pass_targets = \
 (
  'all',
+ 'again',
+ 'newer',
  'domain',
  'footer',
  'metas',
@@ -73,6 +75,8 @@ pass_targets = \
  'template'
 )
 
+multi_chk = ('all', 'again')
+
 action = ''
 target = ''
 noaction = False
@@ -97,6 +101,12 @@ if not action in actions:
 try:    target = sys.argv[2]
 except: target = ''
 
+
+# Some conditions exiting
+if action == "check" and target == "newer":
+  logs.out("11", '"%s" with "%s"'%(target, action), True)
+
+
 # Set action and target for binary
 def set_action():
   return(action)
diff --git a/src/var/lib/tyto/program/check.py b/src/var/lib/tyto/program/check.py
index 0fc7009..7b23dd5 100644
--- a/src/var/lib/tyto/program/check.py
+++ b/src/var/lib/tyto/program/check.py
@@ -35,11 +35,12 @@ def manage_check(target):
   domain.domain_needed()
 
   # target needed
-  if not target: logs.out("5", '', True)
+  if not target:
+    logs.out("5", '', True)
   
-  elif not target == 'all' \
+  elif not target in args.multi_chk \
        and db.db_exists \
-       and db.hash_chk == db.hash_post:
+       and not db.old_chk:
     logs.out("20", db.date_chk, False)
     ask = ' ├ Check again this article ? '
     try:
@@ -51,13 +52,13 @@ def manage_check(target):
       return
     check_process(target)
   
-  elif target == "all":
-    check_all()
+  elif target in args.multi_chk:
+    check_all(target)
   
   else:
     check_process(target)
 
-  if post_err and not target == 'all':
+  if post_err and not target in args.multi_chk:
     logs.out("7", '', True)
   
 
@@ -66,7 +67,7 @@ def manage_check(target):
 # check all realdy checkded articles           #
 # but check only if article source has changed #
 #----------------------------------------------#
-def check_all():
+def check_all(option):
   found = False
 
   for post_db in os.listdir(db.articles_db):
@@ -77,19 +78,16 @@ def check_all():
       args.target = post_src.rsplit('%s/'%db.in_dir)[1]
       importlib.reload(db)
       
-      if not db.db_exists \
-         or  db.hash_post == db.hash_chk:
-        continue
+      if not db.db_exists:                     continue
+      if option == 'again' and not db.old_chk: continue
       
       found = True
-      print(':> [%s] | %s'%(db.title, db.post_src))
+      print(':> [%s] - %s'%(db.title, db.post_src))
       check_process(args.target)
       
-      if post_err:
-        logs.out("44", args.target, False)
+      if post_err: logs.out("44", args.target, False)
       
-  if not found:
-    logs.out("28", '(check)', True)
+  if not found: logs.out("28", '(check)', True)
     
 
 #========================#
diff --git a/src/var/lib/tyto/program/db.py b/src/var/lib/tyto/program/db.py
index a5f02d2..fd53e9f 100644
--- a/src/var/lib/tyto/program/db.py
+++ b/src/var/lib/tyto/program/db.py
@@ -33,10 +33,12 @@ target = args.target
 #---------
 domain_active     = False
 incomplete_domain = False
-db_exists         = False
-post_exists       = False
-db_remove         = False
-home_dir          = os.path.expanduser('~')
+db_exists = post_exists = db_remove = False
+no_chk    = no_wip      = no_www    = False
+old_chk   = old_wip     = old_www   = False
+file_wip  = file_www                = False
+sync_srvs = False
+home_dir  = os.path.expanduser('~')
 
 try:    in_dir = os.getcwd()
 except: logs.out("13", '', True)
@@ -189,7 +191,7 @@ if         args.target \
     post_exists = False
     logs.out("1", uri_file, False)
 
-  #
+  # Remove DB if unused source article or corrupted DB
   if db_exists:
     if not post_exists:
       db_remove = True
@@ -203,3 +205,24 @@ if         args.target \
     else:
       exec(open(post_db).read(),globals())
 
+  # Set Statuses for chk, wip, www
+  if db_exists:
+    # File exists on servers
+    if os.path.exists(post_wip): file_wip = True
+    if os.path.exists(post_www): file_www = True
+    
+    # Source article has changed
+    if hash_post != hash_chk: old_chk = True
+    
+    # WIP article is old
+    if hash_chk  != hash_wip: old_wip = True
+    
+    # WWW article is old
+    if hash_wip  != hash_www: old_www = True
+    
+    # Statuses not set in Db
+    if not hash_chk: no_chk = True
+    if not hash_wip: no_wip = True
+    if not hash_www: no_www = True
+    
+    if hash_chk == hash_wip == hash_www: sync_srvs = True
diff --git a/src/var/lib/tyto/program/domain.py b/src/var/lib/tyto/program/domain.py
index 04bf405..2b2c481 100644
--- a/src/var/lib/tyto/program/domain.py
+++ b/src/var/lib/tyto/program/domain.py
@@ -477,6 +477,28 @@ def create_domain(target):
   tyto.set_file(db.domain_conf, False, set_f)
   
   
+  # Show article source ?
+  #----------------------
+  try:
+    db.article_code
+    article_code = "True"
+  except:
+    article_code = "False"
+  
+  ask = ''
+  try:
+    ask = input(' ├ Show Article\'s code ? {%s} (Y/y) '%article_code)
+  except KeyboardInterrupt:
+    print('')
+    logs.out("255", '', True)  
+  
+  if ask in ['y', 'Y']:
+    article_code = "True"
+
+  set_f = 'article_code    = %s'%article_code
+  tyto.set_file(db.domain_conf, False, set_f)
+
+
   # Profile for rel="me" (optionnal)
   #---------------------------------
   try:    domain_relme = db.domain_relme
diff --git a/src/var/lib/tyto/program/html.py b/src/var/lib/tyto/program/html.py
index 4381bf5..f2f8727 100644
--- a/src/var/lib/tyto/program/html.py
+++ b/src/var/lib/tyto/program/html.py
@@ -179,10 +179,26 @@ def create_html_infos_section(process):
     date_new = date_pub.rsplit('-')
     date_pub = date_new[2] + '/' + date_new[1] + '/' + date_new[0] 
   
+  # Show source code in article-infos if True in DB
+  post_code = ''
+  if db.article_code:
+    # Set HTML
+    post_code = \
+    '<span id="article_code"> ' + \
+    '<a id="article_code_link" href="./%s" title="%s">{%s}</a>'%(
+     os.path.basename(db.short_src),
+     tyto.trans[21][tyto.n],
+     tyto.trans[3][tyto.n]
+     )
+    ' </span>'
+
+
   # in <article> > section info : line with new date
   post_pub = \
   '%s<p>'%(8 * ' ') + \
-  '<span id="article_title" title="%s">Article</span> '%(title) + \
+  '<span id="article_title" title="%s">Article%s</span> '%(title,
+                                                           post_code
+                                                          ) + \
   '<span title="%s %s">%s</span> '%(tyto.trans[10][tyto.n], 
                                     date[0],
                                     tyto.trans[7][tyto.n]
@@ -217,6 +233,13 @@ def create_sidebar(option):
   except:
     logs.out("1", 'Sidebar load file', True)
   
+  try:
+    db.sidebar_items
+    if int(db.sidebar_items) > 16: db.sidebar_items = 6
+  except:
+    db.sidebar_items = 6
+  
+  
   pub_opts = ('www', 'pub')
   if   option == 'wip': target = db.wip_sidebar
   elif option == 'www': target = db.www_sidebar
diff --git a/src/var/lib/tyto/program/infos.py b/src/var/lib/tyto/program/infos.py
index 09833ad..1b363b8 100644
--- a/src/var/lib/tyto/program/infos.py
+++ b/src/var/lib/tyto/program/infos.py
@@ -71,7 +71,9 @@ def tyto(target):
   '    wip     : Create page in wip server\n'
   '    publish : Create page in www server\n\n'
   '  - [target] according to [action]\n'
-  '    all     : Convert again all already converted articles\n'
+  '    all     : Convert all already converted articles\n'
+  '    again   : convert only converted articles and matches'
+  '    newer   : convert only newer articles'
   '    template: - Create navbar/sidebar/metas/footer in www server\n'
   '              - Copy wip/template/ files to www server\n'
   '    domain  : Create/Show config file\n'
diff --git a/src/var/lib/tyto/program/publish.py b/src/var/lib/tyto/program/publish.py
index ea447d6..4c7ef7e 100644
--- a/src/var/lib/tyto/program/publish.py
+++ b/src/var/lib/tyto/program/publish.py
@@ -34,6 +34,8 @@ def manage_publish(target):
   if target in args.pass_targets:
     do = {
           'all'      : publish_all,
+          'again'    : publish_all,
+          'newer'    : publish_all,
           'sidebar'  : html.create_sidebar,
           'navbar'   : html.create_navbar,
           'metas'    : html.create_user_metas,
@@ -158,18 +160,11 @@ def publish_template(option):
 # Option 'all' to publish again, based on DB #
 #--------------------------------------------#
 def publish_all(option):
-  ask = ''
-  try:
-    ask = input(" ├ Publish again all already published articles ? ")
-  except KeyboardInterrupt:
-    print('')
-    logs.out("255", '', True)
-  
-  if not ask in ['y', 'Y']:
-    logs.out("255", '', True)
+  tyto.process_all('Publish')
   
   # Load domain configuration DB
-  published = False
+  option = args.target
+  found = False
   for post_db in os.listdir(db.articles_db):
     if post_db.endswith('.conf'):
       # Load DB
@@ -178,21 +173,22 @@ def publish_all(option):
       args.target = post_src.rsplit('%s/'%db.in_dir)[1]
       importlib.reload(db)
 
-      # Conditions to publish
-      if not db.db_exists \
-         or not db.hash_www:
-        continue
+      if not db.hash_wip: continue    
+      if option == "again" and not db.sync_srvs: continue
+      if option == "newer" and not db.old_www:   continue
       
-      print(':> [%s] | %s'%(db.title, db.post_src))
-      hash_post = tyto.get_filesum(db.post_src, True) # From content file
+      print(':> [%s] - %s'%(db.title, db.post_src))
+      if db.old_chk:
+        logs.out("9", '', False)
+        continue
 
       check_to_publish('all')
       if err_pub: continue
       
-      published = True
+      found = True
       publish_article()
   
-  if not published:
+  if not found:
     logs.out("28", '(publish)', True)
   else:
     # Create new ATOM/RSS file
@@ -206,24 +202,22 @@ def check_to_publish(process):
   global err_pub
   err_pub = False
 
-  # Article must exists in wip server
-  if not os.path.exists(db.post_wip):
-    logs.out("1", db.post_wip, True)
-
-  # Checking if article can be publish
-  if not db.hash_chk:
+  # Article was not checked or changed
+  if db.no_chk:
     logs.out("25", db.uri_file, False)
     err_pub = 25
-  elif not db.hash_wip:
-    logs.out("30", db.uri_file, False)
-    err_pub = 30
-  elif db.hash_post != db.hash_chk:
+  elif db.old_chk:
     logs.out("9", db.uri_file, False)
     err_pub = 9
-  elif db.hash_wip != db.hash_chk:
+
+  # Article must exists in wip server
+  if db.no_wip or db.old_wip:
     logs.out("30", db.uri_file, False)
     err_pub = 30
-  
+  elif not db.file_wip:
+    logs.out("1", db.post_wip, False)
+    err_pub = 1
+
   if err_pub:
     if   process == 'all': return
     elif process == 'one': sys.exit(err_pub)
diff --git a/src/var/lib/tyto/program/rss.py b/src/var/lib/tyto/program/rss.py
index 992a0dd..8c20d4a 100644
--- a/src/var/lib/tyto/program/rss.py
+++ b/src/var/lib/tyto/program/rss.py
@@ -18,6 +18,8 @@
 #*********************************************************************
 
 import os
+from pathlib import Path
+
 import logs, db, tyto
 
 
@@ -50,13 +52,16 @@ def create_feed():
   '    <webMaster>%s</webMaster>\n'%db.domain_mail + \
   '    <generator>%s</generator>'%tyto.Tyto
   
+  # Sort by newer articles (created by last check)
+  db_articles = sorted(Path(db.articles_db).iterdir(), 
+                       key=os.path.getmtime,
+                       reverse=True
+                      )
   rss_item = False
   # Loop published articles. Get databases of articles
-  for post_db in os.listdir(db.articles_db):
-    if not post_db.endswith('.conf'): continue
-  
+  for post_db in db_articles:    
+    if not str(post_db).endswith('.conf'): continue
     # Load DB
-    post_db = '%s%s'%(db.articles_db, post_db)
     exec(open(post_db).read(),globals())
     if not hash_www or hash_chk != hash_www: continue
     
diff --git a/src/var/lib/tyto/program/stats.py b/src/var/lib/tyto/program/stats.py
index 2b18d20..dd5748f 100644
--- a/src/var/lib/tyto/program/stats.py
+++ b/src/var/lib/tyto/program/stats.py
@@ -62,7 +62,7 @@ def loop_articles(process):
   sti_articles = 0
   
   # Get databases of articles
-  for post_db in os.listdir(db.articles_db):
+  for post_db in os.listdir(db.articles_db):   
     if not post_db.endswith('.conf'): continue
     
     # Load DB
@@ -72,6 +72,7 @@ def loop_articles(process):
     importlib.reload(db)
     
     # Check hash status (wip/www)
+    hash_srv = ''
     if   process == 'wip': hash_srv = db.hash_wip
     elif process == 'www': hash_srv = db.hash_www
     
diff --git a/src/var/lib/tyto/program/tyto.py b/src/var/lib/tyto/program/tyto.py
index cc49be1..48ee311 100644
--- a/src/var/lib/tyto/program/tyto.py
+++ b/src/var/lib/tyto/program/tyto.py
@@ -20,7 +20,8 @@
 import os, sys, subprocess, locale, base64, datetime, shutil
 from hashlib import blake2b
 
-import db, logs
+import args, logs, db
+
 domain_active  = False
 home_dir       = os.path.expanduser('~')
 try:    in_dir = os.getcwd()
@@ -62,7 +63,8 @@ trans = [
 ('Conditions Générales d\'Utilisation', 'Terms of Use'), # 17
 ('Loi', 'Law'), # 18
 ('Articles de', 'Articles from'), # 19
-('Lire l\'article:', 'Read the article:') #20
+('Lire l\'article:', 'Read the article:'), #20
+('Voir le code source au format Tyto de cet article', 'See this article\'s source code in Tyto format') #21 (title)
 ]
 
 
@@ -112,7 +114,9 @@ words_tags = [
 tpl_tags = [
 ('(', ')'),
 ('[', ']'),
-('{', '}')
+('{', '}'),
+('«', '»'),
+('<!--', '-->')
 ]
 
 
@@ -444,7 +448,7 @@ def replace_in_db(post_db, process, hash_post):
 #===================================#
 # Copy files used by article to srv #
 #-----------------------------------#
-def files_to_srv(server):
+def files_to_srv(server):  
   for uri in db.uris:
     # Destination file to server
     f_src = '%s%s'%(db.domain_articles, uri)
@@ -464,3 +468,36 @@ def files_to_srv(server):
       logs.out("33", f_dst, False)
     except:
       logs.out('4', f_dst, True)
+  
+  if db.article_code:
+    if   server == "wip": base_srv = db.srv_wip
+    elif server == "www": base_srv = db.srv_www
+    
+    f_dst = "%s%s"%(base_srv, db.short_src)
+    try:
+      shutil.copy2(db.post_src, f_dst)
+      logs.out("33", f_dst, False)
+    except:
+      logs.out('4', f_dst, True)
+
+
+#
+#
+#
+def process_all(process):
+  #  if target == 'all':
+  if args.target == "all":
+    ask = ''
+    try:
+      ask = input(" ├ %s all already converted articles ? "%process)
+    except KeyboardInterrupt:
+      print('')
+      logs.out("255", '', True) 
+    
+    if not ask in ['y', 'Y']:
+      logs.out("255", '', True)
+  
+  elif args.target == "again":
+    print(' ├ %s again current matches articles'%process)
+  elif args.target == "newer":
+    print(' ├ %s newer articles'%process)
diff --git a/src/var/lib/tyto/program/wip.py b/src/var/lib/tyto/program/wip.py
index 8e2d464..f56e4ec 100644
--- a/src/var/lib/tyto/program/wip.py
+++ b/src/var/lib/tyto/program/wip.py
@@ -18,7 +18,7 @@
 #**********************************************************************
 
 import os, re, sys, shutil, importlib
-import logs, args, db, tyto, html, domain, stats
+import args, logs, db, tyto, html, domain, stats
 
 
 #=========================================#
@@ -37,6 +37,8 @@ def manage_wip(target):
   if target in args.pass_targets:
     do = {
           'all'     : wip_all,
+          'again'   : wip_all,
+          'newer'   : wip_all,
           'sidebar' : html.create_sidebar,
           'navbar'  : html.create_navbar,
           'metas'   : html.create_user_metas,
@@ -50,18 +52,17 @@ def manage_wip(target):
 
   # Per article with target
   #------------------------
+  # Exit with these conditions
   if not target:         logs.out("5", '', True)
   if not db.post_exists: sys.exit(1)
   if not db.db_exists:   logs.out("25", db.uri_file, True)
-  
-  # Exit if article has changed
-  if db.hash_chk != db.hash_post:
-    logs.out("9", db.uri_file, True)  
+  if db.old_chk:         logs.out("9", db.uri_file, True)  
 
-  # Compare and check file
   # Article has changed or wip file missing
-  if db.hash_wip != db.hash_chk or not os.path.exists(db.post_wip):
+  if db.old_wip or not db.file_wip:
     wip_article(db.uri_file)
+  
+  # wip is up-to-date: ask to wip again
   else:
     logs.out("19", db.date_wip, False)
     try:
@@ -78,14 +79,11 @@ def manage_wip(target):
 # Option 'all' to wip again, based on DB #
 #----------------------------------------#
 def wip_all(process):
-
-#  if target == 'all':
-  ask = ''
-  ask = input(" ├ Wip again all already converted articles ? ")
-  if not ask in ['y', 'Y']:
-    return
+  tyto.process_all('Wip')
   
   # Load domain configuration DB
+  option = args.target
+  found  = False
   for post_db in os.listdir(db.articles_db):
     if post_db.endswith('.conf'):
       # Load DB
@@ -93,15 +91,20 @@ def wip_all(process):
       exec(open(post_db).read(),globals())
       args.target = post_src.rsplit('%s/'%db.in_dir)[1]
       importlib.reload(db)
-      if not db.hash_wip:
+      
+      if not db.hash_wip: continue    
+      if option == "again" and db.old_wip: continue
+      if option == "newer" and not db.old_wip: continue
+      
+      print(':> [%s] - %s'%(db.title, db.post_src))
+      if db.old_chk:
+        logs.out("9", '', False)
         continue
       
-      print(':> [%s] | %s'%(db.title, db.post_src))
-      hash_post = tyto.get_filesum(db.post_src, True) # From content file
-      if hash_post != db.hash_chk:
-        logs.out("25", db.uri_file, False)
-        continue
+      found = True
       wip_article(db.post_src)
+  
+  if not found: logs.out("28", '(wip)', True)
 
 
 #===================#