[1.9.21] - 3 lines tags supported, cleaner code, see changelog

CHANGELOG.md

@@ -9,6 +9,16 @@ Tyto - Littérateur
 
 # CURRENTLY IN DEV !
 
+## [1.9.21]
+- new indentation (3 spaces)
+- added 'raw:' marker
+- (for wip process):
+- - added html titles to post database
+- - added html comments to post database (default: ';; a comment')
+- - added val3 tag as html comment to content, and convert content to base64
+- - - added values to post database
+- cleaner code
+
 ## [1.9.20]
 - working on 'check' process
 - - updated 'logo:' process

README.md

@@ -9,6 +9,11 @@ tyto
 
 ## ToDo next (working on)
 - 'check' action processes
-- create template post database
+- - support for words tags (bolds...)
+- - support lists, anchors
+- - thinking about creating an auto top article menu from titles
+- - stats for article words
+- manage template post database
+- - check valid database
 - Translate logs in english !
 

src/usr/bin/tyto

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-# Version: 1.9.20
+# Version: 1.9.21
-# Updated: 2023-10-06 1696580458
+# Updated: 2023-10-08 1696756865
 # Tyto - Littérateur
 
 # Copyright (C) 2023 Cyrille Louarn <echolib+tyto@a-lec.org>

src/var/lib/tyto/program/check.py

@@ -35,7 +35,7 @@
 import sys, os
 from dateutil.parser import parse
 
-import args, domain, langs, debug, post, tools
+import args, domain, langs, debug, post, tools, tyto
 
 
 #=====================================#
@@ -252,72 +252,36 @@ def is_valid_date(date):
 #============================#
 # multiple and optional Tags #
 # Written using 3 lines      #
+# Tags are stric start line  #
 #----------------------------#
 def ml_tags():
-  global value1
+   #global value1
-  
-  value1 = ""
-  R_tag = post.ml_tags[3] # raw:
-  C_tag = post.ml_tags[4] # code:
 
    c = 0 # Continue for next x lines, as tags are 3 lines values
+
    for ln, line in enumerate(headers):
       if c != 0:
          c -= 1
          continue
       
-      if not line or line.isspace() or line.startswith("#"): continue
+      if line.startswith(post.ml_tags):
+         tag = line.rsplit(":")[0] + ":"
+         post.ml_tags_stats[tag] = post.ml_tags_stats[tag] + 1
          
-      # link:
+         if not ml_tag_values(ln, tag, post.ml_tags_stats[tag]):
-      if line.startswith(post.ml_tags[0]):
-         post.stats_links += 1
-         if not ml_tag_values(ln, post.ml_tags[0], post.stats_links):
-            return False # value errors
-         
-         c = 2 ; continue
-      
-      # file:
-      elif line.startswith(post.ml_tags[2]):
-           post.stats_files += 1
-           if not ml_tag_values(ln, post.ml_tags[2], post.stats_files):
-              return False # value errors
-           
-           c = 2 ; continue
-      
-      # image:
-      elif line.startswith(post.ml_tags[1]):
-           post.stats_images += 1
-           if not ml_tag_values(ln, post.ml_tags[1], post.stats_images):
-              return False # value errors
-           # image: logo is reserved for post logo
-           if value1 == "logo":
-              debug.out(56, "%s) 'logo'"%(ln+1), post.uri, True, 2, False)
             return False
          
-           c = 2 ; continue
+         c = 2
-      
+         continue
-      # ABBRS:
-      elif line.startswith(post.ml_tags[5]):
-           post.stats_abbrs += 1
-           if not ml_tag_values(ln, post.ml_tags[5], post.stats_abbrs):
-              return False # value errors
-           
-           c = 2 ; continue
-
-
-      # value1 must not starts with "_", as used by Tyto
-      if value1.startswith("_"):
-         debug.out(56, "%s) '_...'"%(ln+1), post.uri, True, 2, False)
-         return False
 
    return True
 
 
 #========================================#
 # Get tag 3 lines values                 #
-# Check if 2nd, 3rd lines starts with a: # 
+# Check if 2nd, 3rd lines starts with:   # 
 # - tag, comment or are empty            #
-# Set globals value                      #
+# Create var and val for post database   #
 # Return True/False (if no value)        #
 #----------------------------------------#
 def ml_tag_values(ln, tag, stats):  
@@ -329,23 +293,40 @@ def ml_tag_values(ln, tag, stats):
    value3 = headers[ln+2].lstrip()
    
    # Check values (not empty or begins with a tag)
+   # value1
+   # ------
    if not value1:
       post.error = \
       debug.out(51, "%s) %s 1/3"%(ln+1, tag), post.uri, True, 2, False)
       return False
    
+   # value1 must not starts or contains:o
+   elif value1.startswith("_"):
+      debug.out(56, "%s) '_...'"%(ln+1), post.uri, True, 2, False)
+      return False
+   
+   # Specific for image: logo name is reserved
+   elif tag == post.ml_tags[1] and value1 == "logo":
+      debug.out(56, "%s) 'logo'"%(ln+1), post.uri, True, 2, False)
+      return False
+   
+   # value2
+   # ------
    if not value2 or value2.startswith(post.ml_tags):
       post.error = \
       debug.out(51, "%s) %s 2/3"%(ln+2, tag), post.uri, True, 2, False)
       return False
    
+   # value3
+   # ------
    if not value3 or value3.startswith(post.ml_tags):
       post.error = \
       debug.out(51, "%s) %s 3/3"%(ln+3, tag), post.uri, True, 2, False)
       return False
    
+   # No error with values
    # Convert value1 in header with tyto_value1 in text
-  tyto_value = post.ml_marks[tag] + value1
+   tyto_value = post.ml_tags_marks[tag] + value1
    
    # CHeck if value is in text and to stats or return error
    if post.text_contents.find(tyto_value) == -1:
@@ -354,14 +335,18 @@ def ml_tag_values(ln, tag, stats):
    
    # Check value2 link for some tags (file, image...)
    if tag in post.value2s_uri \
-     and not value2.startswith(post.value2s_ext_uris):
+      and not value2.startswith(post.value2s_ext_uris) \
-     if not is_value2_file_exists(ln+2, tag, value2):
+      and not is_value2_file_exists(ln+2, tag, value2):
       return False
 
-  # Convert values to HTML (put in post database)
+
+   #-----------------------------------------------#
+   # Convert values to HTML (put in post database) #
+   #-----------------------------------------------#
    link_var = "%s_%s"%(tag.replace(":", ""), stats)
    html_var = "html_%s"%stats
    
+   # link:
    if tag == post.ml_tags[0]:
       section = "LINKS"
       post.stats_text_links += post.text_contents.count(tyto_value)
@@ -369,6 +354,7 @@ def ml_tag_values(ln, tag, stats):
                     value2, css, "%%s", value3, value1
                     )
    
+   # file:
    elif tag == post.ml_tags[2]:
       section = "FILES"
       post.stats_text_files += post.text_contents.count(tyto_value)
@@ -376,6 +362,7 @@ def ml_tag_values(ln, tag, stats):
                     value2, css, "%%s", value3, value1
                     )
    
+   # image:
    elif tag == post.ml_tags[1]:
       section = "IMAGES"
       post.stats_text_images += post.text_contents.count(tyto_value)
@@ -383,6 +370,37 @@ def ml_tag_values(ln, tag, stats):
                     value2, "%%s", "%%s", value3, "%%s"
                     )
    
+   # raw: (content file converted to base64)
+   elif tag == post.ml_tags[3]:
+      section = "RAWS"
+      post.stats_text_raws += post.text_contents.count(tyto_value)
+      html_value = "<!-- %s -->"%value3
+      with open(value2_uri, "r") as f:
+         html_value = "%s\n%s"%(html_value, f.read())
+
+      html_value = tools.b64_convert("encode", html_value)
+
+   # code: (content file converted to HTML + base64)
+   elif tag == post.ml_tags[4]:
+      section = "CODES"
+      post.stats_text_codes += post.text_contents.count(tyto_value)
+      htmlbcode = "<!-- %s -->"%value3
+      with open(value2_uri, "r") as f:
+         for line in f.read().rsplit("\n"):
+            line = tools.convert_html_signs(line)
+            line = '<li class="%s_blockcode">%s</li>'%(domain.css, line)
+            htmlbcode = "%s\n%s"%(htmlbcode, line)
+      
+      html_value = tyto.pre_bcode%(
+                                   domain.css, domain.css, domain.css,
+                                   htmlbcode
+                                  )
+      print()
+      print(html_value)
+      print()
+      html_value = tools.b64_convert("encode", html_value)
+            
+   # abbr:
    elif tag == post.ml_tags[5]:
       section = "ABBRS"
       post.stats_text_abbrs += post.text_contents.count(tyto_value)
@@ -404,7 +422,7 @@ def ml_tag_values(ln, tag, stats):
 # - ...  for post directory      #
 #--------------------------------#
 def is_value2_file_exists(ln, tag, val2):
-  global value2, src_uri
+   global value2, src_uri, value2_uri
    
    # uri "@..." means generic folders
    if val2[0].startswith("@"):
@@ -412,30 +430,30 @@ def is_value2_file_exists(ln, tag, val2):
       else:              val2 = val2[1:]
      
       #Set directory for files
-     test_uri = os.path.join(domain.wrk_files + val2)
+      value2_uri = os.path.join(domain.wrk_files + val2)
       value2     = src_uri = os.path.join("/files", val2)
      
       # Set directory for images in /images
       if tag == post.ml_tags[1] or tag == post.logo[0]: # image:
-        test_uri = os.path.join(domain.wrk_images, val2)
+         value2_uri = os.path.join(domain.wrk_images, val2)
          value2     = src_uri = os.path.join("/images", val2)
    
    # uri "/..." means from wrk root folder
    elif val2[0].startswith("/"):
       val2 = val2[1:]
-       test_uri = os.path.join(domain.wrk_articles, val2)
+      value2_uri = os.path.join(domain.wrk_articles, val2)
       src_uri = val2
    
    # uri "..." means from legacy post folder
    else:
-       test_uri = os.path.dirname(post.uri) + "/" + val2
+      value2_uri = os.path.dirname(post.uri) + "/" + val2
       value2 = "./" + val2
-       src_uri = test_uri.rsplit("articles/")[1]
+      src_uri = value2_uri.rsplit("articles/")[1]
        
    # Check if file exists
-  if not os.path.exists(test_uri):
+   if not os.path.exists(value2_uri):
       post.error = \
-     debug.out(5, "%s) %s"%(ln, tag), test_uri, True, 2, False)
+      debug.out(5, "%s) %s"%(ln, tag), value2_uri, True, 2, False)
       return False
    
    # Add file to [SOURCE_FILES] post database
@@ -547,13 +565,11 @@ def sl_paired(markers):
 #============================#
 # Check optional title tags  #
 # Count tyto + html comments #
-# Add stat for logo          #
+# Add stat for _image:logo   #
 # Return True/False          #
 #----------------------------#
 def titles():
   for ln, line in enumerate(texts, post.head_lines + 1):    
-      if not line or line.isspace(): continue
-    
       # legacy Tyto Titles
       if line.startswith(post.tyto_titles):
          if not line[3:]:
@@ -569,6 +585,16 @@ def titles():
      
          post.stats_titles += 1
          
+         # Create html value for this title in database
+         link_var = "title_%s"%post.stats_titles
+         post.cf.set("TITLES", link_var, line)
+         
+         html_var = "html_%s"%post.stats_titles
+         mark  = line[0:2]
+         title = line[3:] 
+         html_val = post.html_titles[mark]%(domain.css, title)
+         post.cf.set("TITLES", html_var, html_val)
+
       # Count Tyto Comments
       elif line.lstrip().startswith("#"):
          post.stats_tyto_text_coms += 1
@@ -576,13 +602,21 @@ def titles():
       # Count HTML comments
       elif line.lstrip().startswith(post.text_comments):
          post.stats_html_coms += 1
+         # Convert tyto commented marker to HTML
+         if line.lstrip().startswith(post.text_comments[0]):
+            real_com = line.lstrip()[3:]
+            link_var = "comm_%s"%post.stats_html_coms
+            post.cf.set("COMMENTS", link_var, line.lstrip())
+            html_var = "html_%s"%post.stats_html_coms
+            post.cf.set("COMMENTS", html_var, '<!-- %s -->'%real_com)
+            
          
       # Add stat + html for [IMAGES] when user wants to show logo in post
       elif line.lstrip().startswith("_image:logo"):
          post.stats_text_images += 1
-           link_var = "image_%s"%(post.stats_images+1)
+         link_var = "image_%s"%(post.ml_tags_stats["image:"] + 1)
          link_val = "_image:logo"
-           html_var = "html_%s"%(post.stats_images+1)
+         html_var = "html_%s"%(post.ml_tags_stats["image:"] + 1)
          html_val = '<a href="%s" class="%s" target="%s" alt="%s">%s</a>'%(
                      post.logo[1], "%%s", "%%s", post.title[1], "%%s"
                      )
@@ -629,10 +663,12 @@ def cf_update_values():
    
    post.cf.set("STATS_HEADERS", "lines",     str(post.head_lines))
    post.cf.set("STATS_HEADERS", "tyto_coms", str(post.stats_tyto_head_coms))
-  post.cf.set("STATS_HEADERS", "links",     str(post.stats_links))
+   post.cf.set("STATS_HEADERS", "links",     str(post.ml_tags_stats["link:"]))
-  post.cf.set("STATS_HEADERS", "files",     str(post.stats_files))
+   post.cf.set("STATS_HEADERS", "files",     str(post.ml_tags_stats["file:"]))
-  post.cf.set("STATS_HEADERS", "images",    str(post.stats_images))
+   post.cf.set("STATS_HEADERS", "images",    str(post.ml_tags_stats["image:"]))
-  post.cf.set("STATS_HEADERS", "abbrs",     str(post.stats_abbrs))
+   post.cf.set("STATS_HEADERS", "abbrs",     str(post.ml_tags_stats["abbr:"]))
+   post.cf.set("STATS_HEADERS", "codes",     str(post.ml_tags_stats["code:"]))
+   post.cf.set("STATS_HEADERS", "raws",     str(post.ml_tags_stats["raw:"]))
    
    post.cf.set("STATS_TEXTS", "lines",     str(post.text_lines))
    post.cf.set("STATS_TEXTS", "tyto_coms", str(post.stats_tyto_text_coms))
@@ -642,6 +678,8 @@ def cf_update_values():
    post.cf.set("STATS_TEXTS", "files",     str(post.stats_text_files))
    post.cf.set("STATS_TEXTS", "images",    str(post.stats_text_images))
    post.cf.set("STATS_TEXTS", "abbrs",     str(post.stats_text_abbrs))
+   post.cf.set("STATS_TEXTS", "codes",     str(post.stats_text_codes))
+   post.cf.set("STATS_TEXTS", "raws",      str(post.stats_text_raws))
    
    with open(post.cf_uri, "w") as f:
       post.cf.write(f)

src/var/lib/tyto/program/debug.py

@@ -129,8 +129,7 @@ def out(nbr, var, val, show, color, stop):
    
    # Exit if stop = True
    if stop:
-    if nbr >= 200:
+      if nbr >= 200: nbr = 0
-      nbr = 0
       sys.exit(nbr)
    
    return nbr

src/var/lib/tyto/program/domain.py

@@ -552,7 +552,6 @@ def cf_update_values(write):
 # if activated, check/create wrk dirs    #
 #----------------------------------------#
 def userset_status(action):
-
    do = {
         "start" : "yes",
         "stop"  : "no"

src/var/lib/tyto/program/langs.py

@@ -52,7 +52,10 @@ def translation_exists(module, lang, out):
    global tr_file
    
    modules = ("logs", "website")
-  if not module in modules: return # in case of internal typo error
+   # in case of internal typo error
+   if not module in modules:
+      print("! langs: internal error: 'logs', 'website'")
+      sys.exit(254)
    
    tr_file = "%s%s_%s.py"%(trfs, module, lang)
    if not os.path.exists(tr_file):
@@ -134,3 +137,4 @@ def load_website_lang():
    except:
       debug.out(208, site_lang, tr_website_uri, False, 0, False)
       set_site = True
+   

src/var/lib/tyto/program/post.py

@@ -127,8 +127,7 @@ def cf_load():
    global cf
    cf = False
    
-  if not os.path.exists(cf_uri):
+   os.path.exists(cf_uri) or tools.create_file(cf_uri, ini_template)
-    tools.create_file(cf_uri, ini_template)
    
    cf = configparser.ConfigParser()
    cf.read(cf_uri)
@@ -150,6 +149,33 @@ def cf_valid():
 #======#
 # MAIN #=======================================================================
 #======#
+# Statistics
+# ==========
+stats_tyto_head_coms = 0
+stats_tyto_text_coms = 0
+stats_html_coms = 0
+stats_titles    = 0
+stats_bcodes    = 0
+stats_quotes    = 0
+stats_parags    = 0
+stats_links     = 0
+stats_images    = 0
+stats_files     = 0
+stats_raws      = 0
+stats_codes     = 0
+stats_abbrs     = 0
+stats_codes     = 0
+stats_raws      = 0
+
+stats_total_files = 0
+
+stats_text_links  = 0
+stats_text_files  = 0
+stats_text_images = 0
+stats_text_abbrs  = 0
+stats_text_codes  = 0
+stats_text_raws   = 0
+
 
 # head_contents
 #==============
@@ -171,15 +197,25 @@ logo  = ("logo:", False)
 
 # Multiple lines (3)
 ml_tags       = ("link:", "image:", "file:", "raw:", "code:", "abbr:")
-ml_marks = {
+ml_tags_marks = {
                  "link:"  : "__",
                  "file:"  : "--",
                  "image:" : "_image:",
-            "abbr:"  : "::"
+                 "abbr:"  : "::",
+                 "raw:"   : "_raw:",
+                 "code:"  : "_code:"
+                }
+ml_tags_stats = {
+                 "link:"  : stats_links,
+                 "file:"  : stats_files,
+                 "image:" : stats_images,
+                 "abbr:"  : stats_abbrs,
+                 "raw:"   : stats_raws,
+                 "code:"  : stats_codes,
                 }
 
 # Markers with uri in value2
-value2s_uri = (ml_tags[1], ml_tags[2], ml_tags[3])
+value2s_uri = (ml_tags[1], ml_tags[2], ml_tags[3], ml_tags[4])
 value2s_ext_uris = ("http", "ftp")
 
 # text_contents
@@ -194,41 +230,17 @@ raw_contents = ("bcodes", "quotes")
 
 # Comments
 text_comments = (";;", "<!--")
-html_comment = { text_comments[0] : "<!-- %s -->" }
 
 # Tyto Titles #1 = <h2>
 tyto_titles = ("#1", "#2", "#3", "#4", "#5")
 html_titles = {
-               "#1" : '<h2 class="title_2">%s</h62>',
+               "#1" : '<h2 class="%s">%s</h2>',
-               "#2" : '<h3 class="title_3">%s</h3>',
+               "#2" : '<h3 class="%s">%s</h3>',
-               "#3" : '<h4 class="title_4">%s</h4>',
+               "#3" : '<h4 class="%s">%s</h4>',
-               "#4" : '<h5 class="title_5">%s</h5>',
+               "#4" : '<h5 class="%s">%s</h5>',
-               "#5" : '<h6 class="title_6">%s</h6>',
+               "#5" : '<h6 class="%s">%s</h6>',
               }
 
-# Statistics
-# ==========
-stats_tyto_head_coms = 0
-stats_tyto_text_coms = 0
-stats_html_coms = 0
-stats_titles    = 0
-stats_bcodes    = 0
-stats_quotes    = 0
-stats_parags    = 0
-stats_links     = 0
-stats_images    = 0
-stats_files     = 0
-stats_raws      = 0
-stats_codes     = 0
-stats_abbrs     = 0
-
-stats_total_files = 0
-
-stats_text_links  = 0
-stats_text_files  = 0
-stats_text_images = 0
-stats_text_abbrs  = 0
-
 
 #=============================#
 # articles configuration file #
@@ -245,6 +257,10 @@ ini_template = """[DOMAIN]
 
 [WWW]
 
+[COMMENTS]
+
+[TITLES]
+
 [LINKS]
 
 [FILES]
@@ -264,5 +280,4 @@ ini_template = """[DOMAIN]
 [STATS_HEADERS]
 
 [STATS_TEXTS]
- 
 """

src/var/lib/tyto/program/tools.py

@@ -33,7 +33,7 @@
 #--------------------------
 
 from hashlib import blake2b
-import sys, os, configparser, datetime, time
+import sys, os, configparser, datetime, time, base64
 import debug, domain
 
 
@@ -155,3 +155,39 @@ def update_ini_file(file_path, section, key, val):
    with open(file_path, "w") as f:
       config.write(f)
 
+
+#====================#
+# Base64 Convertions #
+#--------------------#
+def b64_convert(action, content):
+   if action == 'encode':
+      global b64_content
+      
+      b64_base64    = ''
+      content_bytes = content.encode("utf8")
+      base64_bytes  = base64.b64encode(content_bytes)
+      b64_content   = base64_bytes.decode("utf8")
+      return b64_content
+    
+   elif action == 'decode':
+      global src_content
+      
+      src_content   = ''
+      content_bytes = content.encode("utf8")
+      base64_bytes  = base64.b64decode(content_bytes)
+      src_content   = base64_bytes.decode("utf8")
+      return src_content
+
+
+#================================#
+# Convert html sign in string    #
+# used to deactivate HTML markup #
+# Return new string              #
+#--------------------------------#
+def convert_html_signs(string):
+   string = string.replace('<', '&lt;')
+   string = string.replace('>', '&gt;')
+   astring = string.replace('"', '&quot;')
+   string = string.replace("'", '&apos;')
+
+   return string

src/var/lib/tyto/program/tyto.py

@@ -93,3 +93,14 @@ ini_domains_list = """[DOMAINS]
 
 # Put here values where posts target cannot begin with
 notarget = ("./", "../")
+
+#======#
+# HTML #=======================================================================
+#======#
+pre_bcode = """<pre class="%s_blockcode">
+<code class="%s_blockcode">
+<ol class="%s_blockcode">
+%s
+</ol>
+</code>
+</pre>"""

src/var/lib/tyto/program/userset.py

@@ -58,4 +58,3 @@ def manage(action, target):
    elif action in ("start", "stop") and target == "domain":
       domain.userset_status(action)
   
-