[1.9.16] - Working on 'check' process and post database

CHANGELOG.md

@@ -9,6 +9,12 @@ Tyto - Littérateur
 
 # CURRENTLY IN DEV !
 
+## [1.9.16]
+- working on 'check' process
+- - dev: post database values (lots are missing)
+- - added links, files supports
+- - testing some new markers
+
 ## [1.9.15]
 - Added 'check' process for bcodes, quotes and paragraphs + stats
 - - Their contents must be indented

src/usr/bin/tyto

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-# Version: 1.9.15
-# Updated: 2023-10-02 1696231003
+# Version: 1.9.16
+# Updated: 2023-10-03 1696344805
 # Tyto - Littérateur
 
 # Copyright (C) 2023 Cyrille Louarn <echolib+tyto@a-lec.org>

src/var/lib/tyto/program/check.py

@@ -73,12 +73,19 @@ def manage(action, target):
 def is_article(target):
   valid(target)
 
+  # When all is OK
+  # Will create post database, but now, show some values
   print("chk_date", chk_date)
   print()
   print("Final text_contents string")
   for ln, line in enumerate(post.text_contents.rsplit("\n"), post.head_lines):
     print(">", ln, line)
 
+  print()
+  # Write to post database
+  cf_update_values()
+  
+
 #===========================================#
 # Check full article contents (head + text) #
 # In error case, exit or return if targetS  #
@@ -89,16 +96,32 @@ def valid(target):
   # Target is a tyto article format
   post.is_article(target) or tools.exit(targets, post.error)
   
+  global css
+  css = domain.cf.get("WEBSITE", "css")
+  
+  global headers, texts
+  headers = post.head_contents.rsplit("\n")
+  texts = post.text_contents.rsplit("\n")
+  
+  # =============
   # Head contents
   # =============
+
   # One Line targs in head_contents
   post.error == 0 and ol_tags() or tools.exit(targets, post.error)
+  #Multiple and optional Tags on 3 linges
+  post.error == 0 and ml_tags() or tools.exit(targets, post.error)
   
+  # ============
   # Text article
   # ============
+  # Start Lines
+  # -----------
+  # Paired tags.
   post.error == 0 and sl_paired("bcodes") or tools.exit(targets, post.error)
   post.error == 0 and sl_paired("quotes") or tools.exit(targets, post.error)
   post.error == 0 and sl_paired("parags") or tools.exit(targets, post.error)
+  # Single tags
   post.error == 0 and titles() or tools.exit(targets, post.error)
 
 
@@ -110,7 +133,7 @@ def multiple_targets():
   
 
 #=====================#
-# check head contents #===============================================================
+# check head contents #========================================================
 #=====================#
 #======================#
 # One Line needed tags #
@@ -119,8 +142,12 @@ def ol_tags():
   global sitemap
   
   sitemap = "True"
-  for ln, line in enumerate(post.head_contents.rsplit("\n"), 1):
-    if not line or line.isspace() or line.startswith("#"): continue
+  for ln, line in enumerate(headers, 1):
+    
+    if not line or line.isspace(): continue
+    if line.startswith("#"):
+      post.stats_tyto_head_coms += 1
+      continue
     
     # One Line tags (Must be set)
     # ===========================
@@ -205,19 +232,118 @@ def is_valid_date(date):
     return False
 
 
+#============================#
+# multiple and optional Tags #
+# Written using 3 lines      #
+#----------------------------#
+def ml_tags():
+  I_tag = post.ml_tags[1] # image:
+  R_tag = post.ml_tags[3] # raw:
+  C_tag = post.ml_tags[4] # code:
+  A_tag = post.ml_tags[5] # abbr:
+  
+  c = 0 # Continue for next x lines, as tags are 3 lines values
+  for ln, line in enumerate(headers):
+    if c != 0:
+       c -= 1
+       continue
+
+    if not line or line.isspace() or line.startswith("#"): continue
+
+    if line.startswith(post.ml_tags[0]):
+       post.stats_links += 1
+       if not ml_tag_values(ln, post.ml_tags[0], post.stats_links):
+          return False # value errors
+       
+       c = 2 ; continue
+    
+    elif line.startswith(post.ml_tags[2]):
+       post.stats_files += 1
+       if not ml_tag_values(ln, post.ml_tags[2], post.stats_files):
+          return False # value errors
+       
+       c = 2 ; continue
+    
+       
+  return True
+
+
+#========================================#
+# Get tag 3 lines values                 #
+# Check if 2nd, 3rd lines starts with a: # 
+# - tag, comment or are empty            #
+# Set globals value                      #
+# Return True/False (if no value)        #
+#----------------------------------------#
+def ml_tag_values(ln, tag, stats):  
+  # Get 3 lines values
+  value1 = headers[ln].rsplit(":")[1].lstrip()
+  value2 = headers[ln+1].lstrip()
+  value3 = headers[ln+2].lstrip()
+  
+  # Check values (not empty or begins with a tag)
+  if not value1:
+     post.error = \
+     debug.out(51, "%s) %s 1/3"%(ln+1, tag), post.uri, True, 2, False)
+     return False
+  
+  if not value2 or value2.startswith(post.ml_tags):
+     post.error = \
+     debug.out(51, "%s) %s 2/3"%(ln+2, tag), post.uri, True, 2, False)
+     return False
+  
+  if not value3 or value3.startswith(post.ml_tags):
+     post.error = \
+     debug.out(51, "%s) %s 3/3"%(ln+3, tag), post.uri, True, 2, False)
+     return False
+  
+  # Convert value1 in header with tyto_value1 in text
+  tyto_value = post.ml_marks[tag] + value1
+  
+  # CHeck if value is in text and to stats or return error
+  if post.text_contents.find(tyto_value) == -1:
+     post_error = debug.out(51, tyto_value, post.uri, True, 2, False)
+     return False
+  
+  # Convert values to HTML (put in post database)
+  html_value = '<a href="%s" class="%s" target="%s" alt="%s">%s</a>'%(
+                value2, css, "%%s", value3, value1
+                )
+  
+  link_var = "%s_%s"%(tag.replace(":", ""), stats)
+  html_var = "html_%s"%stats
+
+  if   tag == post.ml_tags[0]:
+       section = "LINKS"
+       post.stats_text_links += post.text_contents.count(tyto_value)
+  
+  elif tag == post.ml_tags[2]:
+       section = "FILES"
+       post.stats_text_files += post.text_contents.count(tyto_value)
+  
+
+  post.cf.set(section, link_var, tyto_value)
+  post.cf.set(section, html_var, html_value)
+  
+  return True
+
+
 #=====================#
 # check text contents #========================================================
 #=====================#
-#=======================================#
+#========================================#
 # First process !                        #
+# Used for bvodes, quotes, parags        #
 # Check if opened and closed tags match  #
-# Check if bcodes contents are indented #
-# Count bcodes for stats                #
-# Remove bcodes lines (for next steps)  #
-# Create new post.text_contents         #
+# Check if contents in tags are indented #
+# Count tags for stats                   #
+# Remove bcodes, quotes lines            #
+#  - Create new post.text_contents       #
 # Return True/False                      #
-#---------------------------------------#
+#----------------------------------------#
 def sl_paired(markers):
+  global texts
+  
   new_text_contents = ""
   opened = closed = in_tag = False
   stats_opened = stats_closed = 0
@@ -228,7 +354,7 @@ def sl_paired(markers):
   elif markers == "parags" : tags = post.parags
 
   # loop lines in text_contents
-  for ln, line in enumerate(post.text_contents.rsplit("\n"), post.head_lines):
+  for ln, line in enumerate(texts, post.head_lines + 1):
     
     # Tag was closed, but not in_tag content line
     if closed and in_tag:
@@ -240,10 +366,23 @@ def sl_paired(markers):
       if opened:
          post.error = \
          debug.out(53, "%s) %s ... %s"%(
-                        ln,
+                       ln+1,
                        tags[0], tags[1]
                        ), post.uri, True, 2, False)
          return False
+      
+      # check next line if exists or is a closed tag
+      try:
+        next_line = post.contents.rsplit("\n")[ln]
+        if next_line.startswith(tags[1]):
+           post.error = \
+           debug.out(55, "%s) '%s'"%(ln, tags[0]), post.uri, True, 2, False)
+           return False
+      except:
+        post.error = \
+        debug.out(55, "%s) '%s'"%(ln, tags[0]), post.uri, True, 2, False)
+        return False
+              
       opened = in_tag = True
       closed = False
       stats_opened += 1
@@ -254,10 +393,11 @@ def sl_paired(markers):
       if closed:
          post.error = \
          debug.out(53, "%s) %s ... %s"%(
-                        ln,
+                       ln+1,
                        tags[0], tags[1]
                        ), post.uri, True, 2, False)
          return False
+      
       closed = True
       opened = False
       stats_closed += 1
@@ -268,7 +408,7 @@ def sl_paired(markers):
       if not line.startswith(tags):
         if len(line) - len(line.lstrip()) < 3:
            post.error = \
-          debug.out(54, "%s) %s..."%(ln, line[0:10]), post.uri, True, 2, False)
+           debug.out(54, "%s) '\t'%s..."%(ln+1, line[0:10]), post.uri, True, 2, False)
            return False
       line = ""
 
@@ -277,7 +417,10 @@ def sl_paired(markers):
       if not new_text_contents: new_text_contents = line
       else:  new_text_contents = "%s\n%s"%(new_text_contents, line)
   
-      post.text_contents = new_text_contents
+      texts = new_text_contents.rsplit("\n")
+  
+  # Create post.stats  
+  post.cf.set("STATS_TEXTS", markers, str(stats_opened))
 
   return True
 
@@ -288,10 +431,10 @@ def sl_paired(markers):
 # Return True/False          #
 #----------------------------#
 def titles():
-  for ln, line in enumerate(post.text_contents.rsplit("\n"),
-                            post.head_lines + 1):
-    if not line or line.isspace():
-      continue
+  for ln, line in enumerate(texts, post.head_lines + 1):
+    print(">>", ln, line)
+    
+    if not line or line.isspace(): continue
     
     # legacy Tyto Titles
     if line.startswith(post.tyto_titles):
@@ -299,7 +442,6 @@ def titles():
          post.error = \
          debug.out(52, "%s. %s ?"%(ln, line), post.uri, True, 2, False)
          return False
-      post.stats_titles += 1
     
       # Avoic wanting #6 - #9 (but accept #1x.. #5x.. as comments...)
       elif line[1].isdigit() and int(line[1]) >= 6:
@@ -307,12 +449,66 @@ def titles():
         debug.out(52, "%s) %s..."%(ln, line[0:10]), post.uri, True, 1, False)
         return False
      
+      post.stats_titles += 1
+
     # Count Tyto Comments
     elif line.startswith("#"):
-      post.stats_tyto_coms += 1
+      print("?? tyto_coms", line)
+      post.stats_tyto_text_coms += 1
     
     # Count HTML comments
     elif line.startswith(post.text_comments):
+      print("?? html_coms", line)
       post.stats_html_coms += 1
 
   return True
+
+
+#================================#
+# Update post configuration file #
+#--------------------------------#
+def cf_update_values():
+  post.cf.set("DOMAIN", "name", domain.name)
+  
+  post.cf.set("FILE", "id", post.uri_id)
+  post.cf.set("FILE", "uri", post.uri)
+  post.cf.set("FILE", "db", post.cf_uri)
+  post.cf.set("FILE", "target", post.wrk_target)
+  
+  post.cf.set("HEADERS", "title", post.title[1])
+  post.cf.set("HEADERS", "about", post.about[1])
+  post.cf.set("HEADERS", "date", post.date[1])
+  post.cf.set("HEADERS", "tags", post.tags[1])
+  post.cf.set("HEADERS", "authors", post.author[1])
+  post.cf.set("HEADERS", "sitemap", str(sitemap))
+  post.cf.set("HEADERS", "static", str(domain.static))
+
+  post.cf.set("CHECK", "hash", post.wrk_id)
+  post.cf.set("CHECK", "date", chk_date)
+
+  post.cf.set("WIP", "web", "%s%s"%(domain.wip_url, post.wrk_target))
+  post.cf.set("WIP", "uri", "%s%s"%(domain.wip, post.wrk_target))
+
+  post.cf.set("WWW", "web", "%s%s"%(domain.www_url, post.wrk_target))
+  post.cf.set("WWW", "uri", "%s%s"%(domain.www, post.wrk_target))
+  
+  stats_tyto_all_coms = post.stats_tyto_text_coms + post.stats_tyto_head_coms
+  post.cf.set("STATS_FILE", "lines", str(post.lines))
+  post.cf.set("STATS_FILE", "tyto_coms", str(stats_tyto_all_coms))
+  
+  post.cf.set("STATS_HEADERS", "lines", str(post.head_lines))
+  post.cf.set("STATS_HEADERS", "links", str(post.stats_links))
+  post.cf.set("STATS_HEADERS", "files", str(post.stats_files))
+  post.cf.set("STATS_HEADERS", "tyto_coms", str(post.stats_tyto_head_coms))
+  
+  post.cf.set("STATS_TEXTS", "lines", str(post.text_lines))
+  post.cf.set("STATS_TEXTS", "tyto_coms", str(post.stats_tyto_text_coms))
+  post.cf.set("STATS_TEXTS", "html_coms", str(post.stats_html_coms))
+  post.cf.set("STATS_TEXTS", "titles", str(post.stats_titles))
+  post.cf.set("STATS_TEXTS", "links", str(post.stats_text_links))
+  post.cf.set("STATS_TEXTS", "files", str(post.stats_text_files))
+  
+  with open(post.cf_uri, "w") as f:
+    post.cf.write(f)
+  
+  

src/var/lib/tyto/program/debug.py

@@ -93,6 +93,7 @@ def out(nbr, var, val, show, color, stop):
     52  : langs.logs.err_post_title,
     53  : langs.logs.err_post_paired,
     54  : langs.logs.err_post_indent,
+    55  : langs.logs.err_post_in_tag,
     # WARNINGS (100-200)
     100 : langs.logs.warn_no_dom,
     101 : langs.logs.domain_created,

src/var/lib/tyto/program/domain.py

@@ -292,10 +292,10 @@ def cf_update_values(write):
 
   # [WEBSITE]
   # =========
-  global www_url, www_url, lang, css, sep, article_code, static
-  www_url = cf_set_value("WEBSITE", "www_url", "")
-  if not www_url:
-     www_url = cf_set_value("WEBSITE", "www_url", create_www_url())
+  global wip_url, www_url, lang, css, sep, article_code, static
+  wip_url = cf_set_value("WEBSITE", "wip_url", "")
+  if not wip_url:
+     wip_url = cf_set_value("WEBSITE", "wip_url", create_wip_url())
   
   www_url = cf_set_value("WEBSITE", "www_url", "")
   if not www_url:
@@ -394,7 +394,7 @@ def cf_update_values(write):
 
   # [USER_MODULES_FILES]
   # ====================
-  global wri_metas, wrk_header, wrk_navbar, wrk_sidebar, wrk_footer
+  global wrk_metas, wrk_header, wrk_navbar, wrk_sidebar, wrk_footer
   wrk_metas = os.path.join(wrk_mods, "tyto_metas.raw")
   cf_set_value("USER_MODULES_FILES", "metas", wrk_metas)
   

src/var/lib/tyto/program/post.py

@@ -32,7 +32,7 @@
 # file program  : 
 #--------------------------
 
-import os, configparser
+import os, sys, configparser
 import domain, debug, tools, tyto
 
 
@@ -78,6 +78,8 @@ def is_article(target):
   wrk_target = uri.rsplit(domain.wrk_articles)[1]
   
   # Load Database
+  global db
+  db = False
   db = cf_load() # True or False
   
   return True
@@ -88,23 +90,17 @@ def is_article(target):
 # Return True or False                    #
 #-----------------------------------------#
 def is_tyto_format():
-  global head_contents, text_contents
+  global head_contents, text_contents, contents
   global head_lines, text_lines, lines
 
-  separator = False
   head_contents = text_contents = ""
 
   with open(uri, "r") as contents:
     contents = contents.read()
-    for lines, line in enumerate(contents.rsplit("\n"), 1):      
-      if line.startswith(sep):
-        separator = True
-        continue
-      
-      if separator: text_contents = "%s%s\n"%(text_contents, line)
-      else:         head_contents = "%s%s\n"%(head_contents, line) 
-  
-  if not separator:
+    try:
+      head_contents = contents.rsplit(sep)[0]
+      text_contents = contents.rsplit(sep)[1]
+    except:
       error = debug.out(21, sep, uri, True, 2, False)
       return False
   
@@ -116,8 +112,9 @@ def is_tyto_format():
     error = debug.out(23, "?", uri, True, 2, False)
     return False
   
-  head_lines = len(head_contents.rsplit("\n")) +1 # after separator
-  text_lines = lines - head_lines
+  lines      = len(contents.splitlines())
+  head_lines = len(head_contents.splitlines())
+  text_lines = len(text_contents.splitlines())
 
   return True
 
@@ -131,7 +128,7 @@ def cf_load():
   cf = False
 
   if not os.path.exists(cf_uri):
-    return False
+    tools.create_file(cf_uri, ini_template)
 
   cf = configparser.ConfigParser()
   cf.read(cf_uri)
@@ -171,6 +168,16 @@ about  = ("about:", False)
 date   = ("date:", False)
 tags   = ("tags:", False)
 author = ("author:", False)
+# optional
+snpic  = ("snpic:", False)
+
+# Multiple lines (3)
+ml_tags  = ("link:", "image:", "file:", "raw:", "code:", "abbr:")
+ml_marks = {
+            "link:" : "__",
+            "file:" : "--",
+            "abbr:" : "::"
+           }
 
 
 # text_contents
@@ -199,9 +206,61 @@ html_titles = {
 
 # Statistics
 # ==========
-stats_tyto_coms = 0
+stats_tyto_head_coms = 0
+stats_tyto_text_coms = 0
 stats_html_coms = 0
 stats_titles    = 0
 stats_bcodes    = 0
 stats_quotes    = 0
 stats_parags    = 0
+stats_links     = 0
+stats_images    = 0
+stats_files     = 0
+stats_raws      = 0
+stats_codes     = 0
+stats_abbrs     = 0
+
+stats_text_links = 0
+stats_text_files = 0
+
+
+
+#=============================#
+# articles configuration file #
+#-----------------------------#
+ini_template = """[DOMAIN]
+
+[FILE]
+
+[HEADERS]
+static =
+snpic =
+
+[CHECK]
+static
+
+[WIP]
+static =
+
+[WWW]
+static =
+
+[LINKS]
+
+[FILES]
+
+[IMAGES]
+
+[RAWS]
+
+[CODES]
+
+[ABBRS]
+
+[STATS_FILE]
+
+[STATS_HEADERS]
+
+[STATS_TEXTS]
+ 
+"""

src/var/lib/tyto/program/tyto.py

@@ -90,54 +90,6 @@ root =
 ini_domains_list = """[DOMAINS]
 """
 
-#=============================#
-# articles configuration file #
-#-----------------------------#
-ini_article = """[DOMAIN]
-name =
-
-[FILE]
-id = 
-uri = 
-db =
-target =
-src_link =
-
-
-[CONTENTS]
-title =
-about = 
-date =
-local_date =
-tags =
-authors =
-snpic =
-sitemap =
-
-[CHECK]
-hash =
-date =
-epoch =
-
-[WIP]
-hash = 
-date =
-epoch =
-static =
-uri =
-web = 
-
-[WWW]
-hash = 
-date =
-epoch =
-static =
-uri =
-web = 
-
-[STATS]
-lines =
-"""
 
 # Put here values where posts target cannot begin with
 notarget = ("./", "../")

src/var/lib/tyto/translations/logs_fr.py

@@ -64,6 +64,7 @@ err_ini_file    = "Configuration invalide"
 err_post_data   = "Donnée manquante"
 err_post_title  = "Titre invalide"
 err_post_paired = "Marqueurs non apairés"
+err_post_in_tag = "Marqueurs sans contenu"
 err_post_indent = "Ligne non indentée"
 
 # Warnings