added search engine optimization functionality

FUTURE.md

@@ -24,10 +24,6 @@ Demo between different models - stateless, webpage, windows
 
 - Training and documentation tool
 
-- Implement using long polling to optimize CLOG sites for use with
-  search engines. If not long polling at least initial output sent by
-  initial html request until first query.
-
 - Optionaly, for local apps, allow direct transport of CLOG data to webpage via apis
 remove need for websocket transport.
 

source/clog-connection.lisp

@@ -304,15 +304,19 @@ the default answer. (Private)"
 		     (host             "0.0.0.0")
 		     (port             8080)
 		     (boot-file        "/boot.html")
+		     (boot-function    nil)
 		     (static-boot-js   nil)
 		     (static-root      #P"./static-files/"))
-  "Initialize CLOG on a socket using HOST and PORT to serve BOOT-FILE as
+  "Initialize CLOG on a socket using HOST and PORT to serve BOOT-FILE
-the default route for '/' to establish web-socket connections and static files
+as the default route for '/' to establish web-socket connections and
-located at STATIC-ROOT. If BOOT-FILE is nil no initial clog-path's will be
+static files located at STATIC-ROOT. If BOOT-FILE is nil no initial
-setup, use clog-path to add. The on-connect-handler needs to indentify the
+clog-path's will be setup, use clog-path to add. The
-path by querying the browser. See PATH-NAME (in CLOG-LOCATION). If
+on-connect-handler needs to indentify the path by querying the
-static-boot-js is nil then boot.js is served from the file /js/boot.js
+browser. See PATH-NAME (in CLOG-LOCATION). If static-boot-js is nil
-instead of the compiled version."
+then boot.js is served from the file /js/boot.js instead of the
+compiled version. boot-function if set is called with the url and the
+contents of boot-file and its return value replaces the contents sent
+to the brower."
   (set-on-connect on-connect-handler)
   (when boot-file
     (set-clog-path "/" boot-file))
@@ -338,6 +342,10 @@ instead of the compiled version."
 			  (let ((page-data (make-string (file-length stream)))
 				(post-data))
 			    (read-sequence page-data stream)
+			    (when boot-function
+			      (setf page-data (funcall boot-function
+						       (getf env :path-info)
+						       page-data)))
 			    (when (search "multipart/form-data;"
 					  (getf env :content-type))
 			      (let ((id  (get-universal-time))
@@ -402,6 +410,7 @@ instead of the compiled version."
 ;;;;;;;;;;;;;;;;;;;
 
 (defun set-clog-path (path boot-file)
+  "Associate URL path to BOOT-FILE"
   (if boot-file
       (setf (gethash path *url-to-boot-file*)
 	    ;; Make clog-path into a relative path of

source/clog-system.lisp

@@ -51,6 +51,7 @@ the same as the clog directy this overides the relative paths used in them.")
        (host           "0.0.0.0")
        (port           8080)
        (boot-file      "/boot.html")
+       (boot-function  nil)
        (static-boot-js nil)       
        (static-root    (merge-pathnames "./static-files/"
 			  (asdf:system-source-directory :clog))))
@@ -64,7 +65,10 @@ is installed in ./static-files\" If the variable clog:*overide-static-root*
 is set STATIC-ROOT will be ignored. If BOOT-FILE is nil no default
 boot-file will be set for root path, i.e. /. If static-boot-js is t
 then boot.js is served from the file /js/boot.js instead of the
-compiled version."
+compiled version. boot-function if set is called with the url and the
+contents of boot-file and its return value replaces the contents sent
+to the brower, this allows adding content for search engine optimization,
+see tutorial 12 for an example."
   (when on-new-window-handler
     (set-on-new-window on-new-window-handler :path "/" :boot-file boot-file))
   (unless *clog-running*
@@ -76,6 +80,7 @@ compiled version."
 				:host           host
 				:port           port
 				:boot-file      boot-file
+				:boot-function  boot-function
 				:static-boot-js static-boot-js
 				:static-root    *static-root*)))
 

static-files/boot.html

@@ -8,8 +8,10 @@
       <meta name="viewport" content="width=device-width, initial-scale=1">
       <script src="/js/jquery.min.js" type="text/javascript"></script>
       <script src="/js/boot.js" type="text/javascript"></script>
+      <noscript><%= (@ meta) %></noscript>
    </HEAD>
 <BODY>
+  <noscript><%= (@ body) %></noscript>
 </BODY>
 <noscript>Your browser must support JavaScript and be HTML 5 compilant to see this site.</noscript>
 </HTML>

static-files/bootstrap.html

@@ -10,8 +10,10 @@
       <script src="/js/boot.js" type="text/javascript"></script>
       <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous">
       <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+      <noscript><%= (@ meta) %></noscript>
    </HEAD>
 <BODY>
+  <noscript><%= (@ body) %></noscript>
 </BODY>
 <noscript>Your browser must support JavaScript and be HTML 5 compilant to see this site.</noscript>
 </HTML>

static-files/debug.html

@@ -9,8 +9,10 @@
       <script src="/js/jquery.min.js" type="text/javascript"></script>
       <script>var clog_debug = true;</script>
       <script src="/js/boot.js" type="text/javascript"></script>
+      <noscript><%= (@ meta) %></noscript>
    </HEAD>
 <BODY>
+  <noscript><%= (@ body) %></noscript>
 </BODY>
 <noscript>Your browser must support JavaScript and be HTML 5 compilant to see this site.</noscript>
 </HTML>

templates/www/www/boot.html

@@ -8,8 +8,10 @@
       <meta name="viewport" content="width=device-width, initial-scale=1">
       <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
       <script src="/js/boot.js" type="text/javascript"></script>
+      <noscript><%= (@ meta) %></noscript>
    </HEAD>
 <BODY>
+  <noscript><%= (@ body) %></noscript>
 </BODY>
 <noscript>Your browser must support JavaScript and be HTML 5 compilant to see this site.</noscript>
 </HTML>

templates/www/www/bootstrap.html

@@ -10,8 +10,10 @@
       <script src="/js/boot.js" type="text/javascript"></script>
       <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous">
       <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
+      <noscript><%= (@ meta) %></noscript>
    </HEAD>
 <BODY>
+  <noscript><%= (@ body) %></noscript>
 </BODY>
 <noscript>Your browser must support JavaScript and be HTML 5 compilant to see this site.</noscript>
 </HTML>

tutorial/12-tutorial.lisp

@@ -48,13 +48,6 @@
   (create-div body :content "You are in on-page2")
   (log-console (window body) "A message in the browser's log"))
 
-(defun on-default (body)
-  (cond ((equalp (path-name (location body))
-		 "/tutorial/tut-11.html")
-	 (on-tutorial11 body))
-	(t
-	 (create-div body :content "No dice!"))))
-
 (defun on-tutorial11 (body)
   (let* ((form         (attach-as-child body "form1" :clog-type 'clog-form))
 	 (good-button  (attach-as-child body "button1id"))
@@ -80,9 +73,31 @@
     (set-on-click good-button 'on-click-good)
     (set-on-click scary-button 'on-click-scary))))
 
+(defun on-default (body)
+  (cond ((equalp (path-name (location body))
+		 "/tutorial/tut-11.html")
+	 (on-tutorial11 body))
+	(t
+	 (create-div body :content "No dice!"))))
+
+(defun add-search-optimizations (path content)
+  ;; The default boot.html that comes with CLOG has template
+  ;; markers inside of the meta section and body section
+  ;; that are set to be transparent to the user but show to
+  ;; search engines and text browser. This allows setting
+  ;; custom data for search engine optimizations which are
+  ;; aware of these type of dynamic sites.
+  (declare (ignore path))
+  (funcall (cl-template:compile-template content)
+	   (list :meta "<meta name='description' content='CLOG Tutorial 12'>"
+		 :body "Tutorial 12 for CLOG")))
+
 (defun start-tutorial ()
   "Start turtorial."
-  (initialize 'on-main)
+  ;; Setup the default route / to on-main
+  ;; :boot-function allows us to add or modify our boot-files content
+  ;; for search engine optimization
+  (initialize 'on-main :boot-function 'add-search-optimizations)
   ;; Navigating to http://127.0.0.1:8080/page1 executes on-page1
   (set-on-new-window 'on-page1 :path "/page1")
   ;; Navigating to http://127.0.0.1:8080/page1.html executes on-page1