fix bugs in modify and publish

file-writing bugs caused problems on file-read depending on wether print or display
was used. publish was not overwriting existing feed.

.dev-log

@@ -1,3 +1,9 @@
+2024/01/17
+- add html cache directories to build script
+TODO:
+  - putting content in atom will require changing csv parsing;
+    that's for v2
+
 2024/01/15
 - cache static logic
 - START HERE:

.woodpecker/deploy.sh

@@ -13,12 +13,17 @@ mv static oxaliq/.
 # add html cache directories
 echo "ADDING HTML CACHE DIRECTORIES"
 mkdir fragment
-mkdir fragment/settled
-mkdir fragment/unsettled
+mkdir settled
+mkdir unsettled
+mkdir tagged
+cp -r settled fragment/.
+cp -r unsettled fragment/.
+cp -r tagged fragment/.
 mv fragment oxaliq/static/.
 mkdir page
-mkdir page/settled
-mkdir page/unsettled
+mv settled page/.
+mv unsettled page/.
+mv tagged page/.
 mv page oxaliq/static/.
 
 # put them on the deployment target

modify.rkt

@@ -8,7 +8,7 @@
          (only-in racket/function thunk identity)
          (only-in csv-reading csv->list)
          (only-in xml read-xml)
-         (only-in "./tool/utils.rkt" list->csv resource-link atom-table-entry add-atom-entry
+         (only-in "./utils.rkt" list->csv resource-link atom-table-entry add-atom-entry
                   make-feed archive-file))
 
 (define-namespace-anchor anc)
@@ -80,7 +80,7 @@
    ;; (. -> . input-port? output-port?)
    (define (get-source-ports)
      (let ([file-handle  (path-add-extension
-       (if (eq? r-type "root")
+       (if (equal? r-type "root")
            (if x-test
                (build-path "publish-test" "source" l-head)
                (build-path "source" l-head))
@@ -212,8 +212,8 @@
 
    ;; write-to-file must manually truncate since out-ports do not truncate
    ;; (output-port? xexpr? xexpr? rb-thunks? . -> . rb-thunks?)
-   (define (write-to-file out-port new-content old-content accumulator)
-     (let ([rb-thunks (cons (thunk (write-to-file-with-retries out-port old-content))
+   (define (write-to-file out-port new-content old-content accumulator #:print-mode? (is-print #f))
+     (let ([rb-thunks (cons (thunk (write-to-file-with-retries out-port old-content is-print))
                             (hash-ref accumulator 'rb-thunks))])
        (if (port-try-file-lock? out-port 'exclusive)
            ;; wrap in handler that raises file-write-exception
@@ -225,7 +225,7 @@
                                              rb-thunks)))
               (thunk
                (file-truncate out-port 0)
-               (display new-content out-port)
+               ((if is-print print display) new-content out-port)
                (port-file-unlock out-port)
                (close-output-port out-port)))
              (hash-update accumulator 'rb-thunks
@@ -235,13 +235,13 @@
                    (current-continuation-marks)
                    rb-thunks)))))
 
-   (define (write-to-file-with-retries out-port content)
+   (define (write-to-file-with-retries out-port content is-print)
      (with-handlers
          ([file-write-exception? (lambda (expn)
                                    (displayln "filewrite failed with" expn)
                                    (displayln "retrying")
-                                   (write-to-file-with-retries out-port content))])
-       (write-to-file out-port content)))
+                                   (write-to-file-with-retries out-port content is-print))])
+       (write-to-file out-port content #:print-mode? is-print)))
    ;; ------------
     
    ;; resource-replacement
@@ -277,7 +277,7 @@
           (write-to-file
            source-output-port
            (make-new-content res-table)
-           old-content accumulator)
+           old-content accumulator #:print-mode? #t)
           'res-table)
          'resource
          identity
@@ -327,7 +327,7 @@
                 [desc (third res-row)]
                 [publish-time (hash-ref accumulator 'publish-time)]
                 [new-table (add-atom-entry old-table
-                                       (atom-table-entry l-head res-link desc publish-time update-time))])
+                                       (atom-table-entry l-head res-link desc publish-time #:update-time update-time))])
            (hash-remove
             (hash-remove
              (hash-update
@@ -390,13 +390,8 @@
         (compose-accumulator rollback-thunks))
 
        ;; only archive after everything else is done
-       (archive-file n-in x-test)
+       (archive-file n-in x-test #:modify-mode #t)
 
        (displayln "modify was successful")))))
 
-;(run modify)
-
-(run modify #("-i" "in-progress/now.txt"
-              "-l" "now"
-              "-r" "root"
-              "-x"))
+(run modify)

publish.rkt

@@ -5,7 +5,7 @@
          (only-in racket/format ~a)
          (only-in racket/list append* first second third fourth fifth rest flatten add-between take)
          (only-in xml xexpr->string)
-         (only-in "./tool/utils.rkt" homepage list->csv add-atom-entry archive-file)
+         (only-in "./utils.rkt" homepage list->csv add-atom-entry atom-table-entry make-feed archive-file)
          csv-reading)
 
 (define-namespace-anchor anc)
@@ -254,6 +254,7 @@
 
 
    ;; update tag table
+   (displayln "updating tag table")
    (define tag-table (csv->list (open-input-file
                                   (if x "publish-test/data/tagged.csv" "data/tagged.csv"))))
    (define new-tag-table (write-new-tag-table tag-table t r res-id))
@@ -263,11 +264,11 @@
        (if x
            (write-new-tagged/index new-tag-table #:test #t)
            (write-new-tagged/index new-tag-table)))
-   ; (displayln new-tag-table)
+   
    (write-csv-to-file (list->csv new-tag-table)
                       (if x "publish-test/data/tagged.csv" "data/tagged.csv"))
 
-
+   (displayln "updating tagged/'tag")
    ;; update tagged/'tag for each tag
    (for-each
        (lambda (tt-row)
@@ -282,19 +283,23 @@
                '())))
        new-tag-table)
    
-
+   (displayln "writing resource to file")
    ;; write to file
    (if (port-try-file-lock? out 'exclusive)
        (begin
-         (write resource out)
+         (print resource out)
          (port-file-unlock out)
          (close-output-port out))
        (error "couldn't obtain file lock on ~a" out))
 
    ;; update 'resource/index
-   (write-new-resource/index r new-res-table #:test (if x #t #f))
-
+   (if (equal? r "root")
+       (displayln "skipping resource/index")
+       (begin
+         (displayln "updating resource/index")
+         (write-new-resource/index r new-res-table #:test (if x #t #f))))
 
+   (displayln "updating feed table")
    ;; update feed table
    (define atom-table (csv->list
                        (open-input-file
@@ -304,31 +309,29 @@
                              (build-path "data" "atom"))
                          #".csv"))))
    (define new-atom-table
-     (add-atom-entry atom-table (list l (~a res-link) d publish-time)))
+     (add-atom-entry atom-table (atom-table-entry l (~a res-link) d publish-time)))
    (write-csv-to-file (list->csv new-atom-table)
                       (if x "publish-test/data/atom.csv" "data/atom.csv"))
 
    
    ;; update feed.atom
    ;; for now, this does not include the html for the post, only a link
-   (define feed ((eval (read (open-input-file (if x
-                                                  "publish-test/data/make-atom.txt"
-                                                  "data/make-atom.txt"))) ns)
-                 "https://oxaliq.net/feed.atom"
-                 publish-time
-                 homepage
-                 (rest new-atom-table)))
+   (displayln "updating feed")
    (define feed-out
      (open-output-file (if x "publish-test/source/feed.atom"
                            "source/feed.atom")
                        ;; need to remove this and just replace once testing is done
-                       #:exists (if x 'replace 'error)))
+                       #:exists 'replace ))
    (if (port-try-file-lock? feed-out 'exclusive)
        (begin
-         (display 
-          (string-append "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
-                                 (xexpr->string feed))
-                   feed-out)
+         (display
+          (make-feed ns (open-input-file
+                         (if x
+                             "publish-test/data/make-atom.txt"
+                             "data/make-atom.txt"))
+                     new-atom-table
+                     publish-time)
+          feed-out)
          (port-file-unlock feed-out)
          (close-output-port feed-out))
        (error "couldn't obtain file lock on ~a" feed-out))                                                  

utils.rkt

@@ -0,0 +1,59 @@
+#lang racket
+(provide (all-defined-out))
+
+(require (only-in racket/list flatten add-between)
+         (only-in xml xexpr->string))
+
+
+(define homepage "https://oxaliq.net")
+  
+;; takes a parsed table as a list of lists and formats for writing as a .csv file
+(define (list->csv l)
+  (foldl (lambda (i res)
+           (string-append res i))
+         ""
+         (flatten
+          (add-between (map (lambda (row)
+                              (add-between row ","))
+                            l)
+                       "\n"))))
+
+;; construct resource-link for use in atom feed
+(define (resource-link resource-type resource-id #:homepage (hp homepage))
+  (if (equal? resource-type "root")
+      (build-path hp (~a resource-id))
+      (build-path hp resource-type (~a resource-id))))
+
+(define (archive-file input-file test #:modify-mode (exists-ok #f))
+  (if test
+      (copy-file input-file (string-replace input-file "in-progress" "publish-test/archive") #t)
+      (rename-file-or-directory input-file (string-replace input-file "in-progress" "archive") exists-ok)))
+
+
+;; atom-table utils
+;; ----------------
+;; atom-table-entry constructs new row to pass to add-atom-entry
+(define (atom-table-entry headline resource-link desc publish-time #:update-time (update-time ""))
+  (list headline (~a resource-link) desc publish-time update-time))
+
+;; add-atom-entry takes old atom table and new row, constructing new
+;; atom table with only first 21 rows (or all rows)
+(define (add-atom-entry atom-table new-row)
+  ;; insert new-row after header
+  (let ([out-length (min (+ 1 (length atom-table)) 21)]
+        [header (first atom-table)]
+        [old-content (rest atom-table)])
+    (take (append (list header new-row) old-content)
+          out-length)))
+;; ----------------
+
+;; atom-feed utils
+;; ---------------
+(define (make-feed ns make-atom-input-port atom-table publish-time #:homepage (hp homepage))
+  (string-append "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+                 (xexpr->string ((eval (read make-atom-input-port) ns)
+                                 "https://oxaliq.net/feed.atom"
+                                 publish-time
+                                 hp
+                                 (rest atom-table)))))
+;; ---------------