upload.clj

In Space:

Written in Babashka.

Runs the static site generator and uploads the given pages to origin, purging the cache of those pages so they can properly update.

#!/usr/bin/env bb
(require '[babashka.curl :as curl]
         '[babashka.process :refer [shell]]
         '[clojure.string :as str]
         '[clojure.java.io :as io])

(def access-key (str/trim (slurp ".bunny-key")))
(def api-key (str/trim (slurp ".api-key")))

;; TODO going to have to make a function to upload an entire dir...

;; Move these at the top so it's clearer
(def in-path "out/")

;; TODO surely there's a library for this lol.
;; I don't even need the keys, this is just a set, and `reader` is weird.
;; If reader could be avoided, this just becomes a map of extensions to mime types, and that certainly already exists lol.
(def file-types
  {:png {:extension ".png"
         :mime "image/png"
         :reader io/input-stream}
   :json {:extension ".json"
          :mime "text/json"
          :reader slurp}
   :html {:mime "text/html"
          :extension ".html"
          :reader slurp}
   :css {:extension ".css"
         :mime "text/css"
         :reader slurp}
   :uf2 {:extension ".uf2"
         :mime "application/octet-stream"
         :reader io/input-stream}
   :bytes {:mime "application/octet-stream"
           :reader io/input-stream}})

(defn file-type [path]
  (file-types
   (or (some (fn [​[k v]]
               (when-let [ext (:extension v)]
                 (when (str/ends-with? path ext)
                   k)))
             file-types)
       :bytes)))

(defn retrying-put! [url m]
  (loop []
    (let [resp (curl/put url (assoc m :throw false))]
      (if (= 429 (:status resp))
        (do
          (println "Rate limited, retrying:" url)
          (Thread/sleep (* 1000 (get-in resp [:body :retry_after_seconds] 1)))
          #_(Thread/sleep (* 1000 (-> resp :body :retry_after_seconds (or 1))))
          (recur))
        resp))))

(defn retrying-post! [url m]
  (loop []
    (let [resp (curl/post url (assoc m :throw false))]
      (if (= 429 (:status resp))
        (do
          (println "Rate limited, retrying:" url)
          (Thread/sleep (* 1000 (get-in resp [:body :retry_after_seconds] 1)))
          (recur))
        resp))))

(defn upload-file! [url file-type file-path]
  (let [content ((:reader file-type) file-path)]
    (retrying-put! url
                   {:headers {"AccessKey" access-key
                              "Content-Type" (:mime file-type)}
                    :body content})))

(def purge-url "https://api.bunny.net/purge")

(def site-url "https://grahp.dev")

(defn permalink [path]
  (cond
    (= path "index.html") ""
    (str/ends-with? path "/index.html") (str/replace path #"\/index\.html$" "")
    :else path))

(defmacro run [& body]
  `(Thread/startVirtualThread
     (fn [] ~@body)))

(defn block-until-complete! [threads]
  (doseq [t threads]
    (Thread/.join t)))

(defmacro complete-all! [& exprs]
  `(block-until-complete!
    ~(reduce (fn [acc expr]
               (conj acc `(run ~expr)))
             [] exprs)))

(defn purge-page! [path]
  (let [page-path (permalink path)
        url (str purge-url "?url=" site-url "/" page-path)]
    (retrying-post! url {:headers {"accesskey" api-key}})))

(defn purge-dir! [dir-path]
  (let [url (str purge-url "?url=" site-url "/" dir-path "/*")
        dir-url (str purge-url "?url=" site-url "/" dir-path)]
    (complete-all! (retrying-post! url {:headers {"accesskey" api-key}})
                   (retrying-post! dir-url {:headers {"accesskey" api-key}}))))

(def storage-url "https://storage.bunnycdn.com/grahp-dev-storage")

(def created 201)

(defn upload-page! [path & {:keys [purge] :or {purge true}}]
  {:pre [path (not (str/starts-with? path "."))]}
  (let [file-type (file-type path)
        file-path (str in-path path)
        url (str storage-url "/" path)]
    (upload-file! url file-type file-path)
    (when purge (purge-page! path))))

(import '[java.io File])

;; Wait isn't this basically just what pmap does lol
(defn io-run! [f coll]
  (block-until-complete!
   (reduce (fn [acc x]
             (conj acc (run (f x))))
           [] coll)))

(defn upload-dir! [path]
  (println "Uploading dir" path "...")
  (let [root (File/.getCanonicalPath (io/file in-path))
        dir (if (= path "/")
              (io/file in-path)
              (io/file in-path path))]
    (io-run! #(upload-page! (-> (File/.getCanonicalPath %)
                                (subs (inc (count root)))
                                (str/replace "\\" "/"))
                            :purge false)
             (filter File/.isFile (file-seq dir)))
    (purge-dir! path)))

(def index #{"home" "index"})

(let [​[& paths] *command-line-args*
      start-time (System/nanoTime)]
  (when (empty? paths)
    (throw (Exception. "Improper usage: No files supplied!")))
  (shell "./build")
  (io-run! (fn [path]
             (cond (index path) (upload-page! "index.html")
                   (= "/" path) (upload-dir! "/")
                   (.isDirectory (io/file in-path path)) (upload-dir! path)
                   :else (upload-page! path))
             (println "Uploaded:" path))
           paths)
  (let [duration-ms (long (/ (- (System/nanoTime) start-time)
                             1000000))]
    (println "Done!")
    (println "Took" (str duration-ms "ms"))))

(comment
  (doc curl/get)
  (def uhh (curl/get (str storage-url "/steno/")
                     {:headers {"AccessKey" access-key}}))
  (def uhh2 (vec (json/parse-string (:body uhh))))
  (rand-nth uhh2)
  (get (json/parse-string (:body uhh)) 3))