diff --git a/.gitignore b/.gitignore index b94bdc6f..5610bfcd 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ docs/*qmd _quarto.yml book +/read-kinds-diffs diff --git a/deps.edn b/deps.edn index c3834cc3..21e6a0e4 100644 --- a/deps.edn +++ b/deps.edn @@ -1,17 +1,20 @@ {:paths ["src" "resources"] - :deps {org.clojure/clojure {:mvn/version "1.12.0"} + :deps {org.clojure/clojure {:mvn/version "1.12.4"} org.clojure/tools.cli {:mvn/version "1.1.230"} nrepl/nrepl {:mvn/version "1.3.1"} com.cnuernber/charred {:mvn/version "1.037"} read-kinds/read-kinds {:local/root "../read-kinds"} + ;; TODO back to lambdaisland/deep-diff2 once fix merged + io.github.onbreath/deep-diff2 {:git/sha "1f969521b68ce9dd9feed9b51a99a4569482b6ad"} org.antlr/antlr4-runtime {:mvn/version "4.7.1"} http-kit/http-kit {:mvn/version "2.8.0"} ring/ring-core {:mvn/version "1.14.1"} io.github.nextjournal/markdown {:mvn/version "0.6.157"} hiccup/hiccup {:mvn/version "2.0.0-RC5"} clj-commons/clj-yaml {:mvn/version "1.0.29"} - org.scicloj/kindly {:mvn/version "4-beta21"} - org.scicloj/kindly-advice {:mvn/version "1-beta14"} + org.scicloj/clay-old {:local/root "../clay-old"} + io.github.scicloj/kindly {:git/sha "2863a8e08c39b1ec50a2ffde2d0874942a113d4b"} + io.github.scicloj/kindly-advice {:git/sha "17f506f169ac09b2c2d90625f1a31246e0b87360"} org.scicloj/tempfiles {:mvn/version "1-beta1"} org.scicloj/kind-portal {:mvn/version "1-beta3"} org.clojure/tools.reader {:mvn/version "1.5.2"} diff --git a/src/scicloj/clay/v2/make.clj b/src/scicloj/clay/v2/make.clj index ba11a905..358dd041 100644 --- a/src/scicloj/clay/v2/make.clj +++ b/src/scicloj/clay/v2/make.clj @@ -17,7 +17,9 @@ [clojure.pprint :as pp] [scicloj.kindly-render.notes.to-html-page :as to-html-page] ;; [hashp.preload] - [scicloj.kindly.v4.api :as kindly])) + [scicloj.kindly.v4.api :as kindly]) + (:import java.time.LocalDateTime + java.time.format.DateTimeFormatter)) (defn spec->source-type [{:keys [source-path]}] (some-> source-path (fs/extension))) @@ -366,6 +368,11 @@ :items items :exception exception)] [(case (first format) + :edn {:spec spec + :notes notes + :items items + :test-forms test-forms + :exception exception} :hiccup (page/hiccup spec-with-items) :html (do (-> spec-with-items (config/add-field :page (if post-process @@ -471,8 +478,12 @@ (fs/delete-tree target)) (util.fs/copy-tree-no-clj subdir target))))))) +(defn ts [] + (.format (LocalDateTime/now) + (DateTimeFormatter/ofPattern "yyyy-MM-dd-HH-mm-ss~N"))) + (defn make! [spec] - (let [config (config/config spec) + (let [config (config/config (assoc spec :diff/timestamp (ts))) {:keys [single-form single-value]} spec {:keys [main-spec single-ns-specs]} (extract-specs config spec) {:keys [ide browse show book base-target-path clean-up-target-dir live-reload]} main-spec @@ -511,4 +522,8 @@ , (make! {:source-path ["notebooks/index.clj"] :format [:gfm] + :show false}) + + (make! {:source-path ["notebooks/demo.clj"] + :format [:edn] :show false})) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index 25447a88..34a70426 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -5,10 +5,14 @@ [scicloj.clay.v2.util.path :as path] [scicloj.clay.v2.item :as item] [scicloj.clay.v2.prepare :as prepare] + [scicloj.clay.v2.old.notebook :as notebook-old] + [scicloj.clay.v2.old.make :as make-old] [scicloj.clay.v2.read :as read] [scicloj.kindly.v4.api :as kindly] - [scicloj.kindly-advice.v1.api :as kindly-advice]) - (:import (java.io StringWriter))) + [scicloj.kindly.v5.api :as kindly-v5] + [scicloj.kindly-advice.v1.completion :as ka-completion] + [scicloj.kindly-advice.v2.completion :as ka-completion-v2] + [scicloj.clay.v2.util.diff :as diff])) (set! *warn-on-reflection* true) @@ -42,29 +46,6 @@ (and (sequential? form) (-> form first (= 'ns)))) -(defn str-and-reset! [w] - (when (instance? StringWriter *out*) - (locking w - (let [s (str w)] - (.setLength (.getBuffer ^StringWriter w) 0) - s)))) - -(def ^:dynamic *out-orig* *out*) - -(defn maybe-println-orig [s] - (when (seq s) - (binding [*out* *out-orig*] - (print s) - (flush)))) - -(def ^:dynamic *err-orig* *err*) - -(defn maybe-err-orig [s] - (when (seq s) - (binding [*out* *err-orig*] - (print s) - (flush)))) - ;; Babashka ;; - Make Clay runnable in Babashka ;; - The way Clay reads - dependency of `carocad/parcera`, maybe we just remove it. @@ -94,56 +75,6 @@ ;; - Can we frame the stack around eval? ;; - Is output being done right? -(defn read-eval-capture - "Captures stdout and stderr while evaluating a note" - [{:as note - :keys [code form]}] - note - #_ - (let [out (StringWriter.) - err (StringWriter.) - note (try - (let [x (binding [*out* out - *err* err] - (cond form (-> form - eval - deref-if-needed) - code (-> code - read-string - eval - deref-if-needed)))] - (assoc note :value x)) - (catch Throwable ex - (assoc note :exception ex))) - out-str (str out) - err-str (str err) - ;; A notebook may have also printed from a thread, - ;; *out* and *err* are replaced with StringWriters in with-out-err-capture - global-out (str-and-reset! *out*) - global-err (str-and-reset! *err*) - ;; Don't show output from requiring other namespaces - show (not (ns-form? form))] - (maybe-println-orig out-str) - (maybe-err-orig err-str) - (maybe-println-orig global-out) - (maybe-err-orig global-err) - (if show - (cond-> note - (seq out-str) (assoc :out out-str) - (seq err-str) (assoc :err err-str) - (seq global-out) (assoc :global-out global-out) - (seq global-err) (assoc :global-err global-err)) - note))) - -(defn complete [{:as note - :keys [comment?]}] - (let [completed (cond-> note - (not (or comment? (contains? note :value))) - (read-eval-capture))] - (cond-> completed - (and (not comment?) (contains? completed :value)) - (kindly-advice/advise)))) - (defn comment->item [comment] (-> comment (str/split #"\n") @@ -323,22 +254,6 @@ code new-code)}))) (@*path->last path)) -(defmacro with-out-err-captured - "Evaluates and computes the items for a notebook of notes" - [& body] - ;; For a notebook, we capture output globally, and per note. - ;; see read-eval-capture for why this is relevant. - `(let [out# (StringWriter.) - err# (StringWriter.)] - ;; Threads may inherit only the root binding - (with-redefs [*out* out# - *err* err#] - ;; Futures will inherit the current binding, - ;; which was not affected by altering the root. - (binding [*out* out# - *err* err#] - ~@body)))) - (defn itemize-notes "Evaluates and computes the items for a notebook of notes" [relevant-notes some-narrowed options] @@ -409,22 +324,18 @@ :format])] (doall (for [note notes] - (complete (kindly/deep-merge opts note)))))) + (kindly/deep-merge opts note))))) (defn relevant-notes [{:keys [full-source-path - single-form - single-value - smart-sync - pprint-margin] - :or {pprint-margin pp/*print-right-margin*}}] - (let [{:keys [code first-line-of-change]} (some-> full-source-path slurp-and-compare) - notes (->> (cond single-value (conj (when code - [{:form (read/read-ns-form code)}]) - {:value single-value}) - single-form (conj (when code - [{:form (read/read-ns-form code)}]) - {:form single-form}) - :else (read/->notes code)) + single-form + single-value + smart-sync + pprint-margin] + :or {pprint-margin pp/*print-right-margin*} + :as spec}] + (let [{:keys [code first-line-of-change]} (some-> full-source-path + slurp-and-compare) + notes (->> (read/->notes (assoc spec :code code)) (map-indexed (fn [i {:as note :keys [code]}] (merge note @@ -464,6 +375,78 @@ "seconds") result#)) +(defn ->old-comment [note] + (let [comment-item (-> note :code notebook-old/comment->item)] + (-> note + (assoc :value (str/replace (:md comment-item) + ;; TODO stripping extra space added + ;; in front of headline Do we want + ;; to add this for read-kinds? + #"\n#" "#") + :kind :kind/md) + (dissoc :code :comment? :region)))) + +(defn ->note-approx [note] + (dissoc note :format)) + +;; TODO Decide on whether we want to remove options form the notebook +(defn old-kindly-options? [note] + (not (empty? (into [] (comp (mapcat (fn [x] (if (coll? x) x [x]))) + (filter #{'kindly/set-options! 'kindly/merge-options!}) + (take 1)) + (:form note))))) + +(defn ->old-notes-approx [notes] + (->> notes + (into [] + (comp (map #(-> % + (cond-> (and (:comment? %) + (:code %)) + ->old-comment) + (dissoc :gen) + ->note-approx)) + (remove old-kindly-options?))))) + +(defn new-kindly-options? [note] + (contains? (some-> note :value meta) :kindly/merge-options)) + +(defn ->new-notes-approx [notes] + (->> notes + (into [] + (comp (map #(-> % + (dissoc :line :column) + ->note-approx + (cond-> (-> % :narrowed nil?) + (dissoc :narrowed) + (-> % :narrower nil?) + (dissoc :narrower)))) + (remove new-kindly-options?))))) + + +(defn old-spec-notes [spec] + (-> (make-old/make! {:source-path (:full-source-path spec) + :format [:edn] + :show false}) + :info + ffirst + first + :notes + ->old-notes-approx)) + +(defn new-spec-notes [{:as spec + :keys [ns-form full-source-path]}] + (with-redefs [;; See kindly-advice branch with v2-namespace for TODO on this + ka-completion/complete-options ka-completion-v2/complete-options + kindly/get-options (constantly nil) + kindly/set-options! kindly-v5/set-options! + kindly/merge-options! kindly-v5/merge-options!] + (-> (assoc spec :collapse-comments-ws? true) + (relevant-notes) + (complete-notes spec) + (log-time (str "Evaluated notebook with read-kinds " + (or (some-> ns-form second name) + (some-> full-source-path fs/file-name))))))) + (defn spec-notes [{:as spec :keys [pprint-margin ns-form full-source-path] :or {pprint-margin pp/*print-right-margin*}}] @@ -471,12 +454,42 @@ *warn-on-reflection* *warn-on-reflection* *unchecked-math* *unchecked-math* pp/*print-right-margin* pprint-margin] - (-> (relevant-notes spec) - (complete-notes spec) - (with-out-err-captured) - (log-time (str "Evaluated " - (or (some-> ns-form second name) - (some-> full-source-path fs/file-name))))))) + ;; TODO this just works for one notebook without a base-source-path etc. + (let [old (old-spec-notes spec) + new-ret (new-spec-notes spec) + new (->new-notes-approx new-ret)] + ;; We can print the plain new and old notes.. + #_(diff/notes old new + :diff/to-repl :clojure/pprint + spec) + ;; ..or only differences + (diff/notes old new + :diff/to-repl :deep-diff2/minimal + spec) + ;; ..or only one difference + #_(diff/notes (take 1 old) (take 1 new) + :diff/to-repl :deep-diff2/minimal + spec) + ;; ..or write old and new files + #_(diff/notes old new + :diff/to-files :clojure/pprint + spec) + ;; ..or old, new and full diff files + #_(diff/notes old new + :diff/to-files :deep-diff2/full + spec) + ;; ..or old, new and minimal diffs, keeping only the last three runs + #_(diff/notes old new + :diff/to-files :deep-diff2/minimal + :diff/keep-dirs 3 + spec) + ;; ..or any combination of the above + #_(diff/notes old new + :diff/to-repl :deep-diff2/minimal + :diff/to-files :deep-diff2/full + :diff/keep-dirs 3 + spec) + new-ret))) (defn items-and-test-forms [notes spec] diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index 42873097..86d3dd26 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -1,6 +1,9 @@ (ns scicloj.clay.v2.read (:require [scicloj.read-kinds.notes :as notes] - [scicloj.read-kinds.read :as read])) + [scicloj.read-kinds.read :as read] + [clojure.tools.reader] + [clojure.tools.reader.reader-types] + [clojure.string :as str])) ;; TODO: not sure if generation is necessary??? @@ -26,9 +29,48 @@ (-> form first (= 'ns))))) first)) -(defn ->notes [code] - (->> (read/read-string-all code) - (into [] notes/notebook-xform))) +;; TODO this is intentionally not a complete replacement +;; for read-kinds.notes/notebook-xform, it doesn't +;; assoc :kind/md so notebook-xform can still look +;; for expected kinds in the pipeline/transform +(defn collapse-comments-ws [collapse-comments-ws? notes] + (if collapse-comments-ws? + (let [collapse (comp #{:kind/whitespace :kind/comment} :kind) + comment? (comp #{:kind/comment} :kind)] + (->> notes + (partition-by (comp boolean collapse)) + (mapcat + (fn [notes*] + (if (some comment? notes*) + ;; TODO Pulling in all comments and whitespace + ;; This is only done to easily get equality with old comments + [{:value (let [comment* (->> notes* + (map #(get % :value (:code %))) + str/join)] + (-> comment* + (str/replace #"^\s+" "") + (str/trim-newline))) + :kind :kind/comment}] + notes*))))) + notes)) + +;; TODO keep this or something like it +(defn ->notes [{:keys [single-form + single-value + code + collapse-comments-ws?]}] + (cond single-value (conj (when code + [{:form (read-ns-form code)}]) + {:value single-value}) + ;; TODO Doesn't actually eval the form + single-form (conj (when code + [{:form (read-ns-form code)}]) + {:form single-form}) + :else (->> code + (read/read-string-all) + (read/eval-notes) + (collapse-comments-ws collapse-comments-ws?) + (into [] notes/notebook-xform)))) ;; TODO: Not needed? read-kinds has a safe-notes wrapper already... (defn ->safe-notes [code] diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj new file mode 100644 index 00000000..d4513355 --- /dev/null +++ b/src/scicloj/clay/v2/util/diff.clj @@ -0,0 +1,102 @@ +(ns scicloj.clay.v2.util.diff + (:require [clojure.string :as str] + [clojure.pprint :as pp] + [babashka.fs :as fs] + [lambdaisland.deep-diff2 :as ddiff] + [scicloj.clay.v2.util.diff.prep :as prep-diff])) + +(defn- print-diffs [diff-print-fn note-diffs] + (doseq [diff note-diffs] + (when (some-> diff ddiff/minimize not-empty) + (diff-print-fn diff)))) + +(defn- diff-print-fn [k] + (case k + :deep-diff2/full ddiff/pretty-print + :deep-diff2/minimal (comp ddiff/pretty-print + ddiff/minimize) + nil)) + +(defn- write-diff-files [old new note-diffs diff-print-fn print-fn + {:diff/keys [keep-dirs + timestamp] + :keys [full-source-path] + :as spec}] + (let [diffs-base-path (fs/absolutize "read-kinds-diffs") + source-name (-> full-source-path (str/replace "/" ".")) + diffs-path (-> diffs-base-path + (fs/path (str source-name "~" timestamp))) + diff-base-file (->> source-name + (fs/path diffs-path) + str) + no-diff-file (str diff-base-file ".no-diff") + diff-file (str diff-base-file ".diff.edn") + old-file (str diff-base-file ".old.edn") + new-file (str diff-base-file ".new.edn")] + (fs/create-dirs diffs-path) + (when (number? keep-dirs) + (let [diff-dirs (->> (fs/list-dir diffs-base-path + #(fs/directory? % {:nofollow-links true})) + (sort-by fs/last-modified-time))] + (doseq [dir (take (max 0 (inc (- (count diff-dirs) keep-dirs))) + diff-dirs)] + (when (= (fs/parent dir) diffs-base-path) + (fs/delete-tree dir))))) + (if (some not-empty (ddiff/minimize note-diffs)) + (when diff-print-fn + (println "Clay: Creating diff file" diff-file "& old/new") + (spit diff-file (with-out-str + (print-diffs diff-print-fn note-diffs)))) + (do (println "Clay: Creating no-diff file" no-diff-file "& old/new") + (spit no-diff-file "no difference"))) + (spit old-file (with-out-str (print-fn old))) + (spit new-file (with-out-str (print-fn new))) + (println "Clay: Copying latest diff files to" (str diffs-base-path)) + (doseq [file (fs/list-dir diffs-base-path + #(fs/regular-file? % {:nofollow-links true}))] + (fs/delete file)) + (doseq [file (fs/list-dir diffs-path + #(fs/regular-file? % {:nofollow-links true}))] + (fs/copy file diffs-base-path)))) + +(defn- pad-notes [old new] + (let [pad-to #(take (max 0 (- (count %1) (count %2))) (repeat {}))] + [(into [] (concat old (pad-to new old))) + (into [] (concat new (pad-to old new)))])) + +(defn notes [old new & {:diff/keys [to-files + to-repl + timestamp] + :as spec}] + (assert (or to-files to-repl) "Please pick an output option") + (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") + (let [[old new] (prep-diff/replace-undiffable (pad-notes old new)) + note-diffs (mapv ddiff/diff old new) + file-diff-print-fn (case to-files + (:deep-diff2/full :deep-diff2/minimal) + (diff-print-fn to-files) + ;; No diff, but we always write old/new + ;; when to-files is specified + (:clojure/pprint nil) nil) + print-fn pp/pprint + repl-print-fn (case to-repl + (:deep-diff2/full :deep-diff2/minimal) + #(print-diffs (diff-print-fn to-repl) note-diffs) + :clojure/pprint #(doseq [[old* new*] (map vector old new)] + (println "--------- old: ") + (print-fn old*) + (println "--------- new: ") + (print-fn new*)) + nil nil)] + (when to-repl + (println "Clay: Notes diff start") + (let [diff (some not-empty (ddiff/minimize note-diffs))] + (when-not diff + (println "Clay: >>>> No difference!")) + (repl-print-fn) + ;; Add note on no difference at the end too when printing all data + (when (and (= to-repl :clojure/pprint) (not diff)) + (println "Clay: >>>> No difference!"))) + (println "Clay: Notes diff end")) + (when to-files + (write-diff-files old new note-diffs file-diff-print-fn print-fn spec)))) diff --git a/src/scicloj/clay/v2/util/diff/prep.clj b/src/scicloj/clay/v2/util/diff/prep.clj new file mode 100644 index 00000000..910c2cda --- /dev/null +++ b/src/scicloj/clay/v2/util/diff/prep.clj @@ -0,0 +1,66 @@ +(ns scicloj.clay.v2.util.diff.prep + (:require [clojure.walk :as walk] + [clojure.string :as str])) + +(defprotocol DiffableBaseType + (diffable-base-type? [this])) + +(extend-protocol DiffableBaseType + java.util.Set + (diffable-base-type? [_] true) + java.util.Map + (diffable-base-type? [_] true) + java.util.List + (diffable-base-type? [_] true)) + +(defn type-str [x] + (-> x type pr-str)) + +(defn describe-type [t] + (type-str t)) + +(declare replaced-value=) + +(defprotocol PReplacedValue + (value? [this])) + +(deftype ReplacedValue [value value-type] + PReplacedValue + (value? [_] + (= (describe-type value) value-type)) + + Object + (equals [this other] + (or (not (value? this)) + (replaced-value= this other))) + (hashCode [this] + (hash-combine (hash value-type) + (if (value? this) + (hash value) + value))) + (toString [this] + (pr-str (.hashCode this) + value-type))) + +(defn replaced-value= [^ReplacedValue this other] + (and (instance? ReplacedValue this) + (instance? ReplacedValue other) + (= (.-value-type this) (.-value-type ^ReplacedValue other)) + (= (.-value this) (.-value ^ReplacedValue other)))) + +(defn diffable-type? [x] + (let [x-type-str (type-str x)] + (some (partial String/.startsWith x-type-str) + ["java.util" + "clojure.lang"]))) + +(defn replace-undiffable* [x] + (cond (fn? x) ::replaced-fn + (and (satisfies? DiffableBaseType x) + (diffable-base-type? x) + (not (diffable-type? x))) + (->ReplacedValue x (describe-type x)) + :else x)) + +(defn replace-undiffable [notes] + (walk/prewalk replace-undiffable* notes))