From 8bb02f2d21a8e952c1cee5811141f30684e3ebbf Mon Sep 17 00:00:00 2001 From: Jay Date: Sun, 15 Dec 2019 04:11:13 -0500 Subject: [PATCH 1/3] light overhaul; tests added --- src/libpython_clj/require.clj | 253 +++++++++++++-------- test/libpython_clj/require_python_test.clj | 78 ++++++- 2 files changed, 232 insertions(+), 99 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 80d5d3d..a700bd8 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -81,7 +81,6 @@ (number? x) "" :else (py-class-argspec x))) - (defn pyarglists ([argspec] (pyarglists argspec (if-let [defaults @@ -141,12 +140,7 @@ (when (not-empty or-map) {:or or-map}) (when (not-empty as-varkw) - as-varkw)) - - - - - opt-args + as-varkw)) opt-args (cond (and (empty? kwargs-map) (nil? varargs)) '() @@ -168,7 +162,6 @@ arglists (recur argspec' defaults' arglists)))))) - (defn py-fn-metadata [fn-name x {:keys [no-arglists?]}] (let [fn-argspec (pyargspec x) fn-docstr (get-pydoc x)] @@ -183,7 +176,6 @@ (catch Throwable e nil)))))) - (defn ^:private load-py-fn [f fn-name fn-module-name-or-ns options] (let [fn-ns (symbol (str fn-module-name-or-ns)) @@ -214,19 +206,13 @@ supported-flags results (first items) - (rest items)) - - - :else + (rest items)) :else ;; unary flag -- add flag but scan current item/s (next-flag-item supported-flags (conj results flag) item - items))) - - - ;; scan flags + items)));; scan flags (next-flag-item [supported-flags results item items] (cond ;; supported flag scanned, begin FSM parse @@ -259,9 +245,37 @@ (rest reqs)))] (trampoline get-flags supported-flags reqs))) - - -(defn ^:private load-python-lib [req] +(defn- extract-refer-symbols + [{:keys [refer this-module]} public-data] + (cond + ;; include everything into the current namespace, + ;; ignore __all__ directive + (or (refer :all) + (and (not (py/has-attr? this-module "__all__")) + (refer :*))) + (keys public-data) + ;; only include that specfied by __all__ attribute + ;; of python module if specified, else same as :all + (refer :*) + (->> (py/get-attr this-module "__all__") + (map (fn [item-name] + (let [item-sym (symbol item-name)] + (when (contains? public-data item-sym) + item-sym)))) + (remove nil?)) + + ;; [.. :refer [..]] behavior + :else + (do + (when-let [missing (->> refer + (remove (partial contains? public-data)) + seq)] + (throw (Exception. + (format "'refer' symbols not found: %s" + (vec missing))))) + refer))) + +(defn ^:private python-lib-configuration [req] (let [supported-flags #{:reload :no-arglists} [module-name & etc] req flags (parse-flags supported-flags etc) @@ -286,80 +300,123 @@ current-ns-sym (symbol (str current-ns)) python-namespace (find-ns module-name-or-ns) this-module (import-module (str module-name))] + {:supported-flags supported-flags + :etc etc + :reload? reload? + :no-arglists? no-arglists? + :module-name module-name + :module-name-or-ns module-name-or-ns + :exclude exclude + :refer refer + :current-ns current-ns + :current-ns-sym current-ns-sym + :python-namespace python-namespace + :this-module this-module})) + +(defn- extract-public-data + [{:keys [exclude python-namespace module-name-or-ns]}] + (let [python-namespace + (or python-namespace + (find-ns module-name-or-ns))] + (->> (ns-publics python-namespace) + (remove #(exclude (first %))) + (into {})))) + +(defn- reload-python-ns! + [module-name this-module module-name-or-ns] + (do + (remove-ns module-name) + (reload-module this-module) + (create-ns module-name-or-ns))) + +(defn- create-python-ns! + [module-name-or-ns] + (create-ns module-name-or-ns)) + +(defn ^:private maybe-reload-or-create-ns! + [{:keys [reload? + this-module + module-name + module-name-or-ns] + python-namespace :python-namespace}] + (cond + reload? (reload-python-ns! module-name + this-module + module-name-or-ns) + (not python-namespace) (create-python-ns! module-name-or-ns))) + +(defn enhanced-python-lib-configuration + [{:keys [python-namespace exclude this-module] + :as lib-config}] + (let [public-data (extract-public-data lib-config)] + (merge + lib-config + {:public-data public-data + :refer-symbols (extract-refer-symbols lib-config + public-data)}))) + +(defn- bind-py-symbols-to-ns! + [{:keys [reload? + python-namespace + this-module + module-name-or-ns + no-arglists?]}] + (when (or reload? (not python-namespace)) + ;;Mutably define the root namespace. + (doseq [[att-name v] (vars this-module)] + (try + (when v + (if (py/callable? v) + (load-py-fn v (symbol att-name) module-name-or-ns + {:no-arglists? + no-arglists?}) + (intern module-name-or-ns (symbol att-name) v))) + (catch Throwable e + (log/warnf e "Failed to require symbol %s" att-name)))))) + +(defn- bind-module-ns! + [{:keys [current-ns-sym module-name-or-ns this-module]}] + (intern current-ns-sym + (with-meta module-name-or-ns + {:doc (doc this-module)}) + this-module)) + +(defn- intern-public-and-refer-symbols! + [{:keys [public-data refer-symbols current-ns-sym]}] + (doseq [[s v] (select-keys public-data refer-symbols)] + (intern current-ns-sym + (with-meta s (meta v)) + (deref v)))) + +(defn preload-python-lib! [req] + (let [lib-config (python-lib-configuration req)] + (maybe-reload-or-create-ns! lib-config) + (bind-py-symbols-to-ns! lib-config) + (bind-module-ns! lib-config) + (enhanced-python-lib-configuration lib-config))) - (cond - reload? - (do - (remove-ns module-name) - (reload-module this-module) - (create-ns module-name-or-ns)) - (not python-namespace) - (create-ns module-name-or-ns)) - - ;; bind the python module to its symbolic name - ;; in the current namespace - - - ;; create namespace for module and bind python - ;; values to namespace symbols - (when (or reload? - (not python-namespace)) - ;;Mutably define the root namespace. - (doseq [[att-name v] (vars this-module)] - (try - (when v - (if (py/callable? v) - (load-py-fn v (symbol att-name) module-name-or-ns - {:no-arglists? - no-arglists?}) - (intern module-name-or-ns (symbol att-name) v))) - (catch Throwable e - (log/warnf e "Failed to require symbol %s" att-name))))) - - - (let [python-namespace (find-ns module-name-or-ns) - ;;ns-publics is a map of symbol to var. Var's have metadata on them. - public-data (->> (ns-publics python-namespace) - (remove #(exclude (first %))) - (into {}))] - - ;;Always make the loaded namespace available to the current namespace. - (intern current-ns-sym - (with-meta module-name-or-ns - {:doc (doc this-module)}) - this-module) - (let [refer-symbols - (cond - ;; include everything into the current namespace, - ;; ignore __all__ directive - (or (refer :all) - (and (not (py/has-attr? this-module "__all__")) - (refer :*))) - (keys public-data) - ;; only include that specfied by __all__ attribute - ;; of python module if specified, else same as :all - (refer :*) - (->> (py/get-attr this-module "__all__") - (map (fn [item-name] - (let [item-sym (symbol item-name)] - (when (contains? public-data item-sym) - item-sym)))) - (remove nil?)) - - ;; [.. :refer [..]] behavior - :else - (do - (when-let [missing (->> refer - (remove (partial contains? public-data)) - seq)] - (throw (Exception. - (format "'refer' symbols not found: %s" - (vec missing))))) - refer))] - (doseq [[s v] (select-keys public-data refer-symbols)] - (intern current-ns-sym - (with-meta s (meta v)) - (deref v))))))) +(defn ^:private load-python-lib [req] + (let [{:keys [supported-flags + flags + etc + reload? + no-arglists? + module-name + module-name-or-ns + exclude + refer + current-ns + current-ns-sym + python-namespace + this-module + python-namespace + public-data + refer-symbols] + :as lib-config} + (preload-python-lib! req)] + + + (intern-public-and-refer-symbols! lib-config))) (defn require-python "## Basic usage ## @@ -438,3 +495,13 @@ (load-python-lib (vector reqs)) (vector? reqs) (load-python-lib reqs))) + +(comment + (require-python '[math + :refer :* + :exclude [sin cos] + :as pymath + :reload true]) + + + ) diff --git a/test/libpython_clj/require_python_test.clj b/test/libpython_clj/require_python_test.clj index 6ff616e..0076a50 100644 --- a/test/libpython_clj/require_python_test.clj +++ b/test/libpython_clj/require_python_test.clj @@ -1,16 +1,17 @@ (ns libpython-clj.require-python-test - (:require [libpython-clj.require :as req :refer [require-python]] + (:require [libpython-clj.require :as req :refer [require-python] + :reload true] [libpython-clj.python :as py] [clojure.test :refer :all])) - -;;Since this test mutates the global environment we have to accept that -;;it may not always work. +;; Since this test mutates the global environment we have to accept that +;; it may not always work. (require-python '[math :refer :* :exclude [sin cos] - :as pymath]) + :as pymath + :reload true]) (deftest base-require-test @@ -22,7 +23,6 @@ (is (= pymath (py/import-module "math"))) (is (thrown? Throwable (require-python '[math :refer [blah]]))))) - (deftest parse-flags-test ;; sanity check (is (= #{:reload :foo} @@ -50,3 +50,69 @@ (is (= #{:b} (parse-flags #{:a :b} '[:b true :a false]))) (is (= #{:b} (parse-flags #{:a :b} '[:b :a false]))) (is (= #{:b} (parse-flags #{:a :b} '[:b :a false :a true]))))) + +(deftest python-lib-configuration-test + (let [python-lib-configuration #'req/python-lib-configuration + simple-req '[csv] + simple-spec (python-lib-configuration simple-req) + {:keys [exclude + supported-flags + current-ns-sym + module-name + module-name-or-ns + reload? + no-arglists? + etc + current-ns + this-module + python-namespace + refer]} simple-spec + csv-module (py/import-module "csv")] + + ;; no exclusions + (is (= #{} exclude)) + (is (= #{:no-arglists :reload} supported-flags)) + (is (= 'libpython-clj.require-python-test + current-ns-sym + (symbol (str current-ns)))) + (is (= 'csv module-name module-name-or-ns)) + (is (nil? reload?)) + (is (nil? no-arglists?)) + (is (= {} etc)) + (is (= csv-module this-module))) + + + (let [python-lib-configuration #'req/python-lib-configuration + simple-req '[requests + :reload true + :refer [get] + :no-arglists] + simple-spec (python-lib-configuration simple-req) + {:keys [exclude + supported-flags + current-ns-sym + module-name + module-name-or-ns + reload? + no-arglists? + etc + current-ns + this-module + python-namespace + refer]} simple-spec + requests-module (py/import-module "requests")] + + ;; no exclusions + (is (= #{} exclude)) + (is (= #{:no-arglists :reload} supported-flags)) + (is (= 'libpython-clj.require-python-test + current-ns-sym + (symbol (str current-ns)))) + (is (= 'requests module-name module-name-or-ns)) + (is (= reload? :reload)) + (is (= no-arglists? :no-arglists)) + (is (= { :refer '[get]}) etc) + (is (= requests-module this-module)) + (is (= #{'get} refer)) + (is (nil? python-namespace)))) + From e10c94426d375e726db853165f957a18093bb0da Mon Sep 17 00:00:00 2001 From: Jay Date: Sun, 15 Dec 2019 04:15:24 -0500 Subject: [PATCH 2/3] whitespace --- src/libpython_clj/require.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index a700bd8..2df860a 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -140,7 +140,8 @@ (when (not-empty or-map) {:or or-map}) (when (not-empty as-varkw) - as-varkw)) opt-args + as-varkw)) + opt-args (cond (and (empty? kwargs-map) (nil? varargs)) '() From c5b91ec3bc431af481bd3355e0ae21dc24b08b13 Mon Sep 17 00:00:00 2001 From: Jay Date: Sun, 15 Dec 2019 04:16:42 -0500 Subject: [PATCH 3/3] formatting --- src/libpython_clj/require.clj | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/libpython_clj/require.clj b/src/libpython_clj/require.clj index 2df860a..f738dac 100644 --- a/src/libpython_clj/require.clj +++ b/src/libpython_clj/require.clj @@ -207,13 +207,16 @@ supported-flags results (first items) - (rest items)) :else + (rest items)) + :else ;; unary flag -- add flag but scan current item/s (next-flag-item supported-flags (conj results flag) item - items)));; scan flags + items))) + + ;; scan flags (next-flag-item [supported-flags results item items] (cond ;; supported flag scanned, begin FSM parse @@ -415,7 +418,6 @@ refer-symbols] :as lib-config} (preload-python-lib! req)] - (intern-public-and-refer-symbols! lib-config))) @@ -497,12 +499,3 @@ (vector? reqs) (load-python-lib reqs))) -(comment - (require-python '[math - :refer :* - :exclude [sin cos] - :as pymath - :reload true]) - - - )