From 97a1125695aa052cd5273e34a41383e57732660d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BCdiger=20Diedrich?= Date: Mon, 22 Jun 2020 22:30:59 +0200 Subject: [PATCH] different http client, looking forward to merging --- project.clj | 2 +- resources/public/css/style.css | 37 +++++++++++++++- src/clojsa/handler.clj | 20 +++++++-- src/clojsa/saclient.clj | 80 +++++----------------------------- src/clojsa/saparser.clj | 63 ++++++++++++++++++++++++++ src/clojsa/views.clj | 18 ++++---- 6 files changed, 136 insertions(+), 84 deletions(-) create mode 100644 src/clojsa/saparser.clj diff --git a/project.clj b/project.clj index 3596379..c326c2a 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :dependencies [[org.clojure/clojure "1.10.0"] [compojure "1.6.1"] [ring/ring-defaults "0.3.2"] - [clj-http "3.10.1"] + [http.async.client "1.3.1"] [hickory "0.7.1"]] :plugins [[lein-ring "0.12.5"]] :ring {:handler clojsa.handler/app} diff --git a/resources/public/css/style.css b/resources/public/css/style.css index be10e77..6b4ee9e 100644 --- a/resources/public/css/style.css +++ b/resources/public/css/style.css @@ -16,5 +16,40 @@ body { nav.pagination { background-color: #fff; - margin-top: 4rem; + margin-top: 2rem; +} + +.userinfo > dd.registered { + font-size: 0.75rem; +} + +.userinfo > dd.title { + margin-top: 1rem; + font-size: inherit; + font-weight: inherit; + line-height: inherit; +} + +.content .bbc-block > h4 { + font-size: 1rem; +} + +.bbc-spoiler { + transition: background-color 0.2s; + background-color: black; + color: black; +} +.bbc-spoiler:hover, .bbc-spoiler.active { + background-color: transparent; + color: auto; +} + +.editedby { + font-size: .75rem; + font-style: italic; + margin-top: 1rem; +} + +.postdate { + font-size: .75rem; } diff --git a/src/clojsa/handler.clj b/src/clojsa/handler.clj index 0c23246..50d31f0 100644 --- a/src/clojsa/handler.clj +++ b/src/clojsa/handler.clj @@ -1,15 +1,27 @@ (ns clojsa.handler - (:require [clojsa.views :refer [index-page thread-page]] + (:require [clojsa.views :as views] + [clojsa.saclient :as client] + [clojsa.saparser :as parser] [compojure.core :refer :all] [compojure.route :as route] + [compojure.coercions :refer [as-int]] [ring.middleware.defaults :refer [wrap-defaults site-defaults]] [ring.middleware.session :refer [wrap-session]] [ring.middleware.session.cookie :refer (cookie-store)])) + +(defn get-thread [session id page] + (let [turl (client/thread-url id page) + tresp (client/thread-response session turl)] + (parser/thread-map id page tresp))) + (defroutes app-routes - (GET "/" request (index-page request)) - (GET "/thread/:id" [id page] - (thread-page (Integer/parseInt id) (Integer/parseInt page))) + (GET "/" request (views/index-page request)) + (GET "/thread/:id" [id :<< as-int + page :<< as-int + :as {session :session}] + (let [thread (get-thread session id page)] + (views/thread-page thread))) (route/not-found "Not Found")) (def app diff --git a/src/clojsa/saclient.clj b/src/clojsa/saclient.clj index 3a84436..6ea0bed 100644 --- a/src/clojsa/saclient.clj +++ b/src/clojsa/saclient.clj @@ -1,11 +1,10 @@ (ns clojsa.saclient (:require [clojure.string :as string] - [clj-http.client :as client] - [hickory.core :refer :all] - [hickory.select :as s] - [hickory.convert :refer [hickory-to-hiccup]])) + [http.async.client :as http])) + (def url "https://forums.somethingawful.com/") + (defn thread-url ([id] (thread-url id 1)) @@ -15,67 +14,12 @@ :pagenumber page}] {:href base-url :params query}))) -(defn thread-response [url] - (let [resp (client/get (:href url) {:query-params (:params url)})] - (:body resp))) - -(def witcher-thread (thread-response (thread-url 3720352 3))) - -(defn hickory-doc [doc] - (-> doc parse as-hickory)) - -(defn parse-title [htree] - (-> (s/select (s/child (s/tag :title)) htree) - first :content first)) - -(defn parse-pagecount [htree] - (-> (s/select (s/descendant - (s/class :pages) (s/tag :option)) htree) - last :content first Integer/parseInt)) - -(defn parse-thread [htree] - (-> (s/select (s/descendant - (s/id :thread)) - htree) - first)) - -(defn select-td [class-key htree] - (s/select (s/descendant - (s/and (s/tag :td) (s/class class-key))) htree)) - -(defn parse-ui [ui] - (let [ui (first (s/select (s/descendant (s/tag :dl)) ui))] - (hickory-to-hiccup ui))) - -(defn parse-pd [pd] - (string/trim (last (hickory-to-hiccup pd)))) - -(defn hickory-div [class content] - {:type :element, - :attrs {:class class}, - :tag :div, - :content content}) - -(defn parse-pb [pb] - (let [pb (-> pb :content)] - (hickory-to-hiccup (hickory-div "postbody" pb)))) - -(defn thread-map [id doc] - (let [htree (hickory-doc doc) - title (parse-title htree) - page-count (parse-pagecount htree) - thread-tree (parse-thread htree) - userinfo (select-td :userinfo thread-tree) - postdate (select-td :postdate thread-tree) - postbody (select-td :postbody thread-tree)] - {:title title - :id id - :page-count page-count - :content - (for [[ui pd pb] (partition 3 (interleave userinfo postdate postbody)) - :when (not= "Adbot" (-> (s/select (s/child (s/class :author)) ui) - first :content first))] - {:ui (parse-ui ui) :pd (parse-pd pd) :pb (parse-pb pb)})})) - -(defn get-thread [id page] - (thread-map id (thread-response (thread-url id page)))) +(defn thread-response [session url] + (with-open [client (http/create-client)] + (let [{:keys [href params]} url + resp (http/GET client href :query params) + status (http/status resp) + headers (http/headers resp)] + (-> resp + http/await + http/string)))) diff --git a/src/clojsa/saparser.clj b/src/clojsa/saparser.clj new file mode 100644 index 0000000..b72dbec --- /dev/null +++ b/src/clojsa/saparser.clj @@ -0,0 +1,63 @@ +(ns clojsa.saparser + (:require [clojsa.saclient :refer [thread-url thread-response]] + [clojure.string :as string] + [hickory.core :refer :all] + [hickory.select :as s] + [hickory.convert :refer [hickory-to-hiccup]])) + +(defn hickory-doc [doc] + (-> doc parse as-hickory)) + +(defn parse-title [htree] + (-> (s/select (s/child (s/tag :title)) htree) + first :content first)) + +(defn parse-pagecount [htree] + (-> (s/select (s/descendant + (s/class :pages) (s/tag :option)) htree) + last :content first Integer/parseInt)) + +(defn parse-thread [htree] + (-> (s/select (s/descendant + (s/id :thread)) + htree) + first)) + +(defn select-td [class-key htree] + (s/select (s/descendant + (s/and (s/tag :td) (s/class class-key))) htree)) + +(defn parse-ui [ui] + (let [ui (first (s/select (s/descendant (s/tag :dl)) ui))] + (hickory-to-hiccup ui))) + +(defn parse-pd [pd] + (string/trim (last (hickory-to-hiccup pd)))) + +(defn hickory-div [class content] + {:type :element, + :attrs {:class class}, + :tag :div, + :content content}) + +(defn parse-pb [pb] + (let [pb (-> pb :content)] + (hickory-to-hiccup (hickory-div "postbody" pb)))) + +(defn thread-map [id page doc] + (let [htree (hickory-doc doc) + title (parse-title htree) + page-count (parse-pagecount htree) + thread-tree (parse-thread htree) + userinfo (select-td :userinfo thread-tree) + postdate (select-td :postdate thread-tree) + postbody (select-td :postbody thread-tree)] + {:title title + :id id + :page page + :page-count page-count + :content + (for [[ui pd pb] (partition 3 (interleave userinfo postdate postbody)) + :when (not= "Adbot" (-> (s/select (s/child (s/class :author)) ui) + first :content first))] + {:ui (parse-ui ui) :pd (parse-pd pd) :pb (parse-pb pb)})})) diff --git a/src/clojsa/views.clj b/src/clojsa/views.clj index 46c9114..a728463 100644 --- a/src/clojsa/views.clj +++ b/src/clojsa/views.clj @@ -1,7 +1,7 @@ (ns clojsa.views (:use [hiccup core page]) - (:require [clojsa.saclient :refer [get-thread]] - [clojure.string :as string] + (:require [clojure.string :as string] + [clojure.pprint] [clojure.java.io :as io] [cheshire.core :as json])) @@ -30,11 +30,10 @@ (include-js "/js/main.js")])) (defn index-page [req] - (let [thread (get-thread 3720352 3)] - (main-template {:title (:title thread)} - [:div.container - (for [post (:content thread)] - [:div.post (:pb post)])]))) + (main-template {} + [:div.container + [:pre + (clojure.pprint/pprint req)]])) (defn paginate [id cur last] [:nav.container.pagination {:hx-boot "false"} @@ -59,9 +58,8 @@ [:a.pagination-link {:href (format "/thread/%d?page=%d" id last)} (str last)]]]]) -(defn thread-page [id page] - (let [thread (get-thread id page) - {:keys [id title page-count content]} thread] +(defn thread-page [thread] + (let [{:keys [id title page page-count content]} thread] (main-template {:title title} [:div.container