rdiedrich/clojsa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

different http client, looking forward to merging

Browse Source
This commit is contained in:
Rüdiger Diedrich 2020-06-22 22:30:59 +02:00
parent a7ad35249a
commit 97a1125695
6 changed files with 136 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
project.clj
View File

@ -5,7 +5,7 @@
:dependencies [[org.clojure/clojure "1.10.0"]
[compojure "1.6.1"]
[ring/ring-defaults "0.3.2"]
[clj-http "3.10.1"]
[http.async.client "1.3.1"]
[hickory "0.7.1"]]
:plugins [[lein-ring "0.12.5"]]
:ring {:handler clojsa.handler/app}

37
resources/public/css/style.css
View File

@ -16,5 +16,40 @@ body {
nav.pagination {
background-color: #fff;
margin-top: 4rem;
margin-top: 2rem;
}
.userinfo > dd.registered {
font-size: 0.75rem;
}
.userinfo > dd.title {
margin-top: 1rem;
font-size: inherit;
font-weight: inherit;
line-height: inherit;
}
.content .bbc-block > h4 {
font-size: 1rem;
}
.bbc-spoiler {
transition: background-color 0.2s;
background-color: black;
color: black;
}
.bbc-spoiler:hover, .bbc-spoiler.active {
background-color: transparent;
color: auto;
}
.editedby {
font-size: .75rem;
font-style: italic;
margin-top: 1rem;
}
.postdate {
font-size: .75rem;
}

20
src/clojsa/handler.clj
View File

@ -1,15 +1,27 @@
(ns clojsa.handler
(:require [clojsa.views :refer [index-page thread-page]]
(:require [clojsa.views :as views]
[clojsa.saclient :as client]
[clojsa.saparser :as parser]
[compojure.core :refer :all]
[compojure.route :as route]
[compojure.coercions :refer [as-int]]
[ring.middleware.defaults :refer [wrap-defaults site-defaults]]
[ring.middleware.session :refer [wrap-session]]
[ring.middleware.session.cookie :refer (cookie-store)]))
(defn get-thread [session id page]
(let [turl (client/thread-url id page)
tresp (client/thread-response session turl)]
(parser/thread-map id page tresp)))
(defroutes app-routes
(GET "/" request (index-page request))
(GET "/thread/:id" [id page]
(thread-page (Integer/parseInt id) (Integer/parseInt page)))
(GET "/" request (views/index-page request))
(GET "/thread/:id" [id :<< as-int
page :<< as-int
:as {session :session}]
(let [thread (get-thread session id page)]
(views/thread-page thread)))
(route/not-found "Not Found"))
(def app

80
src/clojsa/saclient.clj
View File

@ -1,11 +1,10 @@
(ns clojsa.saclient
(:require [clojure.string :as string]
[clj-http.client :as client]
[hickory.core :refer :all]
[hickory.select :as s]
[hickory.convert :refer [hickory-to-hiccup]]))
[http.async.client :as http]))
(def url "https://forums.somethingawful.com/")
(defn thread-url
([id]
(thread-url id 1))
@ -15,67 +14,12 @@
:pagenumber page}]
{:href base-url :params query})))
(defn thread-response [url]
(let [resp (client/get (:href url) {:query-params (:params url)})]
(:body resp)))
(def witcher-thread (thread-response (thread-url 3720352 3)))
(defn hickory-doc [doc]
(-> doc parse as-hickory))
(defn parse-title [htree]
(-> (s/select (s/child (s/tag :title)) htree)
first :content first))
(defn parse-pagecount [htree]
(-> (s/select (s/descendant
(s/class :pages) (s/tag :option)) htree)
last :content first Integer/parseInt))
(defn parse-thread [htree]
(-> (s/select (s/descendant
(s/id :thread))
htree)
first))
(defn select-td [class-key htree]
(s/select (s/descendant
(s/and (s/tag :td) (s/class class-key))) htree))
(defn parse-ui [ui]
(let [ui (first (s/select (s/descendant (s/tag :dl)) ui))]
(hickory-to-hiccup ui)))
(defn parse-pd [pd]
(string/trim (last (hickory-to-hiccup pd))))
(defn hickory-div [class content]
{:type :element,
:attrs {:class class},
:tag :div,
:content content})
(defn parse-pb [pb]
(let [pb (-> pb :content)]
(hickory-to-hiccup (hickory-div "postbody" pb))))
(defn thread-map [id doc]
(let [htree (hickory-doc doc)
title (parse-title htree)
page-count (parse-pagecount htree)
thread-tree (parse-thread htree)
userinfo (select-td :userinfo thread-tree)
postdate (select-td :postdate thread-tree)
postbody (select-td :postbody thread-tree)]
{:title title
:id id
:page-count page-count
:content
(for [[ui pd pb] (partition 3 (interleave userinfo postdate postbody))
:when (not= "Adbot" (-> (s/select (s/child (s/class :author)) ui)
first :content first))]
{:ui (parse-ui ui) :pd (parse-pd pd) :pb (parse-pb pb)})}))
(defn get-thread [id page]
(thread-map id (thread-response (thread-url id page))))
(defn thread-response [session url]
(with-open [client (http/create-client)]
(let [{:keys [href params]} url
resp (http/GET client href :query params)
status (http/status resp)
headers (http/headers resp)]
(-> resp
http/await
http/string))))

63
src/clojsa/saparser.clj Normal file
View File

@ -0,0 +1,63 @@
(ns clojsa.saparser
(:require [clojsa.saclient :refer [thread-url thread-response]]
[clojure.string :as string]
[hickory.core :refer :all]
[hickory.select :as s]
[hickory.convert :refer [hickory-to-hiccup]]))
(defn hickory-doc [doc]
(-> doc parse as-hickory))
(defn parse-title [htree]
(-> (s/select (s/child (s/tag :title)) htree)
first :content first))
(defn parse-pagecount [htree]
(-> (s/select (s/descendant
(s/class :pages) (s/tag :option)) htree)
last :content first Integer/parseInt))
(defn parse-thread [htree]
(-> (s/select (s/descendant
(s/id :thread))
htree)
first))
(defn select-td [class-key htree]
(s/select (s/descendant
(s/and (s/tag :td) (s/class class-key))) htree))
(defn parse-ui [ui]
(let [ui (first (s/select (s/descendant (s/tag :dl)) ui))]
(hickory-to-hiccup ui)))
(defn parse-pd [pd]
(string/trim (last (hickory-to-hiccup pd))))
(defn hickory-div [class content]
{:type :element,
:attrs {:class class},
:tag :div,
:content content})
(defn parse-pb [pb]
(let [pb (-> pb :content)]
(hickory-to-hiccup (hickory-div "postbody" pb))))
(defn thread-map [id page doc]
(let [htree (hickory-doc doc)
title (parse-title htree)
page-count (parse-pagecount htree)
thread-tree (parse-thread htree)
userinfo (select-td :userinfo thread-tree)
postdate (select-td :postdate thread-tree)
postbody (select-td :postbody thread-tree)]
{:title title
:id id
:page page
:page-count page-count
:content
(for [[ui pd pb] (partition 3 (interleave userinfo postdate postbody))
:when (not= "Adbot" (-> (s/select (s/child (s/class :author)) ui)
first :content first))]
{:ui (parse-ui ui) :pd (parse-pd pd) :pb (parse-pb pb)})}))

18
src/clojsa/views.clj
View File

@ -1,7 +1,7 @@
(ns clojsa.views
(:use [hiccup core page])
(:require [clojsa.saclient :refer [get-thread]]
[clojure.string :as string]
(:require [clojure.string :as string]
[clojure.pprint]
[clojure.java.io :as io]
[cheshire.core :as json]))
@ -30,11 +30,10 @@
(include-js "/js/main.js")]))
(defn index-page [req]
(let [thread (get-thread 3720352 3)]
(main-template {:title (:title thread)}
[:div.container
(for [post (:content thread)]
[:div.post (:pb post)])])))
(main-template {}
[:div.container
[:pre
(clojure.pprint/pprint req)]]))
(defn paginate [id cur last]
[:nav.container.pagination {:hx-boot "false"}
@ -59,9 +58,8 @@
[:a.pagination-link
{:href (format "/thread/%d?page=%d" id last)} (str last)]]]])
(defn thread-page [id page]
(let [thread (get-thread id page)
{:keys [id title page-count content]} thread]
(defn thread-page [thread]
(let [{:keys [id title page page-count content]} thread]
(main-template
{:title title}
[:div.container