Browse Source

First commit

master
Stephen Starkey 2 years ago
commit
264d7d5f6f
  1. 18
      .gitignore
  2. 42
      README.md
  3. 11
      env/dev/clj/email_counter/dev_middleware.clj
  4. 15
      env/dev/clj/email_counter/env.clj
  5. 31
      env/dev/clj/user.clj
  6. 1
      env/dev/resources/config.edn
  7. 33
      env/dev/resources/logback.xml
  8. 11
      env/prod/clj/email_counter/env.clj
  9. 2
      env/prod/resources/config.edn
  10. 24
      env/prod/resources/logback.xml
  11. 1
      env/test/resources/config.edn
  12. 33
      env/test/resources/logback.xml
  13. 71
      project.clj
  14. BIN
      resources/public/favicon.ico
  15. BIN
      resources/public/img/warning_clojure.png
  16. 13
      src/clj/email_counter/config.clj
  17. 58
      src/clj/email_counter/core.clj
  18. 33
      src/clj/email_counter/emails.clj
  19. 29
      src/clj/email_counter/handler.clj
  20. 13
      src/clj/email_counter/middleware.clj
  21. 35
      src/clj/email_counter/middleware/exception.clj
  22. 15
      src/clj/email_counter/middleware/formats.clj
  23. 27
      src/clj/email_counter/nrepl.clj
  24. 58
      src/clj/email_counter/routes/services.clj
  25. 31
      test/clj/email_counter/test/emails.clj
  26. 70
      test/clj/email_counter/test/handler.clj

18
.gitignore

@ -0,0 +1,18 @@
/target
/lib
/classes
/checkouts
pom.xml
dev-config.edn
test-config.edn
*.jar
*.class
/.lein-*
profiles.clj
/.env
.nrepl-port
/node_modules
/log
*.iml
.idea

42
README.md

@ -0,0 +1,42 @@
# email-counter
This project is used to demonstrate how to build a simple web service
which, when given a list of email addresses, will return the total
number of unique instances of each address, after stripping out
content that Gmail would normally ignore.
Given the peculiarities of the clojure web application landscape,
quite a lot of this project is generated boilerplate using the
Luminus micro-framework. You can learn more at
<http://www.luminusweb.net/>
This particular service was generated using Luminus version 3.48
## Prerequisites
You will need [Leiningen][1] 2.0 or above installed.
[1]: https://github.com/technomancy/leiningen
## Developing
Unit tests can be found in
test/clj/email_counter/test
Run the unit tests using:
lein test
Main API routes can be found in
src/clj/email_counter/routes/services.clj
## Running
To start a web server for the application, run:
lein run
Then, after the application has started, open a web browser to
<http://localhost:3000>

11
env/dev/clj/email_counter/dev_middleware.clj

@ -0,0 +1,11 @@
(ns email-counter.dev-middleware
(:require
[ring.middleware.reload :refer [wrap-reload]]
[selmer.middleware :refer [wrap-error-page]]
[prone.middleware :refer [wrap-exceptions]]))
(defn wrap-dev [handler]
(-> handler
wrap-reload
wrap-error-page
(wrap-exceptions {:app-namespaces ['email-counter]})))

15
env/dev/clj/email_counter/env.clj

@ -0,0 +1,15 @@
(ns email-counter.env
(:require
[selmer.parser :as parser]
[clojure.tools.logging :as log]
[email-counter.dev-middleware :refer [wrap-dev]]))
(def defaults
{:init
(fn []
(parser/cache-off!)
(log/info "\n-=[email-counter started successfully using the development profile]=-"))
:stop
(fn []
(log/info "\n-=[email-counter has shut down successfully]=-"))
:middleware wrap-dev})

31
env/dev/clj/user.clj

@ -0,0 +1,31 @@
(ns user
"Userspace functions you can run by default in your local REPL."
(:require
[email-counter.config :refer [env]]
[clojure.spec.alpha :as s]
[expound.alpha :as expound]
[mount.core :as mount]
[email-counter.core :refer [start-app]]))
(alter-var-root #'s/*explain-out* (constantly expound/printer))
(add-tap (bound-fn* clojure.pprint/pprint))
(defn start
"Starts application.
You'll usually want to run this on startup."
[]
(mount/start-without #'email-counter.core/repl-server))
(defn stop
"Stops application."
[]
(mount/stop-except #'email-counter.core/repl-server))
(defn restart
"Restarts application."
[]
(stop)
(start))

1
env/dev/resources/config.edn

@ -0,0 +1 @@
{}

33
env/dev/resources/logback.xml

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="10 seconds">
<statusListener class="ch.qos.logback.core.status.NopStatusListener" />
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<charset>UTF-8</charset>
<pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg %n</pattern>
</encoder>
</appender>
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>log/email-counter.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>log/email-counter.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>100MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<!-- keep 30 days of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder>
<charset>UTF-8</charset>
<pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg %n</pattern>
</encoder>
</appender>
<logger name="org.apache.http" level="warn" />
<logger name="org.xnio.nio" level="warn" />
<root level="DEBUG">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>

11
env/prod/clj/email_counter/env.clj

@ -0,0 +1,11 @@
(ns email-counter.env
(:require [clojure.tools.logging :as log]))
(def defaults
{:init
(fn []
(log/info "\n-=[email-counter started successfully]=-"))
:stop
(fn []
(log/info "\n-=[email-counter has shut down successfully]=-"))
:middleware identity})

2
env/prod/resources/config.edn

@ -0,0 +1,2 @@
{:prod true
:port 3000}

24
env/prod/resources/logback.xml

@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<statusListener class="ch.qos.logback.core.status.NopStatusListener" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>log/email-counter.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>log/email-counter.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>100MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<!-- keep 30 days of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder>
<charset>UTF-8</charset>
<pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg %n</pattern>
</encoder>
</appender>
<logger name="org.apache.http" level="warn" />
<logger name="org.xnio.nio" level="warn" />
<root level="INFO">
<appender-ref ref="FILE" />
</root>
</configuration>

1
env/test/resources/config.edn

@ -0,0 +1 @@
{}

33
env/test/resources/logback.xml

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="10 seconds">
<statusListener class="ch.qos.logback.core.status.NopStatusListener" />
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<charset>UTF-8</charset>
<pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg %n</pattern>
</encoder>
</appender>
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>log/email-counter.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>log/email-counter.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>100MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<!-- keep 30 days of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder>
<charset>UTF-8</charset>
<pattern>%date{ISO8601} [%thread] %-5level %logger{36} - %msg %n</pattern>
</encoder>
</appender>
<logger name="org.apache.http" level="warn" />
<logger name="org.xnio.nio" level="warn" />
<root level="DEBUG">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>

71
project.clj

@ -0,0 +1,71 @@
(defproject email-counter "0.1.0-SNAPSHOT"
:description "FIXME: write description"
:url "http://example.com/FIXME"
:dependencies [[ch.qos.logback/logback-classic "1.2.3"]
[cheshire "5.8.1"]
[clojure.java-time "0.3.2"]
[com.sun.mail/javax.mail "1.6.0"]
[cprop "0.1.14"]
[expound "0.7.2"]
[funcool/struct "1.4.0"]
[javax.mail/javax.mail-api "1.6.2"]
[javax.mail/mail "1.4"]
[luminus-http-kit "0.1.6"]
[luminus-transit "0.1.1"]
[luminus/ring-ttl-session "0.3.3"]
[markdown-clj "1.10.0"]
[metosin/muuntaja "0.6.4"]
[metosin/reitit "0.3.9"]
[metosin/ring-http-response "0.9.1"]
[mount "0.1.16"]
[nrepl "0.6.0"]
[org.clojure/clojure "1.10.1"]
[org.clojure/tools.cli "0.4.2"]
[org.clojure/tools.logging "0.5.0"]
[org.webjars.npm/bulma "0.7.5"]
[org.webjars.npm/material-icons "0.3.0"]
[org.webjars/webjars-locator "0.36"]
[prismatic/schema "1.1.12"]
[ring-webjars "0.2.0"]
[ring/ring-core "1.7.1"]
[ring/ring-defaults "0.3.2"]
[selmer "1.12.14"]]
:min-lein-version "2.0.0"
:source-paths ["src/clj"]
:test-paths ["test/clj"]
:resource-paths ["resources"]
:target-path "target/%s/"
:main ^:skip-aot email-counter.core
:plugins []
:profiles
{:uberjar {:omit-source true
:aot :all
:uberjar-name "email-counter.jar"
:source-paths ["env/prod/clj"]
:resource-paths ["env/prod/resources"]}
:dev [:project/dev :profiles/dev]
:test [:project/dev :project/test :profiles/test]
:project/dev {:jvm-opts ["-Dconf=dev-config.edn"]
:dependencies [[pjstadig/humane-test-output "0.9.0"]
[prone "2019-07-08"]
[ring/ring-devel "1.7.1"]
[ring/ring-mock "0.4.0"]]
:plugins [[com.jakemccrary/lein-test-refresh "0.24.1"]]
:source-paths ["env/dev/clj"]
:resource-paths ["env/dev/resources"]
:repl-options {:init-ns user}
:injections [(require 'pjstadig.humane-test-output)
(pjstadig.humane-test-output/activate!)]}
:project/test {:jvm-opts ["-Dconf=test-config.edn"]
:resource-paths ["env/test/resources"]}
:profiles/dev {}
:profiles/test {}})

BIN
resources/public/favicon.ico

BIN
resources/public/img/warning_clojure.png

After

Width: 165  |  Height: 256  |  Size: 21 KiB

13
src/clj/email_counter/config.clj

@ -0,0 +1,13 @@
(ns email-counter.config
(:require
[cprop.core :refer [load-config]]
[cprop.source :as source]
[mount.core :refer [args defstate]]))
(defstate env
:start
(load-config
:merge
[(args)
(source/from-system-props)
(source/from-env)]))

58
src/clj/email_counter/core.clj

@ -0,0 +1,58 @@
(ns email-counter.core
(:require
[email-counter.handler :as handler]
[email-counter.nrepl :as nrepl]
[luminus.http-server :as http]
[email-counter.config :refer [env]]
[clojure.tools.cli :refer [parse-opts]]
[clojure.tools.logging :as log]
[mount.core :as mount])
(:gen-class))
;; log uncaught exceptions in threads
(Thread/setDefaultUncaughtExceptionHandler
(reify Thread$UncaughtExceptionHandler
(uncaughtException [_ thread ex]
(log/error {:what :uncaught-exception
:exception ex
:where (str "Uncaught exception on" (.getName thread))}))))
(def cli-options
[["-p" "--port PORT" "Port number"
:parse-fn #(Integer/parseInt %)]])
(mount/defstate ^{:on-reload :noop} http-server
:start
(http/start
(-> env
(assoc :handler (handler/app))
(update :io-threads #(or % (* 2 (.availableProcessors (Runtime/getRuntime)))))
(update :port #(or (-> env :options :port) %))))
:stop
(http/stop http-server))
(mount/defstate ^{:on-reload :noop} repl-server
:start
(when (env :nrepl-port)
(nrepl/start {:bind (env :nrepl-bind)
:port (env :nrepl-port)}))
:stop
(when repl-server
(nrepl/stop repl-server)))
(defn stop-app []
(doseq [component (:stopped (mount/stop))]
(log/info component "stopped"))
(shutdown-agents))
(defn start-app [args]
(doseq [component (-> args
(parse-opts cli-options)
mount/start-with-args
:started)]
(log/info component "started"))
(.addShutdownHook (Runtime/getRuntime) (Thread. stop-app)))
(defn -main [& args]
(start-app args))

33
src/clj/email_counter/emails.clj

@ -0,0 +1,33 @@
(ns email-counter.emails
(:require [clojure.string :as str]
[clojure.tools.logging :as log])
(:import (javax.mail.internet AddressException InternetAddress)))
(defn parse-email
"Attempt to get the address portion of an assumed rfc822-compliant email
address. If we can't, we return nil."
[s]
(try
(when s (-> s InternetAddress. .getAddress))
(catch AddressException _)))
(defn parse-for-gmail
"Given a valid email address, strip any periods and any content after a +
from the name portion. If the email is nil, we will simply return nil"
[email]
(let [email (parse-email email)
[_ n d] (when email (re-matches #"^(.*?)(@[^@]+$)" email))]
(some-> n
(str/replace #"\." "")
(str/replace #"\+.*" "")
(str d))))
(defn unique-gmail-parsed-emails
"Given a bunch of email addresses, strip out the bits Gmail would have
stripped out, lower-case them, and then add them to a hash set"
[emails]
(->> emails
(map parse-for-gmail)
(remove nil?)
(map str/lower-case)
(into #{})))

29
src/clj/email_counter/handler.clj

@ -0,0 +1,29 @@
(ns email-counter.handler
(:require
[email-counter.middleware :as middleware]
[email-counter.routes.services :refer [service-routes]]
[reitit.ring :as ring]
[ring.middleware.content-type :refer [wrap-content-type]]
[ring.middleware.webjars :refer [wrap-webjars]]
[email-counter.env :refer [defaults]]
[mount.core :as mount]))
(mount/defstate init-app
:start ((or (:init defaults) (fn [])))
:stop ((or (:stop defaults) (fn []))))
(mount/defstate app-routes
:start
(ring/ring-handler
(ring/router
[["/" {:get
{:handler (constantly {:status 301 :headers {"Location" "/api/api-docs/index.html"}})}}]
(service-routes)])
(ring/routes
(ring/create-resource-handler
{:path "/"})
(wrap-content-type (wrap-webjars (constantly nil)))
(ring/create-default-handler))))
(defn app []
(middleware/wrap-base #'app-routes))

13
src/clj/email_counter/middleware.clj

@ -0,0 +1,13 @@
(ns email-counter.middleware
(:require
[email-counter.env :refer [defaults]]
[email-counter.config :refer [env]]
[ring-ttl-session.core :refer [ttl-memory-store]]
[ring.middleware.defaults :refer [site-defaults wrap-defaults]]))
(defn wrap-base [handler]
(-> ((:middleware defaults) handler)
(wrap-defaults
(-> site-defaults
(assoc-in [:security :anti-forgery] false)
(assoc-in [:session :store] (ttl-memory-store (* 60 30)))))))

35
src/clj/email_counter/middleware/exception.clj

@ -0,0 +1,35 @@
(ns email-counter.middleware.exception
(:require [clojure.tools.logging :as log]
[expound.alpha :as expound]
[reitit.coercion :as coercion]
[reitit.ring.middleware.exception :as exception]))
(defn render-error [printer status & [exception]]
{:status status
:headers {"Content-Type" "text/html"}
:body (with-out-str (printer (-> exception ex-data :problems)))})
(defn coercion-error-handler [printer status]
(partial render-error printer status))
(defn log-error [e]
(log/error e (.getMessage e)))
(def exception-middleware
(let [printer (expound/custom-printer {:print-specs? false})]
(exception/create-exception-middleware
(merge
exception/default-handlers
{;; log stack-traces for all exceptions
::exception/wrap (fn [handler e request]
(if (= :reitit.coercion/request-coercion
(:type (ex-data e)))
{:status 400
:headers {"Content-Type" "application/edn"}
:body (pr-str (-> e ex-data :errors))}
(do
(log-error e)
(handler e request))))
;; human-optimized validation messages
::coercion/request-coercion (coercion-error-handler printer 400)
::coercion/response-coercion (coercion-error-handler printer 500)}))))

15
src/clj/email_counter/middleware/formats.clj

@ -0,0 +1,15 @@
(ns email-counter.middleware.formats
(:require
[cognitect.transit :as transit]
[luminus-transit.time :as time]
[muuntaja.core :as m]))
(def instance
(m/create
(-> m/default-options
(update-in
[:formats "application/transit+json" :decoder-opts]
(partial merge time/time-deserialization-handlers))
(update-in
[:formats "application/transit+json" :encoder-opts]
(partial merge time/time-serialization-handlers)))))

27
src/clj/email_counter/nrepl.clj

@ -0,0 +1,27 @@
(ns email-counter.nrepl
(:require
[nrepl.server :as nrepl]
[clojure.tools.logging :as log]))
(defn start
"Start a network repl for debugging on specified port followed by
an optional parameters map. The :bind, :transport-fn, :handler,
:ack-port and :greeting-fn will be forwarded to
clojure.tools.nrepl.server/start-server as they are."
[{:keys [port bind transport-fn handler ack-port greeting-fn]}]
(try
(log/info "starting nREPL server on port" port)
(nrepl/start-server :port port
:bind bind
:transport-fn transport-fn
:handler handler
:ack-port ack-port
:greeting-fn greeting-fn)
(catch Throwable t
(log/error t "failed to start nREPL")
(throw t))))
(defn stop [server]
(nrepl/stop-server server)
(log/info "nREPL server stopped"))

58
src/clj/email_counter/routes/services.clj

@ -0,0 +1,58 @@
(ns email-counter.routes.services
(:require
[email-counter.emails :as emails]
[email-counter.middleware.formats :as formats]
[email-counter.middleware.exception :as exception]
[reitit.coercion.schema :as schema-coercion]
[reitit.ring.coercion :as coercion]
[reitit.ring.middleware.multipart :as multipart]
[reitit.ring.middleware.muuntaja :as muuntaja]
[reitit.ring.middleware.parameters :as parameters]
[reitit.swagger :as swagger]
[reitit.swagger-ui :as swagger-ui]
[ring.util.http-response :refer :all]
[schema.core :as s]))
(def NaturalNumber (s/constrained s/Int (complement neg?) 'NaturalNumber))
(def EmailAddress (s/constrained s/Str emails/parse-for-gmail 'EmailAddress))
(defn count-emails [{{{:keys [emails]} :body} :parameters}]
(ok {:total (count (emails/unique-gmail-parsed-emails emails))}))
(defn service-routes []
["/api"
{:coercion schema-coercion/coercion
:muuntaja formats/instance
:swagger {:id ::api}
:middleware [parameters/parameters-middleware
muuntaja/format-negotiate-middleware
muuntaja/format-response-middleware
exception/exception-middleware
muuntaja/format-request-middleware
coercion/coerce-response-middleware
coercion/coerce-request-middleware
multipart/multipart-middleware]}
["" {:no-doc true
:swagger {:info {:title "email-count"}}}
["/swagger.json"
{:get (swagger/create-swagger-handler)}]
["/api-docs/*"
{:get (swagger-ui/create-swagger-ui-handler
{:url "/api/swagger.json"
:config {:validator-url nil}})}]]
["/ping"
{:get (constantly (ok {:message "pong"}))}]
["/v1"
["/emailcount"
{:post {:summary "plus with body parameters"
:description (str "Given a list of email addresses, count how "
"many unique email addresses show up, after "
"accounting for Gmail's unique approach to "
"parsing.")
:parameters {:body {:emails [EmailAddress]}}
:responses {200 {:body {:total NaturalNumber}}}
:handler count-emails}}]]])

31
test/clj/email_counter/test/emails.clj

@ -0,0 +1,31 @@
(ns email-counter.test.emails
(:require [clojure.test :refer :all]
[email-counter.emails :refer :all]))
(deftest parse-email-test
(are [expected given] (= expected (parse-email given))
nil nil
nil ""
"me@here.com" "me@here.com "
"m.e@here.com" "\n\t m.e@here.com"
"me@here.com" "Me <me@here.com>"))
(deftest parse-for-gmail-test
(are [expected given] (= expected (parse-for-gmail given))
nil nil
nil ""
"me@here.com" "me@here.com"
"me@here.com" "m.e@here.com"
"me@here.com" "Me <me@here.com>"))
(deftest unique-gmail-parsed-emails-test
(is (= #{"me@here.com"
"you@there.com"}
(unique-gmail-parsed-emails
["me@here.com"
"m.e@here.com"
"Me <me@here.com>"
"Me <ME@HERE.COM>"
"you@there.com"
""
nil]))))

70
test/clj/email_counter/test/handler.clj

@ -0,0 +1,70 @@
(ns email-counter.test.handler
(:require
[clojure.test :refer :all]
[ring.mock.request :refer :all]
[email-counter.handler :refer :all]
[muuntaja.core :as m]
[mount.core :as mount]
[clojure.string :as str]
[email-counter.middleware.exception :as exception]))
(use-fixtures
:once
(fn [f]
(mount/start #'email-counter.config/env
#'email-counter.handler/app-routes)
(f)
(mount/stop)))
(defn decode-response-body [{{:strs [Content-Type]} :headers :as response}]
(case (first (str/split Content-Type #";"))
"application/json"
(m/decode-response-body response)
"application/edn"
(:body response)
(pr-str response)))
(defn post-body [url body]
(let [response ((app) (-> (request :post url)
(json-body body)))]
(-> response
(assoc :body (decode-response-body response))
(dissoc :headers))))
(defn request-emails [emails]
(post-body "/api/v1/emailcount" {:emails emails}))
(defn print-exceptions [exceptions-encountered]
(doseq [e @exceptions-encountered]
(.printStackTrace e)))
(deftest email-count
(let [exceptions-encountered (atom [])
print-exceptions (partial print-exceptions exceptions-encountered)]
(with-redefs [exception/log-error
(fn [e]
(swap! exceptions-encountered conj e))]
(testing "emailcount"
(is (= {:status 200, :body {:total 2}}
(request-emails ["test.email@gmail.com"
"test.email+spam@gmail.com"
"Test Email <TestEmail@gmail.com>"
"scstarkey@gmail.com"]))))
(testing "invalid email address"
(is (= {:status 400,
:body "{:emails [(not (EmailAddress \"busted\"))]}"}
(request-emails ["busted"]))))
(testing "invalid request"
(is (= {:status 400,
:body "{:emails missing-required-key, :test disallowed-key}"}
(post-body "/api/v1/emailcount" {:test "stuff"}))))
(testing "no exceptions were actually encountered"
(is (= 0 (count @exceptions-encountered))
(print-exceptions))))))
(clojure.test/run-tests)
Loading…
Cancel
Save