my blog lives here now
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

553 lines
18 KiB

2 years ago
  1. {-# LANGUAGE BangPatterns #-}
  2. {-# LANGUAGE MultiWayIf #-}
  3. {-# LANGUAGE OverloadedStrings #-}
  4. {-# LANGUAGE BlockArguments #-}
  5. {-# LANGUAGE LambdaCase #-}
  6. {-# LANGUAGE StandaloneDeriving #-}
  7. {-# LANGUAGE DeriveAnyClass #-}
  8. import Control.DeepSeq (rnf)
  9. import Control.Concurrent
  10. import Control.Exception
  11. import Control.Monad
  12. import qualified Data.ByteString.Lazy.Char8 as BS
  13. import Data.ByteString.Lazy.Char8 (pack, unpack)
  14. import qualified Data.HashMap.Strict as HMap
  15. import qualified Data.Text.Encoding as T
  16. import qualified Data.Map.Strict as Map
  17. import qualified Data.Text as T
  18. import Data.Functor
  19. import Data.Monoid
  20. import Data.Binary
  21. import Data.Maybe
  22. import Data.Aeson
  23. import Data.List
  24. import Data.Char
  25. import Hakyll.Core.Compiler.Internal
  26. import Hakyll.Core.Compiler
  27. import Hakyll.Web.Sass
  28. import Hakyll
  29. import qualified Network.URI.Encode as URI (encode)
  30. import qualified Skylighting as Sky
  31. import System.Directory
  32. import System.Environment
  33. import System.Process
  34. import System.Exit
  35. import System.IO
  36. import Text.Pandoc.Walk (query, walkM, walk)
  37. import Text.Pandoc.Highlighting
  38. import Text.Pandoc.Definition
  39. import Text.Pandoc.Options
  40. import Text.Sass.Functions
  41. import Data.Text (Text)
  42. import Data.IORef
  43. import Data.Hashable (Hashable (hashWithSalt))
  44. import GHC.Stack
  45. import Text.Read (readMaybe)
  46. import GHC.Show (showCommaSpace)
  47. import Data.Traversable
  48. import qualified Data.Text.Lazy as LT
  49. readerOpts :: ReaderOptions
  50. readerOpts = def { readerExtensions = pandocExtensions
  51. , readerIndentedCodeClasses = ["amulet"] }
  52. writerOptions :: Compiler WriterOptions
  53. writerOptions = do
  54. syntaxMap <- loadAllSnapshots "syntax/*.xml" "syntax"
  55. <&> foldr (Sky.addSyntaxDefinition . itemBody) Sky.defaultSyntaxMap
  56. pure $ defaultHakyllWriterOptions
  57. { writerExtensions = extensionsFromList
  58. [ Ext_tex_math_dollars
  59. , Ext_tex_math_double_backslash
  60. , Ext_latex_macros
  61. ] <> writerExtensions defaultHakyllWriterOptions
  62. , writerSyntaxMap = syntaxMap
  63. , writerHighlightStyle = Just kate
  64. }
  65. rssfeed :: FeedConfiguration
  66. rssfeed
  67. = FeedConfiguration { feedTitle = "Amelia's Blag: Latest articles"
  68. , feedDescription = ""
  69. , feedAuthorName = "Amélia"
  70. , feedAuthorEmail = "[email protected]"
  71. , feedRoot = "https://amelia.how"
  72. }
  73. conf :: Configuration
  74. conf = def { destinationDirectory = ".site"
  75. , storeDirectory = ".store"
  76. , tmpDirectory = ".store/tmp"
  77. , deployCommand = "./sync" }
  78. katexFilter :: MVar KatexCache -> Pandoc -> Compiler Pandoc
  79. katexFilter cacheVar (Pandoc meta doc) =
  80. do
  81. id <- compilerUnderlying <$> compilerAsk
  82. t <- getMetadata id
  83. invalidateCache id (abbrevs t) cacheVar
  84. doc <- Pandoc meta <$> walkM (go (show id) (abbrevs t)) doc
  85. unsafeCompiler $ flushCache cacheVar
  86. pure doc
  87. where
  88. abbrevs :: HMap.HashMap Text Value -> [String]
  89. abbrevs x =
  90. case HMap.lookup "abbreviations" x of
  91. Just (Object map) -> concat $ mapMaybe oneAbbrev (HMap.toList map)
  92. _ -> []
  93. oneAbbrev (x, String t) = Just ["-m", '\\':T.unpack x ++ ':':T.unpack t]
  94. oneAbbrev _ = Nothing
  95. go :: String -> [String] -> Inline -> Compiler Inline
  96. go id abbrevs (Math kind math) = unsafeCompiler $ do
  97. cache <- readMVar cacheVar
  98. case HMap.lookup (id, kind, math) (spanMap cache) of
  99. Just x -> pure (RawInline "html" x)
  100. Nothing -> do
  101. let args = flip (:) abbrevs $ case kind of { DisplayMath -> "-td"; InlineMath -> "-t" }
  102. (contents, _) <- readProcessBS "katex" args . BS.fromStrict . T.encodeUtf8 $ math
  103. let text = T.init . T.init . T.decodeUtf8 . BS.toStrict $ contents
  104. modifyMVar cacheVar (\m -> pure (bumpCacheEntry cache id abbrevs kind text math, ()))
  105. pure $ RawInline "html" text
  106. go id _ x = pure x
  107. bumpCacheEntry (KatexCache spans depends abbrevVars) id abbrevs kind text math =
  108. let
  109. str = T.unpack text
  110. usedAbbrevs = map (\x -> (T.pack (takeWhile (/= ':') (tail x)), T.pack (tail (dropWhile (/= ':') (tail x)))))
  111. $ filter (\x -> (takeWhile (/= ':') (tail x)) `isInfixOf` str)
  112. $ filter (not . isPrefixOf "-") abbrevs
  113. addDeps [] x = x
  114. addDeps ((k, _):xs) vl = HMap.alter (\v -> Just (maybe [(id, kind, math)] ((id, kind, math):) v)) (id, k) $ addDeps xs vl
  115. recordVars [] x = x
  116. recordVars ((k, v):xs) x = HMap.insert (id, k) v (recordVars xs x)
  117. in
  118. case usedAbbrevs of
  119. [] -> KatexCache (HMap.insert (id, kind, math) text spans) depends abbrevVars
  120. xs -> KatexCache (HMap.insert (id, kind, math) text spans) (addDeps xs depends) (recordVars xs abbrevVars)
  121. abbreviationFilter :: Pandoc -> Compiler Pandoc
  122. abbreviationFilter (Pandoc meta doc) =
  123. do
  124. id <- compilerUnderlying <$> compilerAsk
  125. t <- getMetadata id
  126. case HMap.lookup "abbreviations" t of
  127. Just (Object map) -> do
  128. pure (Pandoc meta (walk (replace map) doc))
  129. _ -> pure (Pandoc meta doc)
  130. where
  131. replace map x =
  132. case x of
  133. Str t | Just (e, r) <- entity t -> fromMaybe (Str t) (toInline r =<< HMap.lookup e map)
  134. x -> x
  135. toInline r (String t) = Just (Str (t <> r))
  136. toInline _ _ = Nothing
  137. entity x
  138. | T.isPrefixOf "&" x && T.length x >= 3 =
  139. let
  140. (name, rest') = T.span (/= ';') (T.tail x)
  141. rest = T.tail rest'
  142. in pure (name, rest)
  143. | otherwise = Nothing
  144. addLanguageTag :: Pandoc -> Pandoc
  145. addLanguageTag (Pandoc meta blocks) = Pandoc meta (map go blocks) where
  146. go :: Block -> Block
  147. go block@(CodeBlock (identifier, classes@(language:classes'), kv) text) =
  148. Div
  149. ( mempty
  150. , "code-container":if haskv then "custom-tag":classes' else classes'
  151. , []
  152. )
  153. $ [block] ++ maybe [Div (mempty, ["code-tag"], []) [Plain [Span (mempty, [], []) [Str tag]]]]
  154. where
  155. language' = case T.uncons language of
  156. Nothing -> mempty
  157. Just (c, cs) -> T.cons (toUpper c) cs
  158. tag = fromMaybe language' (lookup "tag" kv)
  159. haskv = fromMaybe False (True <$ lookup "tag" kv)
  160. maybe
  161. | "notag" `elem` classes' = const []
  162. | otherwise = id
  163. go block@(CodeBlock (identifier, [], kv) text) =
  164. Div (mempty, ["code-container"], [])
  165. [block, Div (mempty, ["empty-code-tag"], []) []]
  166. go x = x
  167. saveSynopsys :: Pandoc -> Compiler Pandoc
  168. saveSynopsys (Pandoc meta doc) =
  169. do
  170. id <- getUnderlying
  171. n <- fromMaybe (1 :: Int) . readMaybe . fromMaybe "" . lookupString "synopsys" <$> getMetadata id
  172. case dropWhile (not . isParagraph) doc of
  173. p:ps -> do
  174. saveSnapshot "synopsys-block" =<< makeItem (map removeFootnotes (take n (p:ps)))
  175. pure ()
  176. [] -> pure ()
  177. pure $ Pandoc meta doc
  178. where
  179. isParagraph Para{} = True
  180. isParagraph _ = False
  181. removeFootnotes (Para xs) = Para $ filter (\case { Note _ -> False; _ -> True }) xs
  182. removeFootnotes x = x
  183. saveWordCount :: Pandoc -> Compiler Pandoc
  184. saveWordCount (Pandoc meta doc) =
  185. do
  186. saveSnapshot "wc" =<< makeItem wordCount
  187. pure $ Pandoc meta doc
  188. where
  189. wordCount = show (getSum (query inlineLen doc))
  190. inlineLen (Str s) = Sum (length (T.words s))
  191. inlineLen _ = mempty
  192. saveTableOfContents :: Pandoc -> Compiler Pandoc
  193. saveTableOfContents (Pandoc meta input) =
  194. do
  195. saveSnapshot "table-of-contents" =<< makeItem toc
  196. pure $ Pandoc meta (fixHeaders 0 doc)
  197. where
  198. headers = filter (\case { Header _ _ _ -> True; _ -> False }) doc
  199. doc = fixHeaders 0 input
  200. fixHeaders n (Header l (_, ms, mt) x:bs) =
  201. Header l (anchor, ms, mt) (Link (anchor, ms, mt) [] (T.singleton '#' <> anchor, mempty):x):fixHeaders (n + 1) bs where
  202. anchor = T.pack ("h" ++ show n)
  203. fixHeaders k (x:bs) = x:fixHeaders k bs
  204. fixHeaders _ [] = []
  205. into :: [Block] -> [[Block]]
  206. into (Header l m@(anchor, _, _) x:ms) =
  207. let
  208. contained (Header l' _ _) = l' > l
  209. contained _ = undefined
  210. (ours, rest) = span contained ms
  211. in [Para [Link (mempty, mempty, mempty) (tail x) (T.singleton '#' <> anchor, mempty)], list (into ours)]:into rest
  212. into [] = []
  213. into _ = undefined
  214. list = BulletList
  215. toc :: Block
  216. toc = list (into headers)
  217. setup :: IO (MVar KatexCache)
  218. setup = do
  219. setEnv "AMC_LIBRARY_PATH" "/usr/lib/amuletml/lib/"
  220. loadCache
  221. compiler :: MVar KatexCache -> Compiler (Item String)
  222. compiler katexCache = do
  223. wops <- writerOptions
  224. pandocCompilerWithTransformM readerOpts wops $
  225. katexFilter katexCache
  226. >=> abbreviationFilter
  227. >=> saveSynopsys
  228. >=> saveWordCount
  229. >=> saveTableOfContents
  230. main :: IO ()
  231. main = setup >>= \katexCache -> hakyllWith conf $ do
  232. match "static/*" do
  233. route idRoute
  234. compile copyFileCompiler
  235. match "static/**/*" $ do
  236. route idRoute
  237. compile copyFileCompiler
  238. match "css/**/*" $ do
  239. route idRoute
  240. compile copyFileCompiler
  241. match "css/*.css" $ do
  242. route idRoute
  243. compile copyFileCompiler
  244. match "css/*.scss" $ do
  245. route $ setExtension "css"
  246. compile $ do
  247. imports <- unsafeCompiler $ newMVar ([] :: [(String, String)])
  248. let add f p = modifyMVar imports (\x -> pure ((f, p):x, []))
  249. body <- sassCompilerWith def
  250. { sassOutputStyle = SassStyleCompressed
  251. , sassImporters = Just [ SassImporter 0 add ]
  252. }
  253. list <- unsafeCompiler $ takeMVar imports
  254. for list $ \(req, path) -> do
  255. load (fromFilePath ("css/" ++ reverse (dropWhile (/= '.') (reverse req)) ++ "scss"))
  256. :: Compiler (Item String)
  257. pure body
  258. match "diagrams/**/*.tex" $ do
  259. route $ setExtension "svg"
  260. compile $ getResourceBody
  261. >>= loadAndApplyTemplate "templates/tikz.tex" (bodyField "body")
  262. >>= withItemBody (return . pack
  263. >=> unixFilterLBS "rubber-pipe" ["--pdf"]
  264. >=> unixFilterLBS "pdftocairo" ["-svg", "-", "-"]
  265. >=> return . unpack)
  266. match "pages/posts/*" do
  267. route $ metadataRoute pathFromTitle
  268. compile $ do
  269. wops <- writerOptions
  270. id <- getUnderlying
  271. r <- compiler katexCache
  272. >>= loadAndApplyTemplate "templates/post.html" postCtx
  273. >>= saveSnapshot "content"
  274. >>= loadAndApplyTemplate "templates/default.html" postCtx
  275. >>= relativizeUrls
  276. loadSnapshot id "synopsys-block"
  277. >>= saveSnapshot "synopsys-text"
  278. . writePandocWith wops
  279. . fmap (Pandoc mempty)
  280. pure r
  281. match "pages/posts/*.lhs" $ version "raw" $ do
  282. route idRoute
  283. compile copyFileCompiler
  284. create ["archive.html"] $ do
  285. route idRoute
  286. compile $ do
  287. posts <- recentFirst =<< onlyPublic =<< loadAll ("pages/posts/*" .&&. hasNoVersion)
  288. let archiveCtx =
  289. listField "posts" postCtx (return posts) <>
  290. constField "title" "Archives" <>
  291. defaultContext
  292. makeItem ""
  293. >>= loadAndApplyTemplate "templates/archive.html" archiveCtx
  294. >>= loadAndApplyTemplate "templates/default.html" archiveCtx
  295. >>= relativizeUrls
  296. match "pages/*.html" $ do
  297. route $ gsubRoute "pages/" (const "")
  298. compile $ do
  299. posts <- fmap (take 5) . recentFirst =<< onlyPublic =<< loadAll ("pages/posts/*" .&&. hasNoVersion)
  300. let indexCtx =
  301. listField "posts" postCtx (return posts) <>
  302. constField "title" "Home" <>
  303. defaultContext
  304. getResourceBody
  305. >>= applyAsTemplate indexCtx
  306. >>= loadAndApplyTemplate "templates/default.html" indexCtx
  307. >>= relativizeUrls
  308. match "pages/*.md" $ do
  309. route $ setExtension "html" <> gsubRoute "pages/" (const "")
  310. compile $ compiler katexCache
  311. >>= loadAndApplyTemplate "templates/page.html" defaultContext
  312. >>= loadAndApplyTemplate "templates/default.html" defaultContext
  313. >>= relativizeUrls
  314. match "syntax/*.xml" $ compile $ do
  315. path <- toFilePath <$> getUnderlying
  316. contents <- itemBody <$> getResourceBody
  317. debugCompiler ("Loaded syntax definition from " ++ show path)
  318. let res = Sky.parseSyntaxDefinitionFromText path (LT.pack contents)
  319. _ <- saveSnapshot "syntax" =<< case res of
  320. Left e -> fail e
  321. Right x -> makeItem x
  322. makeItem contents
  323. match "templates/*" $ compile templateBodyCompiler
  324. create ["feed.xml"] $ do
  325. route idRoute
  326. compile $ do
  327. let feedCtx = postCtx <> bodyField "description"
  328. posts <- fmap (take 10) . recentFirst =<< onlyPublic =<< loadAllSnapshots ("pages/posts/*" .&&. hasNoVersion) "synopsys-text"
  329. renderRss rssfeed feedCtx posts
  330. onlyPublic :: [Item String] -> Compiler [Item String]
  331. onlyPublic = filterM isPublic where
  332. isPublic item = do
  333. t <- getMetadata (itemIdentifier item)
  334. case HMap.lookup "public" t of
  335. Just (Bool False) -> pure False
  336. _ -> pure True
  337. postCtx :: Context String
  338. postCtx =
  339. dateField "date" "%B %e, %Y"
  340. <> snapshotField "synopsys" "synopsys-text"
  341. <> snapshotField "words" "wc"
  342. <> snapshotField' render "toc" "table-of-contents"
  343. <> defaultContext
  344. where
  345. snapshotField = snapshotField' pure
  346. snapshotField' f key snap = field key $ \x -> do
  347. let id = itemIdentifier x
  348. fmap itemBody . f =<< loadSnapshot id snap
  349. render x = do
  350. wops <- writerOptions
  351. pure . writePandocWith wops . fmap (Pandoc mempty . pure) $ x
  352. readProcessBS :: FilePath -> [String] -> BS.ByteString -> IO (BS.ByteString, String)
  353. readProcessBS path args input =
  354. let process = (proc path args)
  355. { std_in = CreatePipe
  356. , std_out = CreatePipe
  357. , std_err = CreatePipe
  358. }
  359. in withCreateProcess process $ \stdin stdout stderr ph ->
  360. case (stdin, stdout, stderr) of
  361. (Nothing, _, _) -> fail "Failed to get a stdin handle."
  362. (_, Nothing, _) -> fail "Failed to get a stdout handle."
  363. (_, _, Nothing) -> fail "Failed to get a stderr handle."
  364. (Just stdin, Just stdout, Just stderr) -> do
  365. out <- BS.hGetContents stdout
  366. err <- hGetContents stderr
  367. withForkWait (evaluate $ rnf out) $ \waitOut ->
  368. withForkWait (evaluate $ rnf err) $ \waitErr -> do
  369. -- Write input and close.
  370. BS.hPutStr stdin input
  371. hClose stdin
  372. -- wait on the output
  373. waitOut
  374. waitErr
  375. hClose stdout
  376. hClose stderr
  377. -- wait on the process
  378. ex <- waitForProcess ph
  379. case ex of
  380. ExitSuccess -> pure (out, err)
  381. ExitFailure ex -> fail (err ++ "Exited with " ++ show ex)
  382. where
  383. withForkWait :: IO () -> (IO () -> IO a) -> IO a
  384. withForkWait async body = do
  385. waitVar <- newEmptyMVar :: IO (MVar (Either SomeException ()))
  386. mask $ \restore -> do
  387. tid <- forkIO $ try (restore async) >>= putMVar waitVar
  388. let wait = takeMVar waitVar >>= either throwIO return
  389. restore (body wait) `onException` killThread tid
  390. pathFromTitle :: Metadata -> Routes
  391. pathFromTitle meta =
  392. let
  393. declaredCategory =
  394. case lookupString "category" meta of
  395. Just s -> ((s ++ "/") ++)
  396. Nothing -> ("posts/" <>)
  397. !titleString =
  398. case lookupString "title" meta of
  399. Just s -> s
  400. Nothing -> error "post has no title?"
  401. title = filter (/= "") . map (filter isAlphaNum . map toLower) . words $ titleString
  402. (category, title') =
  403. if | "or" `elem` title -> (declaredCategory, takeWhile (/= "or") title)
  404. | ["a", "quickie"] `isPrefixOf` title -> (("quick/" ++), drop 2 title)
  405. | otherwise -> (declaredCategory, title)
  406. in
  407. case lookupString "path" meta of
  408. Just p -> constRoute (category (p <> ".html"))
  409. Nothing -> constRoute (category (intercalate "-" title' <> ".html"))
  410. foldMapM :: (Monad w, Monoid m, Foldable f) => (a -> w m) -> f a -> w m
  411. foldMapM k = foldr (\x y -> do { m <- k x; (m <>) <$> y }) (pure mempty)
  412. loadCache :: HasCallStack => IO (MVar KatexCache)
  413. loadCache = do
  414. t <- doesFileExist ".katex_cache"
  415. let fixup (a, b, c) = KatexCache (HMap.fromList a) (HMap.fromList b) (HMap.fromList c)
  416. map <- if t
  417. then (fixup <$> decodeFile ".katex_cache") `catch` \e ->
  418. const (print e *> pure (KatexCache mempty mempty mempty)) (e :: SomeException)
  419. else pure (KatexCache mempty mempty mempty)
  420. var <- newMVar map
  421. pure var
  422. flushCache :: MVar KatexCache -> IO ()
  423. flushCache var = do
  424. withMVar var $ \(KatexCache x y z) -> do
  425. Data.Binary.encodeFile ".katex_cache" (HMap.toList x, HMap.toList y, HMap.toList z)
  426. invalidateCache :: Identifier -> [String] -> MVar KatexCache -> Compiler KatexCache
  427. invalidateCache id abbrevs cacheVar = unsafeCompiler $ modifyMVar cacheVar (\x -> pure (go x, go x)) where
  428. currentValues = map (\x -> (T.pack (takeWhile (/= ':') (tail x)), T.pack (tail (dropWhile (/= ':') (tail x)))))
  429. $ filter (not . isPrefixOf "-") abbrevs
  430. ident = show id
  431. go (KatexCache spanMap abbrKeys abbrText) =
  432. let
  433. go (abbr, val) (spanMap, abbrKeys, abbrText) =
  434. case HMap.lookup (ident, abbr) abbrText of
  435. Just vl | vl /= val ->
  436. let l = HMap.lookupDefault [] (ident, abbr) abbrKeys
  437. in (foldr HMap.delete spanMap l, abbrKeys, abbrText)
  438. _ -> (spanMap, abbrKeys, abbrText)
  439. (a, b, c) = foldr go (spanMap, abbrKeys, abbrText) currentValues
  440. in KatexCache a b c
  441. data KatexCache
  442. = KatexCache { spanMap :: HMap.HashMap (String, MathType, Text) Text
  443. , abbreviationKeys :: HMap.HashMap (String, Text) [(String, MathType, Text)]
  444. , abbreviationText :: HMap.HashMap (String, Text) Text
  445. }
  446. instance Hashable MathType where
  447. hashWithSalt s DisplayMath = hashWithSalt s (0 :: Int)
  448. hashWithSalt s InlineMath = hashWithSalt s (1 :: Int)
  449. deriving instance Binary Block
  450. deriving instance Binary Inline
  451. deriving instance Binary Format
  452. deriving instance Binary ListNumberStyle
  453. deriving instance Binary ListNumberDelim
  454. deriving instance Binary Caption
  455. deriving instance Binary Alignment
  456. deriving instance Binary ColWidth
  457. deriving instance Binary TableHead
  458. deriving instance Binary TableBody
  459. deriving instance Binary TableFoot
  460. deriving instance Binary QuoteType
  461. deriving instance Binary Citation
  462. deriving instance Binary Row
  463. deriving instance Binary MathType
  464. deriving instance Binary RowHeadColumns
  465. deriving instance Binary CitationMode
  466. deriving instance Binary Cell
  467. deriving instance Binary RowSpan
  468. deriving instance Binary ColSpan