Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[#220] avoid resource leaking in scanners #236

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

[#220] avoid resource leaking in scanners #236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f7b8d9c
[#220] Avoid resource leaking in scanners
YuriRomanowski Dec 11, 2022
2f5950d
Add markdownParallelScanner for parallel file processing
YuriRomanowski Dec 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ library:
- reflection
- nyan-interpolation
- safe-exceptions
- parallel

executables:
xrefcheck:
Expand Down
8 changes: 5 additions & 3 deletions src/Xrefcheck/Scan.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
-
- SPDX-License-Identifier: MPL-2.0
-}

{-# LANGUAGE DeriveAnyClass #-}
{-# OPTIONS_GHC -Wno-orphans #-}

-- | Generalised repo scanner and analyser.
Expand Down Expand Up @@ -93,7 +93,8 @@ data ScanError = ScanError
{ sePosition :: Position
, seFile :: FilePath
, seDescription :: ScanErrorDescription
} deriving stock (Show, Eq)
} deriving stock (Show, Eq, Generic)
deriving anyclass NFData

instance Given ColorMode => Buildable ScanError where
build ScanError{..} = [int||
Expand All @@ -118,7 +119,8 @@ data ScanErrorDescription
| FileErr
| ParagraphErr Text
| UnrecognisedErr Text
deriving stock (Show, Eq)
deriving stock (Show, Eq, Generic)
deriving anyclass (NFData)
Copy link
Contributor

@aeqz aeqz Dec 13, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can derive NFData as in Xrefcheck.Core module if you prefer to follow that style:

instance NFData Position
instance NFData Reference
instance NFData AnchorType
instance NFData Anchor
instance NFData FileInfo
instance NFData RepoInfo


instance Buildable ScanErrorDescription where
build = \case
Expand Down
19 changes: 12 additions & 7 deletions src/Xrefcheck/Scanners/Markdown.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ module Xrefcheck.Scanners.Markdown

, defGithubMdConfig
, markdownScanner
, markdownParallelScanner
, markdownSupport
, parseFileInfo
, makeError
Expand All @@ -24,18 +25,18 @@ import CMarkGFM
import Control.Lens (_Just, makeLenses, makeLensesFor, (.=))
import Control.Monad.Trans.Writer.CPS (Writer, runWriter, tell)
import Data.Aeson (FromJSON (..), genericParseJSON)
import Data.ByteString.Lazy qualified as BSL
import Data.ByteString qualified as BS
import Data.DList qualified as DList
import Data.Default (def)
import Data.Text qualified as T
import Data.Text.Lazy qualified as LT
import Fmt (Buildable (..), nameF)
import Text.HTML.TagSoup
import Text.Interpolation.Nyan

import Xrefcheck.Core
import Xrefcheck.Scan
import Xrefcheck.Util
import Control.Parallel.Strategies

data MarkdownConfig = MarkdownConfig
{ mcFlavor :: Flavor
Expand Down Expand Up @@ -406,16 +407,20 @@ textToMode ("ignore" : [x])
| otherwise = InvalidMode x
textToMode _ = NotAnAnnotation

parseFileInfo :: MarkdownConfig -> FilePath -> LT.Text -> (FileInfo, [ScanError])
parseFileInfo :: MarkdownConfig -> FilePath -> T.Text -> (FileInfo, [ScanError])
parseFileInfo config fp input
= runWriter
$ flip runReaderT config
$ nodeExtractInfo fp
$ commonmarkToNode [optFootnotes] [extAutolink]
$ toStrict input
$ commonmarkToNode [optFootnotes] [extAutolink] input

markdownScanner :: MarkdownConfig -> ScanAction
markdownScanner config path = parseFileInfo config path . decodeUtf8 <$> BSL.readFile path
markdownScanner config path = parseFileInfo config path . decodeUtf8 <$> BS.readFile path

markdownParallelScanner :: MarkdownConfig -> ScanAction
markdownParallelScanner config path = do
resThunk <- parseFileInfo config path . decodeUtf8 <$> BS.readFile path
resThunk `usingIO` rparWith rdeepseq

markdownSupport :: MarkdownConfig -> ([Extension], ScanAction)
markdownSupport config = ([".md"], markdownScanner config)
markdownSupport config = ([".md"], markdownParallelScanner config)