diff --git a/cabal2nix.nix b/cabal2nix.nix index 7959f94..063001b 100644 --- a/cabal2nix.nix +++ b/cabal2nix.nix @@ -1,8 +1,8 @@ { mkDerivation, aeson, array, attoparsec, base, bytestring -, containers, directory, generics-sop, hspec, http-conduit +, containers, directory, either, generics-sop, hspec, http-conduit , monad-control, mtl, optparse-applicative, QuickCheck, reflection -, regex-tdfa, resource-pool, sqlite-simple, stdenv, text, time -, transformers-base, universum, unliftio, zip-archive +, regex-tdfa, resource-pool, safe-exceptions, sqlite-simple, stdenv +, text, time, transformers-base, universum, unliftio, zip-archive }: mkDerivation { pname = "log-classifier"; @@ -11,10 +11,11 @@ mkDerivation { isLibrary = true; isExecutable = true; libraryHaskellDepends = [ - aeson array attoparsec base bytestring containers directory + aeson array attoparsec base bytestring containers directory either generics-sop http-conduit monad-control mtl optparse-applicative - QuickCheck reflection regex-tdfa resource-pool sqlite-simple text - time transformers-base universum unliftio zip-archive + QuickCheck reflection regex-tdfa resource-pool safe-exceptions + sqlite-simple text time transformers-base universum unliftio + zip-archive ]; executableHaskellDepends = [ base universum ]; testHaskellDepends = [ diff --git a/log-classifier.cabal b/log-classifier.cabal index 883e652..23a62d5 100644 --- a/log-classifier.cabal +++ b/log-classifier.cabal @@ -41,6 +41,7 @@ library , bytestring , containers , directory + , either , http-conduit -- Effects & company , mtl @@ -56,6 +57,7 @@ library , universum , zip-archive , QuickCheck + , safe-exceptions -- Database backend , sqlite-simple , resource-pool diff --git a/src/DataSource/DB.hs b/src/DataSource/DB.hs index 525c2f8..52d9730 100644 --- a/src/DataSource/DB.hs +++ b/src/DataSource/DB.hs @@ -113,7 +113,7 @@ emptyDBLayer = DBLayer -- | The simple connection Zendesk layer. Used for database querying. -- We need to sync occasionaly. -connDataLayer :: forall m. (MonadIO m, MonadReader Config m) => DataLayer m +connDataLayer :: forall m. (MonadIO m, MonadReader Config m, MonadCatch m) => DataLayer m connDataLayer = DataLayer { zlGetTicketInfo = \tId -> withProdDatabase $ \conn -> getTicketInfoByTicketId conn tId , zlListDeletedTickets = zlListDeletedTickets basicDataLayer @@ -147,7 +147,7 @@ connDBLayer = DBLayer -- | The connection pooled Zendesk layer. Used for database querying. -- We need to sync occasionaly. -connPoolDataLayer :: forall m. (MonadBaseControl IO m, MonadIO m, MonadReader Config m) => DBConnPool -> DataLayer m +connPoolDataLayer :: forall m. (MonadBaseControl IO m, MonadIO m, MonadReader Config m, MonadCatch m) => DBConnPool -> DataLayer m connPoolDataLayer connPool = DataLayer { zlGetTicketInfo = \tId -> withConnPool connPool $ \conn -> getTicketInfoByTicketId conn tId , zlListDeletedTickets = zlListDeletedTickets basicDataLayer @@ -448,4 +448,3 @@ deleteAllData conn = do deleteCommentAttachments conn deleteTicketComments conn deleteTickets conn - diff --git a/src/DataSource/Http.hs b/src/DataSource/Http.hs index 3cc890d..19c6a21 100644 --- a/src/DataSource/Http.hs +++ b/src/DataSource/Http.hs @@ -11,14 +11,18 @@ module DataSource.Http import Universum import Control.Concurrent (threadDelay) +import Control.Exception.Safe (Handler (..), catches, throwM) import Control.Monad.Reader (ask) import Data.Aeson (parseJSON) import Data.Aeson.Text (encodeToLazyText) import Data.List (nub) +import Data.Text (pack) +import Network.HTTP.Client.Conduit (HttpException (..)) import Network.HTTP.Simple (Request, getResponseBody, httpLBS, parseRequest_) -import HttpLayer (HTTPNetworkLayer (..), apiRequest, apiRequestAbsolute) +import HttpLayer (HTTPNetworkLayer (..), JSONException (..), apiRequest, + apiRequestAbsolute) import DataSource.Types (Attachment (..), AttachmentContent (..), Comment (..), CommentBody (..), CommentId (..), Config (..), DataLayer (..), @@ -39,7 +43,7 @@ import DataSource.Types (Attachment (..), AttachmentContent (..), Comm -- - get returns a single result (wrapped in @Maybe@) -- - list returns multiple results -- - post submits a result (maybe PUT?!) -basicDataLayer :: (MonadIO m, MonadReader Config m) => DataLayer m +basicDataLayer :: (MonadIO m, MonadReader Config m, MonadCatch m) => DataLayer m basicDataLayer = DataLayer { zlGetTicketInfo = getTicketInfo , zlListDeletedTickets = listDeletedTickets @@ -74,34 +78,47 @@ emptyDataLayer = DataLayer -- | Get single ticket info. getTicketInfo - :: (HasCallStack, MonadIO m, MonadReader Config m) + :: forall m. (MonadIO m, MonadReader Config m, MonadCatch m) => TicketId -> m (Maybe TicketInfo) -getTicketInfo ticketId = do - cfg <- ask +getTicketInfo ticketId = + catches getInfo handlerList + where + getInfo :: m (Maybe TicketInfo) + getInfo = do + cfg <- ask - let url = showURL $ TicketsURL ticketId - let req = apiRequest cfg url + let url = showURL $ TicketsURL ticketId + req <- apiRequest cfg url - apiCall <- asksHTTPNetworkLayer hnlApiCall + apiCall <- asksHTTPNetworkLayer hnlApiCall + Just <$> apiCall parseTicket req - Just <$> apiCall parseTicket req + handlerList :: [Handler m (Maybe TicketInfo)] + handlerList = [handlerJSON, handlerHTTP] + handlerJSON :: Handler m (Maybe TicketInfo) + handlerJSON = Handler $ \(ex :: JSONException) -> return Nothing + handlerHTTP :: Handler m (Maybe TicketInfo) + handlerHTTP = Handler $ \(ex :: HttpException) -> return Nothing -- | Return list of deleted tickets. listDeletedTickets - :: forall m. (MonadIO m, MonadReader Config m) + :: forall m. (MonadIO m, MonadReader Config m, MonadThrow m) => m [DeletedTicket] -listDeletedTickets = do - cfg <- ask +listDeletedTickets = go + where + go :: m [DeletedTicket] + go = do + cfg <- ask - let url = showURL $ DeletedTicketsURL - let req = apiRequest cfg url + let url = showURL $ DeletedTicketsURL + req <- apiRequest cfg url - iteratePages req + wrapIteratePages req -- | Return list of ticketIds that has been requested by config user. listRequestedTickets - :: forall m. (MonadIO m, MonadReader Config m) + :: forall m. (MonadIO m, MonadReader Config m, MonadThrow m) => UserId -> m [TicketInfo] listRequestedTickets userId = do @@ -110,11 +127,11 @@ listRequestedTickets userId = do let url = showURL $ UserRequestedTicketsURL userId let req = apiRequest cfg url - iteratePages req + wrapIteratePages req -- | Return list of ticketIds that has been assigned by config user. listAssignedTickets - :: forall m. (MonadIO m, MonadReader Config m) + :: forall m. (MonadIO m, MonadReader Config m, MonadThrow m) => UserId -> m [TicketInfo] listAssignedTickets userId = do @@ -123,11 +140,11 @@ listAssignedTickets userId = do let url = showURL $ UserAssignedTicketsURL userId let req = apiRequest cfg url - iteratePages req + wrapIteratePages req -- | Return list of ticketIds that has been unassigned. listUnassignedTickets - :: forall m. (MonadIO m, MonadReader Config m) + :: forall m. (MonadIO m, MonadReader Config m, MonadThrow m) => m [TicketInfo] listUnassignedTickets = do cfg <- ask @@ -135,23 +152,23 @@ listUnassignedTickets = do let url = showURL $ UserUnassignedTicketsURL let req = apiRequest cfg url - iteratePages req + wrapIteratePages req listAdminAgents - :: forall m. (MonadIO m, MonadReader Config m) + :: forall m. (MonadIO m, MonadReader Config m, MonadThrow m) => m [User] listAdminAgents = do cfg <- ask let url = showURL AgentGroupURL let req = apiRequest cfg url - iteratePages req + wrapIteratePages req -- | Export tickets from Zendesk - https://developer.zendesk.com/rest_api/docs/core/incremental_export -- NOTE: If count is less than 1000, then stop paginating. -- Otherwise, use the next_page URL to get the next page of results. getExportedTickets - :: forall m. (MonadIO m, MonadReader Config m) + :: forall m. (MonadIO m, MonadReader Config m, MonadThrow m) => ExportFromTime -> m [TicketInfo] getExportedTickets time = do @@ -162,12 +179,20 @@ getExportedTickets time = do let url = showURL $ ExportDataByTimestamp time let req = apiRequestAbsolute cfg url - iterateExportedTicketsWithDelay req (apiCall parseJSON) + -- iterateExportedTicketsWithDelay req (apiCall parseJSON) + wrappedIterate req (apiCall parseJSON) where + wrappedIterate + :: Either String Request + -> (Request -> m (Either String (PageResultList TicketInfo))) + -> m [TicketInfo] + wrappedIterate (Left e) _ = throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + wrappedIterate (Right r) f = iterateExportedTicketsWithDelay r f + iterateExportedTicketsWithDelay :: Request - -> (Request -> m (PageResultList TicketInfo)) + -> (Request -> m (Either String (PageResultList TicketInfo))) -> m [TicketInfo] iterateExportedTicketsWithDelay req apiCall = do cfg <- ask @@ -177,23 +202,32 @@ getExportedTickets time = do liftIO $ threadDelay $ 10 * 1000000 -- Wait, Zendesk allows for 10 per minute. let req' = apiRequestAbsolute cfg nextPage' - (PageResultList pagen nextPagen count) <- apiCall req' - case nextPagen of - Just nextUrl -> if maybe False (>= 1000) count - then go (list' <> pagen) nextUrl - else pure (list' <> pagen) - - Nothing -> pure (list' <> pagen) - - - (PageResultList page0 nextPage _) <- apiCall req - case nextPage of - Just nextUrl -> go page0 nextUrl - Nothing -> pure page0 + case req' of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right req'' -> do + req''' <- apiCall req'' + case req''' of + Right (PageResultList pagen nextPagen count) -> + case nextPagen of + Just nextUrl -> if maybe False (>= 1000) count + then go (list' <> pagen) nextUrl + else pure (list' <> pagen) + + Nothing -> pure (list' <> pagen) + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + + + req' <- apiCall req + case req' of + Right (PageResultList page0 nextPage _) -> do + case nextPage of + Just nextUrl -> go page0 nextUrl + Nothing -> pure page0 + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception -- | Send API request to post comment postTicketComment - :: (MonadIO m, MonadReader Config m) + :: (MonadIO m, MonadReader Config m, MonadThrow m) => TicketInfo -> ZendeskResponse -> m () @@ -205,8 +239,16 @@ postTicketComment ticketInfo zendeskResponse = do let responseTicket = createResponseTicket (cfgAgentId cfg) ticketInfo zendeskResponse let url = showURL $ TicketsURL (zrTicketId zendeskResponse) - let req = addJsonBody responseTicket (apiRequest cfg url) - void $ apiCall (pure . encodeToLazyText) req + -- either (throwM . JSONEncodingException . pack) + let req = apiRequest cfg url + case req of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right r -> do + let r' = addJsonBody responseTicket r + case r' of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right r'' -> void $ apiCall (pure . encodeToLazyText) r'' + -- | Create response ticket createResponseTicket :: Integer -> TicketInfo -> ZendeskResponse -> Ticket @@ -231,7 +273,7 @@ createResponseTicket agentId TicketInfo{..} ZendeskResponse{..} = -- | Get user information. _getUser - :: (MonadIO m, MonadReader Config m) + :: (MonadIO m, MonadReader Config m, MonadThrow m) => m User _getUser = do cfg <- ask @@ -240,8 +282,13 @@ _getUser = do let url = showURL UserInfoURL let req = apiRequest cfg url - - apiCall parseJSON req + case req of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right r -> do + r' <- apiCall parseJSON r + case r' of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right u -> pure u -- | Given attachmentUrl, return attachment in bytestring getAttachment @@ -255,7 +302,7 @@ getAttachment Attachment{..} = Just . AttachmentContent . getResponseBody <$> ht -- | Get ticket's comments getTicketComments - :: (MonadIO m, MonadReader Config m) + :: (MonadIO m, MonadReader Config m, MonadThrow m) => TicketId -> m [Comment] getTicketComments tId = do @@ -265,14 +312,16 @@ getTicketComments tId = do let url = showURL $ TicketCommentsURL tId let req = apiRequest cfg url + case req of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right r -> do + result <- apiCallSafe parseComments r - result <- apiCallSafe parseComments req - - -- TODO(ks): For now return empty if there is an exception. - -- After we have exception handling, we propagate this up. - case result of - Left _ -> pure [] - Right r -> pure r + -- TODO(ks): For now return empty if there is an exception. + -- After we have exception handling, we propagate this up. + case result of + Left _ -> pure [] + Right r -> pure r ------------------------------------------------------------ -- Utility @@ -280,15 +329,20 @@ getTicketComments tId = do -- | Iterate all the ticket pages and combine into a result. iteratePages - :: forall m a. (MonadIO m, MonadReader Config m, FromPageResultList a) + :: forall m a. (MonadIO m, MonadReader Config m, MonadThrow m, FromPageResultList a) => Request - -> m [a] -iteratePages req = iteratePagesWithDelay 0 req + -> m (Maybe [Request]) +iteratePages req = catches tryIteratePages handlerList + where + tryIteratePages = iteratePagesWithDelay 0 req + handlerList :: Handler m (Maybe Request) + handlerList = handlerInvalidUrl + handlerInvalidUrl = Handler $ \(ex :: InvalidUrlException ) -> return Nothing -- | Iterate all the ticket pages and combine into a result. Wait for -- some time in-between the requests. iteratePagesWithDelay - :: forall m a. (MonadIO m, MonadReader Config m, FromPageResultList a) + :: forall m a. (MonadIO m, MonadReader Config m, MonadThrow m, FromPageResultList a) => Int -> Request -> m [a] @@ -303,13 +357,22 @@ iteratePagesWithDelay seconds req = do threadDelay $ seconds * 1000000 let req' = apiRequestAbsolute cfg nextPage' - (PageResultList pagen nextPagen _) <- apiCall parseJSON req' - case nextPagen of - Just nextUrl -> go (list' <> pagen) nextUrl - Nothing -> pure (list' <> pagen) - - (PageResultList page0 nextPage _) <- liftIO $ apiCall parseJSON req - case nextPage of - Just nextUrl -> liftIO $ go page0 nextUrl - Nothing -> pure page0 - + case req' of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right req'' -> do + req''' <- apiCall parseJSON req'' + case req''' of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right (PageResultList pagen nextPagen _) -> do + case nextPagen of + Just nextUrl -> go (list' <> pagen) nextUrl + Nothing -> pure (list' <> pagen) + + -- <- liftIO $ apiCall parseJSON req + req' <- liftIO $ apiCall parseJSON req + case req' of + Left e -> throwM $ InvalidUrlException "" "" -- TODO(md): See how to convert a String 'e' to an appropriate exception + Right (PageResultList page0 nextPage _) -> do + case nextPage of + Just nextUrl -> liftIO $ go page0 nextUrl + Nothing -> pure page0 diff --git a/src/DataSource/Types.hs b/src/DataSource/Types.hs index 6dcf407..af03655 100644 --- a/src/DataSource/Types.hs +++ b/src/DataSource/Types.hs @@ -204,10 +204,10 @@ data DataLayer m = DataLayer -- the @Http@ module, say in the @ZendeskaLayer@, but it's good enough for now. -- We need to use RankNTypes due to some complications that appeared. data HTTPNetworkLayer = HTTPNetworkLayer - { hnlAddJsonBody :: forall a. (ToJSON a) => a -> Request -> Request + { hnlAddJsonBody :: forall m a. (ToJSON a, MonadThrow m) => a -> Request -> m Request -- TODO(ks): These two below are basically the same, will fix in future PR, requires refactoring. - , hnlApiCall :: forall m a. (MonadIO m, FromJSON a) => (Value -> Parser a) -> Request -> m a - , hnlApiCallSafe :: forall m a. (MonadIO m, FromJSON a) => (Value -> Parser a) -> Request -> m (Either String a) + , hnlApiCall :: forall m a. (MonadIO m, MonadThrow m, FromJSON a) => (Value -> Parser a) -> Request -> m a + , hnlApiCallSafe :: forall m a. (MonadIO m, MonadThrow m, FromJSON a) => (Value -> Parser a) -> Request -> m a } -- | The IOLayer interface that we can expose. diff --git a/src/HttpLayer.hs b/src/HttpLayer.hs index fc32350..d36f070 100644 --- a/src/HttpLayer.hs +++ b/src/HttpLayer.hs @@ -1,5 +1,6 @@ module HttpLayer ( HTTPNetworkLayer (..) + , JSONException (..) , basicHTTPNetworkLayer , emptyHTTPNetworkLayer , apiRequest @@ -8,15 +9,26 @@ module HttpLayer import Universum -import Data.Aeson (FromJSON, ToJSON, Value, encode) +import Control.Exception.Safe (throwM) +import Data.Aeson (FromJSON, ToJSON, Value) import Data.Aeson.Types (Parser, parseEither) -import Network.HTTP.Simple (Request, addRequestHeader, getResponseBody, httpJSON, - parseRequest_, setRequestBasicAuth, setRequestBodyJSON, - setRequestMethod, setRequestPath) +import Data.Text (pack) +import Network.HTTP.Client.Conduit (parseUrlThrow) +import Network.HTTP.Simple (JSONException (..), Request, addRequestHeader, + getResponseBody, httpJSON, parseRequest_, setRequestBasicAuth, + setRequestBodyJSON, setRequestMethod, setRequestPath) import DataSource.Types (Config (..), HTTPNetworkLayer (..)) +------------------------------------------------------------ +-- Internal Exceptions +------------------------------------------------------------ +newtype APICallException + = MkAPICallException Text deriving Show + +instance Exception APICallException + ------------------------------------------------------------ -- Layer ------------------------------------------------------------ @@ -40,53 +52,75 @@ emptyHTTPNetworkLayer = HTTPNetworkLayer ------------------------------------------------------------ -- | General api request function -apiRequest :: Config -> Text -> Request -apiRequest Config{..} u = - setRequestPath (encodeUtf8 path) $ - addRequestHeader "Content-Type" "application/json" $ - setRequestBasicAuth - (encodeUtf8 cfgEmail <> "/token") - (encodeUtf8 cfgToken) $ - parseRequest_ (toString (cfgZendesk <> path)) +-- This function can throw a 'JSONException' and an 'HttpException'. +apiRequest + :: MonadThrow m + => Config + -> Text + -> m Request +apiRequest Config{..} u = do + req <- parseUrlThrow (toString (cfgZendesk <> path)) -- TODO(md): Get a list of exceptions that parseUrlThrow can throw + return $ setRequestPath (encodeUtf8 path) $ + addRequestHeader "Content-Type" "application/json" $ + setRequestBasicAuth + (encodeUtf8 cfgEmail <> "/token") + (encodeUtf8 cfgToken) $ req where + -- handlerList :: [Handler m Request] + -- handlerList = [ + -- handlerJSON + -- , handlerHTTP + -- ] + -- handlerJSON :: Handler m Request + -- handlerJSON = Handler $ \(ex :: JSONException) -> throwM ex + -- handlerHTTP :: Handler m Request + -- handlerHTTP = Handler $ \(ex :: HttpException) -> throwM ex path :: Text path = "/api/v2" <> u -- | Api request but use absolute path -apiRequestAbsolute :: Config -> Text -> Request +apiRequestAbsolute + :: Config + -> Text + -> Either String Request apiRequestAbsolute Config{..} u = - addRequestHeader "Content-Type" "application/json" $ + Right <$> addRequestHeader "Content-Type" "application/json" $ setRequestBasicAuth (encodeUtf8 cfgEmail <> "/token") (encodeUtf8 cfgToken) $ parseRequest_(toString u) -- | Request PUT -addJsonBody :: forall a. ToJSON a => a -> Request -> Request -addJsonBody body req = setRequestBodyJSON body $ setRequestMethod "PUT" req +addJsonBody + :: forall m a. (ToJSON a, MonadThrow m) + => a + -> Request + -> m Request +addJsonBody body req = + return <$> setRequestBodyJSON body $ setRequestMethod "PUT" req + -- | Make an api call --- TODO(ks): Switch to @Either@. apiCall - :: forall m a. (MonadIO m, FromJSON a) + :: forall m a. (MonadIO m, MonadThrow m, FromJSON a) => (Value -> Parser a) -> Request -> m a apiCall parser req = do putTextLn $ show req v <- getResponseBody <$> httpJSON req - case parseEither parser v of - Right o -> pure o - Left e -> error $ "couldn't parse response " - <> toText e <> "\n" <> decodeUtf8 (encode v) + either (throwEx v) return (parseEither parser v) + where + throwEx :: Value -> String -> m a + throwEx v msg = throwM $ MkAPICallException . pack $ + "Exception occured when parsing " ++ (show v) ++ ": " ++ msg + -- case parseEither parser v of + -- Right o -> pure o + -- Left e -> Left $ MkJSONParsingException $ + -- "couldn't parse response " <> toText e <> "\n" <> decodeUtf8 (encode v) -- | Make a safe api call. apiCallSafe - :: forall m a. (MonadIO m, FromJSON a) + :: forall m a. (MonadIO m, MonadThrow m, FromJSON a) => (Value -> Parser a) -> Request - -> m (Either String a) -apiCallSafe parser req = do - putTextLn $ show req - v <- getResponseBody <$> httpJSON req - pure $ parseEither parser v - - + -> m a +apiCallSafe = apiCall