Skip to content
Browse files

maintain set of files pendingAdd

Kqueue needs to remember which files failed to be added due to being open,
and retry them. This commit gets the data in place for such a retry thread.

Broke KeySource out into its own file, and added Eq and Ord instances
so it can be stored in a Set.
  • Loading branch information...
1 parent ad11de9 commit e0fdfb2e706da2cb1451193c658dc676b0530968 @joeyh committed Jun 20, 2012
Showing with 89 additions and 30 deletions.
  1. +1 −1 Assistant.hs
  2. +37 −18 Assistant/Committer.hs
  3. +9 −1 Assistant/DaemonStatus.hs
  4. +3 −3 Backend.hs
  5. +1 −0 Backend/SHA.hs
  6. +1 −0 Backend/WORM.hs
  7. +1 −0 Command/Add.hs
  8. +1 −0 Command/AddUrl.hs
  9. +1 −0 Command/Migrate.hs
  10. +1 −7 Types/Backend.hs
  11. +33 −0 Types/KeySource.hs
View
2 Assistant.hs
@@ -75,8 +75,8 @@ startDaemon foreground
-- begin adding files and having them
-- committed, even while the startup scan
-- is taking place.
- _ <- forkIO $ commitThread st changechan
_ <- forkIO $ daemonStatusThread st dstatus
+ _ <- forkIO $ commitThread st dstatus changechan
_ <- forkIO $ sanityCheckerThread st dstatus changechan
-- Does not return.
watchThread st dstatus changechan
View
55 Assistant/Committer.hs
@@ -7,6 +7,7 @@ module Assistant.Committer where
import Common.Annex
import Assistant.Changes
+import Assistant.DaemonStatus
import Assistant.ThreadedMonad
import Assistant.Watcher
import qualified Annex
@@ -18,23 +19,23 @@ import qualified Command.Add
import Utility.ThreadScheduler
import qualified Utility.Lsof as Lsof
import qualified Utility.DirWatcher as DirWatcher
-import Types.Backend
+import Types.KeySource
import Data.Time.Clock
import Data.Tuple.Utils
import qualified Data.Set as S
{- This thread makes git commits at appropriate times. -}
-commitThread :: ThreadState -> ChangeChan -> IO ()
-commitThread st changechan = runEvery (Seconds 1) $ do
+commitThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
+commitThread st dstatus changechan = runEvery (Seconds 1) $ do
-- We already waited one second as a simple rate limiter.
-- Next, wait until at least one change has been made.
cs <- getChanges changechan
-- Now see if now's a good time to commit.
time <- getCurrentTime
if shouldCommit time cs
then do
- handleAdds st changechan cs
+ handleAdds st dstatus changechan cs
void $ tryIO $ runThreadState st commitStaged
else refillChanges changechan cs
@@ -79,19 +80,20 @@ shouldCommit now changes
-
- When a file is added, Inotify will notice the new symlink. So this waits
- for additional Changes to arrive, so that the symlink has hopefully been
- - staged before returning, and will be committed immediately. OTOH, for
- - kqueue, eventsCoalesce, so instead the symlink is directly created and
- - staged.
+ - staged before returning, and will be committed immediately.
+ -
+ - OTOH, for kqueue, eventsCoalesce, so instead the symlink is directly
+ - created and staged, if the file is not open.
-}
-handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
-handleAdds st changechan cs
+handleAdds :: ThreadState -> DaemonStatusHandle -> ChangeChan -> [Change] -> IO ()
+handleAdds st dstatus changechan cs
| null toadd = noop
| otherwise = do
- toadd' <- safeToAdd st toadd
+ toadd' <- safeToAdd st dstatus toadd
unless (null toadd') $ do
added <- filter id <$> forM toadd' add
unless (DirWatcher.eventsCoalesce || null added) $
- handleAdds st changechan
+ handleAdds st dstatus changechan
=<< getChanges changechan
where
toadd = map changeFile $ filter isPendingAdd cs
@@ -122,8 +124,8 @@ handleAdds st changechan cs
- opened for write, so lsof is run on the temp directory
- to check them.
-}
-safeToAdd :: ThreadState -> [FilePath] -> IO [KeySource]
-safeToAdd st files = do
+safeToAdd :: ThreadState -> DaemonStatusHandle -> [FilePath] -> IO [KeySource]
+safeToAdd st dstatus files = do
locked <- catMaybes <$> lockdown files
runThreadState st $ ifM (Annex.getState Annex.force)
( return locked -- force bypasses lsof check
@@ -134,16 +136,33 @@ safeToAdd st files = do
catMaybes <$> forM locked (go open)
)
where
+ {- When a file is still open, it can be put into pendingAdd
+ - to be checked again later. However when closingTracked
+ - is supported, another event will be received once it's
+ - closed, so there's no point in doing so. -}
go open keysource
| S.member (contentLocation keysource) open = do
- warning $ keyFilename keysource
- ++ " still has writers, not adding"
- -- remove the hard link
- --_ <- liftIO $ tryIO $
- -- removeFile $ contentLocation keysource
+ if DirWatcher.closingTracked
+ then do
+ warning $ keyFilename keysource
+ ++ " still has writers, not adding"
+ void $ liftIO $ canceladd keysource
+ else void $ addpending keysource
return Nothing
| otherwise = return $ Just keysource
+ canceladd keysource = tryIO $
+ -- remove the hard link
+ removeFile $ contentLocation keysource
+
+ {- The same file (or a file with the same name)
+ - could already be pending add; if so this KeySource
+ - superscedes the old one. -}
+ addpending keysource = modifyDaemonStatusM dstatus $ \s -> do
+ let set = pendingAdd s
+ mapM_ canceladd $ S.toList $ S.filter (== keysource) set
+ return $ s { pendingAdd = S.insert keysource set }
+
lockdown = mapM $ \file -> do
ms <- catchMaybeIO $ getSymbolicLinkStatus file
case ms of
View
10 Assistant/DaemonStatus.hs
@@ -9,12 +9,14 @@ import Common.Annex
import Assistant.ThreadedMonad
import Utility.ThreadScheduler
import Utility.TempFile
+import Types.KeySource
import Control.Concurrent
import System.Posix.Types
import Data.Time.Clock.POSIX
import Data.Time
import System.Locale
+import qualified Data.Set as S
data DaemonStatus = DaemonStatus
-- False when the daemon is performing its startup scan
@@ -25,6 +27,8 @@ data DaemonStatus = DaemonStatus
, sanityCheckRunning :: Bool
-- Last time the sanity checker ran
, lastSanityCheck :: Maybe POSIXTime
+ -- Files that are in the process of being added to the annex.
+ , pendingAdd :: S.Set KeySource
}
deriving (Show)
@@ -36,13 +40,17 @@ newDaemonStatus = DaemonStatus
, lastRunning = Nothing
, sanityCheckRunning = False
, lastSanityCheck = Nothing
+ , pendingAdd = S.empty
}
getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus
getDaemonStatus = liftIO . readMVar
modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex ()
-modifyDaemonStatus handle a = liftIO $ modifyMVar_ handle (return . a)
+modifyDaemonStatus handle a = modifyDaemonStatusM handle (return . a)
+
+modifyDaemonStatusM :: DaemonStatusHandle -> (DaemonStatus -> IO DaemonStatus) -> Annex ()
+modifyDaemonStatusM handle a = liftIO $ modifyMVar_ handle a
{- Load any previous daemon status file, and store it in the MVar for this
- process to use as its DaemonStatus. -}
View
6 Backend.hs
@@ -6,7 +6,6 @@
-}
module Backend (
- B.KeySource(..),
list,
orderedList,
genKey,
@@ -23,6 +22,7 @@ import Config
import qualified Annex
import Annex.CheckAttr
import Types.Key
+import Types.KeySource
import qualified Types.Backend as B
-- When adding a new backend, import it here and add it to the list.
@@ -54,12 +54,12 @@ orderedList = do
{- Generates a key for a file, trying each backend in turn until one
- accepts it.
-}
-genKey :: B.KeySource -> Maybe Backend -> Annex (Maybe (Key, Backend))
+genKey :: KeySource -> Maybe Backend -> Annex (Maybe (Key, Backend))
genKey source trybackend = do
bs <- orderedList
let bs' = maybe bs (: bs) trybackend
genKey' bs' source
-genKey' :: [Backend] -> B.KeySource -> Annex (Maybe (Key, Backend))
+genKey' :: [Backend] -> KeySource -> Annex (Maybe (Key, Backend))
genKey' [] _ = return Nothing
genKey' (b:bs) source = do
r <- B.getKey b source
View
1 Backend/SHA.hs
@@ -11,6 +11,7 @@ import Common.Annex
import qualified Annex
import Types.Backend
import Types.Key
+import Types.KeySource
import qualified Build.SysConfig as SysConfig
type SHASize = Int
View
1 Backend/WORM.hs
@@ -10,6 +10,7 @@ module Backend.WORM (backends) where
import Common.Annex
import Types.Backend
import Types.Key
+import Types.KeySource
backends :: [Backend]
backends = [backend]
View
1 Command/Add.hs
@@ -12,6 +12,7 @@ import Annex.Exception
import Command
import qualified Annex
import qualified Annex.Queue
+import Types.KeySource
import Backend
import Logs.Location
import Annex.Content
View
1 Command/AddUrl.hs
@@ -20,6 +20,7 @@ import Annex.Content
import Logs.Web
import qualified Option
import Types.Key
+import Types.KeySource
import Config
def :: [Command]
View
1 Command/Migrate.hs
@@ -11,6 +11,7 @@ import Common.Annex
import Command
import Backend
import qualified Types.Key
+import Types.KeySource
import Annex.Content
import qualified Command.ReKey
View
8 Types/Backend.hs
@@ -10,13 +10,7 @@
module Types.Backend where
import Types.Key
-
-{- The source used to generate a key. The location of the content
- - may be different from the filename associated with the key. -}
-data KeySource = KeySource
- { keyFilename :: FilePath
- , contentLocation :: FilePath
- }
+import Types.KeySource
data BackendA a = Backend
{ name :: String
View
33 Types/KeySource.hs
@@ -0,0 +1,33 @@
+{- KeySource data type
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.KeySource where
+
+import Data.Ord
+
+{- When content is in the process of being added to the annex,
+ - and a Key generated from it, this data type is used.
+ -
+ - The contentLocation may be different from the filename
+ - associated with the key. For example, the add command
+ - temporarily puts the content into a lockdown directory
+ - for checking. The migrate command uses the content
+ - of a different Key. -}
+data KeySource = KeySource
+ { keyFilename :: FilePath
+ , contentLocation :: FilePath
+ }
+ deriving (Show)
+
+{- KeySources are assumed to be equal when the same filename is associated
+ - with the key. The contentLocation can be a random temp file.
+ -}
+instance Eq KeySource where
+ x == y = keyFilename x == keyFilename y
+
+instance Ord KeySource where
+ compare = comparing keyFilename

0 comments on commit e0fdfb2

Please sign in to comment.
Something went wrong with that request. Please try again.