diff --git a/MS5/dna-compiler/Step.hs b/MS5/dna-compiler/Step.hs deleted file mode 100644 index 7a827423..00000000 --- a/MS5/dna-compiler/Step.hs +++ /dev/null @@ -1,126 +0,0 @@ -{-# LANGUAGE DeriveFoldable #-} -{-# LANGUAGE DeriveFunctor #-} -{-# LANGUAGE DeriveTraversable #-} -{-# LANGUAGE GADTs #-} -{-# LANGUAGE LambdaCase #-} --- | --- Compilation @[Step] → AST@ -module Step where - -import Control.Applicative -import Control.Monad -import qualified Data.HashMap.Strict as HM -import Data.Foldable (Foldable) -import Data.Traversable (Traversable) -import Bound - -import Flow -import Flow.Internal -import Flow.Vector -import DNA - - --- | --- Untyped expression tree for compiling to DNA -data StepE a - = V a - -- ^ Untyped variable - | Pair (StepE a) (StepE a) - -- ^ 2-tuple - | List [StepE a] - -- ^ List of variables - - | SDom (IO Domain) - -- ^ Create domain - | SKern KernelBind [StepE a] [StepE a] - -- ^ Kernel call - -- - -- * Kernel description - -- * Parameters - -- * Output domain - | SBind (StepE a) (Scope () StepE a) - -- ^ Monadic bind. We only introduce new variables in monadic context. - deriving (Functor,Foldable,Traversable) - --- | Variable name -data V - = KernVar Int - | DomVar Int - deriving (Show,Eq) - -instance Applicative StepE where - pure = return - (<*>) = ap -instance Monad StepE where - return = V - V a >>= f = f a - Pair a b >>= f = Pair (a >>= f) (b >>= f) - List xs >>= f = List (map (>>= f) xs) - SKern k xs ys >>= f = SKern k (map (>>= f) xs) (map (>>= f) ys) - SBind e g >>= f = SBind (e >>= f) (g >>>= f) - - ----------------------------------------------------------------- --- Compilation to expression tree ----------------------------------------------------------------- - -compileSteps :: [Step] -> StepE V -compileSteps [] = error "compileSteps: empty list" -compileSteps [x] = fst $ singleStep x -compileSteps (x:xs) = - let (expr,v) = singleStep x - rest = compileSteps xs - in expr `SBind` abstract1 v rest - -singleStep :: Step -> (StepE V, V) -singleStep = \case - DomainStep dh -> (SDom (dhCreate dh), DomVar (dhId dh)) - KernelStep kb -> ( SKern kb - [ Pair (V $ KernVar i) (List $ V . DomVar <$> ds) - | (i,ds) <- kernDeps kb ] - (kbDomList kb) - , KernVar (kernId kb)) - DistributeStep{} -> error "DistributeStep is not supported" - SplitStep{} -> error "SplitStep is not supported" - where - kbDomList kb = case kernRepr kb of - ReprI r -> V . DomVar <$> reprDomain r - - ----------------------------------------------------------------- --- Interpretation ----------------------------------------------------------------- - -data Box - = VDom Domain - | VVec (Vector ()) - -interpretAST :: StepE V -> DNA Box -interpretAST e = case closed e of - Just e' -> go e' - Nothing -> error "interpretAST: expression is not closed!" - where - go = \case - V{} -> error "Naked variable at the top level" - Pair{} -> error "Naked pair at the top level" - List{} -> error "Naked list at the top level" - -- Create new domain - SDom dom -> - DNA.kernel "dom" [] $ liftIO $ VDom <$> dom - -- Call kernel - SKern kb deps dom -> - let toDom = \case - V (VDom d) -> d - V (VVec _) -> error "Vector where domain expected" - _ -> error "Only variables expected" - toParam = \case - Pair (V (VVec v)) (List p) -> (v, map toDom p) - _ -> error "Ill formed parameters" - xs = map toParam deps - out = map toDom dom - in DNA.kernel "kern" [] $ liftIO $ VVec <$> kernCode kb xs out - -- Monadic bind - SBind expr lam -> - let dnaE = go expr - lamE = \a -> go $ instantiate1 (V a) lam - in dnaE >>= lamE diff --git a/MS5/dna-flow/Flow/DnaCompiler.hs b/MS5/dna-flow/Flow/DnaCompiler.hs new file mode 100644 index 00000000..43f71642 --- /dev/null +++ b/MS5/dna-flow/Flow/DnaCompiler.hs @@ -0,0 +1,653 @@ +{-# LANGUAGE DeriveDataTypeable #-} +{-# LANGUAGE DeriveFoldable #-} +{-# LANGUAGE DeriveFunctor #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DeriveTraversable #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} +-- | +-- Compilation @[Step] → AST@ +module Flow.DnaCompiler ( + -- * Ext. steps + ActorTree(..) + , walkActorTree + -- ** Extended steps + , VV(..) + , ExtStep(..) + , Vars(..) + -- ** Transformations + , makeActorTree + , findInOut + , addCommands + -- * Compilation to AST for DNA + -- ** AST + , StepE(..) + -- ** Compilation + , DnaActor(..) + , V(..) + , compileProgram + -- ** Pretty-printing + , prettyprint + , prettyprintLam + -- * Interpretation + , interpretAST + ) where + +import Control.Arrow (Arrow(..)) +import Control.Applicative +import Control.Monad +import Control.Monad.State.Strict +import Control.Monad.Writer +import Control.Distributed.Process (RemoteTable) +import Control.Distributed.Process.Closure (mkClosureValSingle,MkTDict) +import qualified Data.HashMap.Strict as HM +import Data.HashMap.Strict ((!)) +import qualified Data.HashSet as HS +import qualified Data.Binary as Bin +import qualified Data.Binary.Get as Bin +import qualified Data.Binary.Put as Bin +import qualified Data.Map as Map +import Data.Typeable +import Data.Hashable +import Data.List +import Data.Monoid +import Data.Foldable (Foldable(foldMap),toList) +import Data.Traversable (Traversable(traverse,sequenceA)) +import Bound +import Prelude.Extras +import GHC.Generics (Generic) +import Text.PrettyPrint hiding ((<>)) + +import Flow +import Flow.Internal +import Flow.Vector +import DNA + +import Debug.Trace + + + +---------------------------------------------------------------- +-- Preprocessing of [Steps] +-- +-- We separate steps into actor which are in turn converted to +-- DNA expression +---------------------------------------------------------------- + +-- | Tree of actors. +data ActorTree a = ActorTree a (HM.HashMap Int (ActorTree a)) + deriving (Show,Eq,Functor) + +walkActorTree :: b -> (a -> a -> b) -> ActorTree a -> ActorTree b +walkActorTree b f (ActorTree a hm) + = ActorTree b (fmap (go a) hm) + where + go a0 (ActorTree a' hmm) = ActorTree (f a0 a') (fmap (go a') hmm) + +-- | Variables as used in Step definition +data VV + = KVar Int [Int] + | DVar Int + deriving (Show,Eq,Generic) +instance Hashable VV + +-- | Extended Step data type +data ExtStep where + Step :: Step -> ExtStep + Call :: Typeable a => Domain a -> [VV] -> Int -> ExtStep + Expect :: Int -> [(Maybe ReprI, VV)] -> ExtStep + Yield :: [VV] -> ExtStep + +-- | Actor type +data ActorTy + = SimpleActor + | DistrActor Int Int -- Split/Distribute actor + | DistrVarActor [VV] [VV] -- Split/Distribute actor with variables + deriving (Show) + +-- | Variables for actor definition +data Vars = Vars + { varsUsed :: HS.HashSet VV -- ^ Variables referenced in actor + , varsProd :: HS.HashSet VV -- ^ Variables defined in actor + , varsMissing :: HS.HashSet VV -- ^ Used variables which are not defined in actor + } + deriving (Show) + + +-- Compilation algo: +-- +-- 1. Separate actors +-- 2. Find out input and output parameters for actors +-- 3. Insert gather for each use of output +-- 4. Convert to AST + +-- Split off code for DNA actors +makeActorTree :: [Step] -> ActorTree (ActorTy, [ExtStep]) +makeActorTree steps + = flip evalState 0 + $ run + $ fmap ((,) SimpleActor) + $ go steps + where + -- + run :: Monad m => WriterT [(Int, ActorTree a)] m a -> m (ActorTree a) + run m = do (a,pairs) <- runWriterT m + return $ ActorTree a (HM.fromList pairs) + -- Transform each step + go :: [Step] -> WriterT [(Int, ActorTree (ActorTy, [ExtStep]))] (State Int) [ExtStep] + go [] = return [] + go (x:xs) = do + x' <- case x of + -- + DistributeStep dh' _sched ss -> do + n <- get + put $! n+1 + let Just pdom = dhParent dh' + child <- lift + $ run + $ fmap ((,) (DistrActor (dhId pdom) (dhId dh'))) + $ go ss + tell [(n , child)] + return (Call dh' [] n) + -- + _ -> return (Step x) + xs' <- go xs + return $ x' : xs' + +-- Determine inputs and outputs for actors +findInOut :: Functor f => ActorTree (f [ExtStep]) -> ActorTree (f (Vars,[ExtStep])) +findInOut = (fmap . fmap) makeVars + where + makeVars ast = (Vars u p (HS.difference u p),ast) + where + u = foldMap collectReferencedVars ast + p = foldMap collectProducedVars ast + +-- Add explicit data transfer commands +addCommands :: ActorTree (ActorTy, (Vars,[ExtStep])) -> ActorTree (ActorTy, (Vars,[ExtStep])) +addCommands + = fmap (\(_,_,a) -> a) + . transform (Vars mempty mempty mempty) + where + -- Parameters: variables defined in parent and missing in child + transform pvars (ActorTree (ty,(vars,steps)) children) + = ActorTree (params,retV, (ty',(vars,steps''))) children' + where + -- Parameters for current actor: + -- * Variables generated by parent by not in child. + params = varsProd pvars `HS.intersection` varsMissing vars + -- Return values + -- * Variables missing in parent and produced in child when + -- adjusted for domain + retV = case ty of + SimpleActor -> mempty + DistrActor dhPar dhCh -> + varsProd vars + `HS.intersection` + HS.map (changeDom dhPar dhCh) (varsMissing pvars) + ty' = case ty of + SimpleActor -> SimpleActor + DistrActor{} -> DistrVarActor + (HS.toList params) + (HS.toList retV) + -- Transform child actors recursively + children' = transform vars <$> children + -- Reference parameter from children + steps' = flip concatMap steps $ \case + Call dh _ i -> let ActorTree (p,rv,_) _ = children' ! i + in [ Call dh (HS.toList p) i + , Expect i [ case v of + DVar{} -> (Nothing, v) + KVar kid _ -> ( getFirst $ mconcat $ map (findReprForK kid) steps + , v + ) + | v <- HS.toList rv + ] + ] + x -> [x] + appendY = case ty' of + DistrVarActor _ rv -> (++ [Yield rv]) + _ -> id + steps'' = appendY steps' + + +changeDom :: Int -> Int -> VV -> VV +changeDom old new (KVar k ds) = KVar k ((\i -> if i == old then new else i) <$> ds) +changeDom _ _ v = v + + + +-- Collect variables referenced by step +collectReferencedVars :: ExtStep -> HS.HashSet VV +collectReferencedVars = withExtStep collectReferencedVars' + +collectProducedVars :: ExtStep -> HS.HashSet VV +collectProducedVars = withExtStep collectProducedVars' + +findReprForK :: Int -> ExtStep -> First ReprI +findReprForK i = withExtStep (findReprForK' i) + + +withExtStep :: Monoid t => (Step -> t) -> ExtStep -> t +withExtStep f = \case + Step s -> f s + -- Call dh _ _ -> f (SplitStep dh []) + _ -> mempty + +findReprForK' :: Int -> Step -> First ReprI +findReprForK' i = \case + KernelStep kb + | kernId kb == i -> First $ Just (kernRepr kb) + | otherwise -> mconcat $ map fromKDep $ kernDeps kb + _ -> mempty + where + fromKDep (KernelDep kid r) | i == kid = First $ Just r + | otherwise = mempty + +-- Collect variables referenced by step +collectReferencedVars' :: Step -> HS.HashSet VV +collectReferencedVars' = \case + DomainStep (Just kid) _ -> HS.singleton (KVar kid []) + DomainStep _ _ -> mempty + KernelStep kb -> HS.fromList $ concat + [ KVar kid (reprDomain repr) + : (DVar <$> reprDomain repr) + | KernelDep kid (ReprI repr) <- kernDeps kb + ] + DistributeStep dh _ ss -> (HS.singleton $ DVar $ dhId dh) + <> foldMap collectReferencedVars' ss + +-- Collect variables produced by step +collectProducedVars' :: Step -> HS.HashSet VV +collectProducedVars' = \case + DomainStep _ dh -> HS.singleton $ DVar $ dhId dh + KernelStep kb -> case kernRepr kb of + ReprI repr -> HS.singleton $ KVar (kernId kb) (reprDomain repr) + DistributeStep _ _ ss -> foldMap collectProducedVars' ss + + + +---------------------------------------------------------------- +-- AST for DNA expression +---------------------------------------------------------------- + +-- | +-- Untyped expression tree for compiling to DNA +data StepE a + = V a + -- ^ Untyped variable + | Pair (StepE a) (StepE a) + -- ^ 2-tuple. At the moment pair of kernel-region box] + | List [StepE a] + -- ^ List of variables + + + | SDom (Maybe (StepE a)) AnyDH (Maybe (StepE a)) + -- ^ Create domain + -- + -- * Region argument + -- * Wrapped domain + -- * Parent domain if any + | SKern KernelBind [StepE a] [StepE a] + -- ^ Kernel call + -- + -- * Kernel description + -- * Parameters + -- * Output domain + + | SSeq (StepE a) (StepE a) + -- ^ Sequence two monadic actions + | SBind (StepE a) (Scope () StepE a) + -- ^ Monadic bind. We only introduce new variables in monadic context. + + | SActorGrp Int [StepE a] + -- ^ Corresponds to group of actor + -- + -- * Actor ID + -- * Input parameters + | SActorRecvK ReprI (StepE a) (StepE a) + -- + -- * Representation of resulting vector + -- * Channel variable + -- * Region of whole vector + | SActorRecvD (StepE a) + -- ^ Receive data from actors + deriving (Show,Functor,Foldable,Traversable,Generic) +instance Show1 StepE + + +data AnyDH where + AnyDH :: Typeable a => Domain a -> AnyDH + +instance Show AnyDH where + show (AnyDH d) = show d + + +instance Applicative StepE where + pure = V + (<*>) = ap +instance Monad StepE where + return = V + V a >>= f = f a + Pair a b >>= f = Pair (a >>= f) (b >>= f) + List xs >>= f = List (map (>>= f) xs) + SDom a m b >>= f = SDom ((>>= f) <$> a) m ((>>= f) <$> b) + SKern k xs ys >>= f = SKern k (map (>>= f) xs) (map (>>= f) ys) + SSeq a b >>= f = SSeq (a >>= f) (b >>= f) + SBind e g >>= f = SBind (e >>= f) (g >>>= f) + SActorGrp i xs >>= f = SActorGrp i (map (>>= f) xs) + SActorRecvK r a b >>= f = SActorRecvK r (a >>= f) (b >>= f) + SActorRecvD a >>= f = SActorRecvD (a >>= f) + + + +---------------------------------------------------------------- +-- Compilation to expression tree +---------------------------------------------------------------- + +-- | Variable name +data V + = KernVar Int + -- ^ @Vector ()@ produced by kernel. It's referenced by KernelID + | DomVar Int + -- ^ Region (referenced by DomainID) + | ChanVar Int + deriving (Show,Eq,Generic) +instance Hashable V + +data DnaActor + = MainActor (StepE V) + | RemoteActor (Scope () StepE V) + + +compileProgram + :: ActorTree (ActorTy, (Vars,[ExtStep])) + -> (DnaActor, HM.HashMap Int DnaActor) +compileProgram atree@(ActorTree a _) + = (compile a, compile <$> flatten atree) + where + compile (ty,(_,steps)) = compileActor ty steps + -- Flatten tree + flatten :: ActorTree a -> HM.HashMap Int a + flatten (ActorTree _ hm) + = fmap (\(ActorTree a _) -> a) hm + <> mconcat (toList $ fmap flatten hm) + +compileActor :: ActorTy -> [ExtStep] -> DnaActor +compileActor ty steps = case ty of + SimpleActor -> MainActor prog + DistrVarActor [DVar i] _ -> RemoteActor $ abstract1 (DomVar i) prog + DistrVarActor [KVar i _] _ -> RemoteActor $ abstract1 (KernVar i) prog + DistrActor{} -> error "DistrActor should not appear there" + DistrVarActor{} -> error "DistrActorVar should have single var only" + where + prog = compileSteps steps + +compileSteps :: [ExtStep] -> StepE V +compileSteps [] = error "compileSteps: empty list" +compileSteps [x] = toStep $ singleStep x +compileSteps (x:xs) = + let rest = compileSteps xs + in case singleStep x of + StepVal expr v -> expr `SBind` abstract1 v rest + Step2Val e1 v1 e2 v2 -> + e1 `SBind` abstract1 v1 (e2 `SBind` abstract1 v2 rest) + StepNoVal expr -> expr `SSeq` rest + +data StepRes + = StepVal (StepE V) V + | Step2Val (StepE V) V (StepE V) V + | StepNoVal (StepE V) + +toStep :: StepRes -> StepE V +toStep = \case + StepVal e _ -> e + StepNoVal e -> e + +singleStep :: ExtStep -> StepRes +singleStep = \case + ---------------------------------------- + -- Single step + Step s -> case s of + DomainStep mkid dh -> + let m = V . KernVar <$> mkid + in StepVal + (SDom m (AnyDH dh) (V . DomVar . dhId <$> dhParent dh)) + (DomVar (dhId dh)) + KernelStep kb -> StepVal + (SKern kb + -- Parameters + [ Pair (V $ KernVar kid) + (List $ V . DomVar <$> reprDomain repr) + | KernelDep kid (ReprI repr) <- kernDeps kb + ] + -- Output domains + (kbDomList kb)) + (KernVar (kernId kb)) + _ -> error "Other steps should not appear in transformed program" + ---------------------------------------- + -- Call actor + Call dh pars i -> + let dhp = case dhParent dh of + Just d -> d + Nothing -> error "Parent??" + in StepVal + -- (SSplit (RegSplit $ dhRegion dhp) (V $ DomVar $ dhId dhp)) + -- (DomVar $ dhId dh) + (SActorGrp i [ case p of + DVar n -> V (DomVar n) + KVar n _ -> V (error "A") + | p <- pars ]) + (ChanVar i) + -- Gather results from vector and build full vector from it. + Expect _ [(Nothing,DVar di)] -> error "Expecting domain is not implemented" + Expect i [(Just r@(ReprI repr), KVar ki _)] -> + let [did] = reprDomain repr + in StepVal ( SActorRecvK r + (V $ ChanVar i) + (V $ DomVar did) + ) + ( KernVar ki ) + Expect{} -> error "Do not support expecting more than 1 element" + -- Yield result + Yield [KVar ki _] -> let v = KernVar ki in StepVal (V v) v + Yield [DVar di] -> let v = DomVar di in StepVal (V v) v + Yield{} -> error "Can only yield single value" + where + kbDomList kb = case kernRepr kb of + ReprI r -> V . DomVar <$> reprDomain r + + + +---------------------------------------------------------------- +-- Interpretation +---------------------------------------------------------------- + +data Box + = VReg RegionBox + | VVec RegionBox (Vector ()) + | VChan (DNA.Group Box) + deriving (Typeable) + +instance Bin.Binary Box where + put = error "No put really!" + get = Bin.getWord8 >>= \case + 0 -> VReg <$> undefined -- Bin.get + 1 -> VVec <$> undefined <*> undefined + _ -> error "Bad tag!" + + +interpretAST + :: HM.HashMap Int DnaActor -> StepE V -> (RemoteTable -> RemoteTable, DNA Box) +interpretAST actorMap mainActor = case closed mainActor of + Nothing -> error "interpretAST: expression is not closed!" + Just e -> (rtable, logMessage "START INTERPRETATION" >> go e) + where + go = \case + V a -> return a + Pair{} -> error "Naked pair at the top level" + List{} -> error "Naked list at the top level" + -- Domains + SDom me (AnyDH dh) mpar -> do + logMessage $ "SDom: " ++ show dh + case mpar of + Nothing -> do let m = case me of Nothing -> Map.empty + Just e -> error "FIXME: region lookup is not implemented" + DNA.kernel "dom" [] $ liftIO $ VReg . pure <$> dhCreate dh m + Just par -> do rs <- DNA.kernel "split" [] $ liftIO $ mapM (dhRegion dh) (toDom par) + return $ VReg $ concat rs + -- Kernel + SKern kb deps dom -> do + let xs = map (Map.fromList . (:[]) . toParam) deps + out = toDom =<< dom + [v] <- DNA.kernel "kern" [] $ liftIO $ kernCode kb xs [out] -- FIXME: wrong + return $ VVec out v + -- Monadic bind + SBind expr lam -> + let dnaE = go expr + lamE = \a -> go $ instantiate1 (V a) lam + in dnaE >>= lamE + SSeq e1 e2 -> go e1 >> go e2 + -- Actor spawning + SActorGrp actID [par] -> do + -- FIXME: only region is passed as parameter + let regs = toDom par + n = length regs + (clos,_) = amap ! actID + logMessage $ "REGS: " ++ show regs + sh <- startGroup (N n) (NNodes 1) $ return clos + let scatter _ xs = map (VReg . pure) xs + distributeWork regs scatter sh + grp <- delayGroup sh + return $ VChan grp + SActorGrp _ _ -> error "Only actor with one parameter are supported" + -- Receiving of parameters + SActorRecvK (ReprI repr) vCh vReg -> do + let ch = toGrp vCh + reg = toDom vReg + xs <- gather ch (flip (:)) [] + let pars = flip map xs $ \case + VVec v p -> (v,p) + _ -> error "Only vector expected!" + DNA.kernel "merge" [] $ liftIO $ do + Just vec <- reprMerge repr (Map.fromList pars) reg + return $ VVec reg vec + SActorRecvD{} -> error "Receiving of domains is not implemented" + -- + toDom = \case + V (VReg d) -> d + V VVec{} -> error "Vector where domain expected" + _ -> error "Only variables expected" + toParam = \case + Pair (V (VVec _ v)) (List p) -> (toDom =<< p, v) + _ -> error "Ill formed parameters" + toGrp = \case + V (VChan ch) -> ch + V _ -> error "Not a chan var" + _ -> error "Only variables expected" + -- + runActor :: DnaActor -> DNA.Actor Box Box + runActor = \case + MainActor a -> error "Not supported" + RemoteActor lam -> + let Just lam' = closed lam + in DNA.actor $ \x -> go (instantiate1 (V x) lam') + -- + Endo rtable = foldMap (Endo . snd) amap + amap = HM.mapWithKey (\i a -> let (clos,reg) = mkClosureValSingle ("DNA_NAME_" ++ show i) $ \_ -> a + in (clos (), reg) + ) + $ fmap runActor actorMap + + + +---------------------------------------------------------------- +-- Pretty printer +---------------------------------------------------------------- + + +-- | Pretty print AST +prettyprint :: Show a => StepE a -> Doc +prettyprint = flip evalState varNames . ppr . fmap Right + +-- | Pretty print AST +prettyprintLam :: Show a => Scope () StepE a -> Doc +prettyprintLam lam + = flip evalState varNames + $ do v <- fresh + doc <- ppr $ instantiate1 (V (Left v)) $ fmap Right lam + return $ vcat + [ text ("λ"++v++" →") + , nest 2 doc + ] + +varNames = map (:[]) ['a' .. 'z'] + + +pprList :: Show a => [StepE (Either String a)] -> State [String] Doc +pprList es = do ss <- mapM ppr es + return $ brackets $ hcat $ intersperse comma ss + +ppr :: Show a => StepE (Either String a) -> State [String] Doc +ppr = \case + V (Left v) -> return $ text v + V (Right a) -> return $ text (show a) + Pair e g -> do se <- ppr e + sg <- ppr g + return $ parens $ se <> comma <> sg + List es -> pprList es + SDom v dh par -> do + sv <- maybe (return $ text "-") ppr v + spar <- maybe (return $ text "-") ppr v + return $ hcat [ text "SDom " + , sv + , text (show dh) + , spar + ] + SKern kb vars dom -> do + vs <- pprList vars + ds <- pprList dom + return $ text "Kernel call" $$ nest 2 + (vcat [ text (show kb), vs, ds ]) + SSeq e1 e2 -> liftM2 ($$) (ppr e1) (ppr e2) + -- SSplit _ e -> do + -- s <- ppr e + -- return $ hcat [ text "SSplit {" + -- , s + -- , text "}" + -- ] + SBind e lam -> do + v <- fresh + se <- ppr e + sl <- ppr $ instantiate1 (V $ Left v) lam + return $ vcat + [ se <> text (" $ λ"++v++" →") + , nest 2 sl + ] + SActorGrp i vars -> do + xs <- pprList vars + return $ hcat [ text "Actor Grp " + , int i + , text " " + , xs + ] + SActorRecvK r vCh vReg -> do + sCh <- ppr vCh + sReg <- ppr vReg + return $ hcat [ text "Actor recv K " + , text $ " {"++(show r)++"} " + , sCh + , text " " + , sReg + ] + SActorRecvD v -> do + s <- ppr v + return $ hcat [ text "Actor recv D " + , s + ] + +fresh = do + v : rest <- get + put rest + return v + diff --git a/MS5/dna-flow/Flow/Internal.hs b/MS5/dna-flow/Flow/Internal.hs index 0383ad7f..003c0f1e 100644 --- a/MS5/dna-flow/Flow/Internal.hs +++ b/MS5/dna-flow/Flow/Internal.hs @@ -1,6 +1,6 @@ {-# LANGUAGE GADTs, TypeFamilies, RankNTypes, ScopedTypeVariables, TypeOperators, DeriveDataTypeable, TypeSynonymInstances, - FlexibleInstances, FlexibleContexts + FlexibleInstances, FlexibleContexts, DeriveGeneric #-} module Flow.Internal where @@ -15,10 +15,12 @@ import Data.Hashable import Data.Int import qualified Data.IntMap as IM import Data.List ( sort, groupBy ) +import Data.Binary (Binary(..)) import qualified Data.Map as Map import Data.Monoid import qualified Data.HashMap.Strict as HM import Data.Typeable +import GHC.Generics (Generic) import Flow.Vector @@ -202,8 +204,8 @@ instance Show DomainI where -- | Domains are just ranges for now. It is *very* likely that we are -- going to have to generalise this in some way. data Region = RangeRegion (Domain Range) Range - | BinRegion (Domain Bins) Bins - deriving Typeable + | BinRegion (Domain Bins) Bins + deriving (Typeable, Generic) instance Show Region where showsPrec _ (RangeRegion dom r) = showString "Region<" . shows (dhId dom) . ('>':) . shows r @@ -225,11 +227,16 @@ instance Eq Region where r0 == r1 = compare r0 r1 == EQ data Range = Range Int Int - deriving (Typeable, Eq, Ord) + deriving (Typeable, Eq, Ord, Generic) + +instance Binary Range instance Show Range where showsPrec _ (Range low high) = shows low . (':':) . shows high + data Bins = Bins (Map.Map (Double, Double) (Map.Map RegionBox Int)) - deriving (Typeable, Eq, Ord) + deriving (Typeable, Eq, Ord, Generic) + + instance Show Bins where showsPrec _ (Bins bins) = showString "Bins" . flip (foldr f) (Map.toList bins) where f ((low, high), m) = (' ':) . shows low . (':':) . shows high . shows (Map.elems m) diff --git a/MS5/dna-flow/Flow/Run.hs b/MS5/dna-flow/Flow/Run.hs index 4ba2439f..317319bc 100644 --- a/MS5/dna-flow/Flow/Run.hs +++ b/MS5/dna-flow/Flow/Run.hs @@ -4,6 +4,7 @@ module Flow.Run ( dumpStrategy , dumpStrategyDOT + , dumpStep , dumpSteps , execStrategy , execStrategyDNA @@ -69,21 +70,20 @@ dumpStrategyDOT file strat = do hPutStrLn h "}" hClose h +dumpStep ind (DomainStep m_kid dh) + = putStrLn $ ind ++ "Domain " ++ show dh ++ + maybe "" (\kid -> " from kernel " ++ show kid) m_kid ++ + maybe "" (\dom -> " split from " ++ show dom) (dhParent dh) +dumpStep ind (KernelStep kb@KernelBind{kernRepr=ReprI rep}) + = putStrLn $ ind ++ "Over " ++ show (reprDomain rep) ++ " run " ++ show kb +dumpStep ind step@(DistributeStep did sched steps) + = do putStrLn $ ind ++ "Distribute " ++ show did ++ " using " ++ show sched ++ + " deps " ++ show (stepKernDeps step) + forM_ steps (dumpStep (" " ++ ind)) + dumpSteps :: Strategy a -> IO () dumpSteps strat = do - - let dump ind (DomainStep m_kid dh) - = putStrLn $ ind ++ "Domain " ++ show dh ++ - maybe "" (\kid -> " from kernel " ++ show kid) m_kid ++ - maybe "" (\dom -> " split from " ++ show dom) (dhParent dh) - dump ind (KernelStep kb@KernelBind{kernRepr=ReprI rep}) - = putStrLn $ ind ++ "Over " ++ show (reprDomain rep) ++ " run " ++ show kb - dump ind step@(DistributeStep did sched steps) - = do putStrLn $ ind ++ "Distribute " ++ show did ++ " using " ++ show sched ++ - " deps " ++ show (stepKernDeps step) - forM_ steps (dump (" " ++ ind)) - - forM_ (runStrategy (void strat)) (dump "") + forM_ (runStrategy (void strat)) (dumpStep "") execStrategy :: Strategy () -> IO () execStrategy strat = do diff --git a/MS5/dna-flow/dna-flow.cabal b/MS5/dna-flow/dna-flow.cabal index 8e5cf2a3..c8da5b97 100644 --- a/MS5/dna-flow/dna-flow.cabal +++ b/MS5/dna-flow/dna-flow.cabal @@ -30,6 +30,10 @@ library bytestring >= 0.10, time, fixed-vector-hetero >= 0.2, + bound >= 1.0.6, + prelude-extras >= 0.4, + pretty, + distributed-process >= 0.5.5, dna >= 0.5, distributed-process, distributed-static, @@ -43,6 +47,7 @@ library Flow.Domain, Flow.Vector, Flow.Halide.Types + Flow.DnaCompiler Flow.Internal, Flow.Halide, Flow.Halide.BufferT, diff --git a/MS5/ms5.cabal b/MS5/ms5.cabal index f33c7cfc..5f3fe8cf 100644 --- a/MS5/ms5.cabal +++ b/MS5/ms5.cabal @@ -34,7 +34,29 @@ executable dotproduct kernel/cpu/dotproduct/generate_g.cpp kernel/cpu/dotproduct/dotp.cpp kernel/cpu/dotproduct/sum.cpp - x-halide-options: -Wall -fno-strict-aliasing -std=c++0x -lm -lstdc++ -lHalide + x-halide-options: -Wall -fno-strict-aliasing -std=c++0x -lm -lstdc++ -lHalide -lLLVM-3.6 + +executable dotproduct-dna + default-language: Haskell2010 + ghc-options: -O2 -Wall -threaded -eventlog -rtsopts + Hs-source-dirs: programs + main-is: dotproduct-dna.hs + build-depends: + base >= 4.6, + dna-flow >= 0.5, + mtl, + bound, + pretty, + distributed-process, + dna, + containers, + unordered-containers, + fixed-vector-hetero + x-halide-sources: kernel/cpu/dotproduct/generate_f.cpp + kernel/cpu/dotproduct/generate_g.cpp + kernel/cpu/dotproduct/dotp.cpp + kernel/cpu/dotproduct/sum.cpp + x-halide-options: -Wall -fno-strict-aliasing -std=c++0x -lm -lstdc++ -lHalide -lLLVM-3.6 executable imaging default-language: Haskell2010 @@ -61,4 +83,4 @@ executable gridding kernel/cpu/gridding/init.cpp kernel/cpu/gridding/detile.cpp kernel/cpu/gridding/fft.cpp - x-halide-options: -Wall -fno-strict-aliasing -std=c++11 -lm -lpthread -lstdc++ -lHalide + x-halide-options: -Wall -fno-strict-aliasing -std=c++11 -lm -lpthread -lstdc++ -lHalide -lLLVM-3.6 diff --git a/MS5/programs/dotproduct-dna.hs b/MS5/programs/dotproduct-dna.hs new file mode 100644 index 00000000..94628f90 --- /dev/null +++ b/MS5/programs/dotproduct-dna.hs @@ -0,0 +1,195 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE DeriveDataTypeable #-} +{-# LANGUAGE ForeignFunctionInterface #-} +{-# LANGUAGE LambdaCase #-} + +module Main where + +import Data.Typeable + +import Control.Arrow (Arrow(..)) +import Control.Monad +import Control.Monad.State +import Data.List +import Data.Monoid (Monoid(..)) +import Flow +import Flow.Vector +import Flow.Halide +import Text.PrettyPrint +-- import DNA (dnaRun) + +-- Needed for FFI to work +import qualified Data.HashMap.Strict as HM +import qualified Data.HashSet as HS +import Data.Vector.HFixed.Class () +import qualified Data.Foldable as T +import Flow.Halide.Types () +import Flow.DnaCompiler +import Flow.Internal +import Bound + +import DNA (dnaRun,logMessage) + +import Data.Dynamic +import Unsafe.Coerce +import qualified Data.Map as Map +import Control.Distributed.Process.Node (initRemoteTable) + +-- Data tags +data Vec deriving Typeable +data Sum deriving Typeable + +-- Abstract flow signatures +f, g :: Flow Vec +f = flow "f" +g = flow "g" +pp :: Flow Vec -> Flow Vec -> Flow Vec +pp = flow "product" +a :: Flow Vec -> Flow Sum +a = flow "sum" +ddp :: Flow Sum +ddp = a $ pp f g + +-- Vector representation +type VecRepr = DynHalideRepr Dim0 Float Vec +vecRepr :: Domain Range -> VecRepr +vecRepr = dynHalideRepr dim0 +type SumRepr = HalideRepr Z Float Sum +sumRepr :: SumRepr +sumRepr = halideRepr Z + +-- Kernels + +fKern :: Domain Range -> Kernel Vec +fKern size = halideKernel0 "f" (vecRepr size) kern_generate_f +foreign import ccall unsafe kern_generate_f :: HalideFun '[] VecRepr + +gKern :: Domain Range -> Kernel Vec +gKern size = halideKernel0 "g" (vecRepr size) kern_generate_g +foreign import ccall unsafe kern_generate_g :: HalideFun '[] VecRepr + +ppKern :: Domain Range -> Flow Vec -> Flow Vec -> Kernel Vec +ppKern size = halideKernel1Write "pp" (vecRepr size) (vecRepr size) kern_dotp +foreign import ccall unsafe kern_dotp :: HalideFun '[ VecRepr ] VecRepr + +aKern :: Domain Range -> Flow Vec -> Kernel Sum +aKern size = halideKernel1 "a" (vecRepr size) sumRepr kern_sum +foreign import ccall unsafe kern_sum :: HalideFun '[ VecRepr ] SumRepr + +printKern :: Flow Sum -> Kernel Sum +printKern = mergingKernel "print" (sumRepr :. Z) NoRepr $ \case + [(sv,_)]-> \_ -> do + s <- peekVector (castVector sv :: Vector Float) 0 + putStrLn $ "Sum: " ++ show s + return nullVector + _other -> fail "printKern: Received wrong number of input buffers!" + +-- | Dot product, non-distributed +dpStrat :: Int -> Strategy () +dpStrat size = do + + -- Make vector domain + dom <- makeRangeDomain 0 size + + -- Calculate ddp for the whole domain + bind f (fKern dom) + bind g (gKern dom) + bindRule pp (ppKern dom) + bindRule a (aKern dom) + calculate ddp + rebind ddp printKern + +-- | Dot product, distributed +ddpStrat :: Int -> Strategy () +ddpStrat size = do + + -- Make vector domain + dom <- makeRangeDomain 0 size + + -- Calculate ddp for the whole domain + regs <- split dom 10 + distribute regs ParSchedule $ do + bind f (fKern regs) + bind g (gKern regs) + bind (pp f g) (ppKern regs f g) + bindRule a (aKern dom) + calculate ddp + void $ bindNew $ printKern ddp + + +dumpES ind (Step s) = dumpStep ind s +dumpES ind (Call dh pars i) = do + putStrLn $ ind ++ "Call " ++ show i ++ " " ++ show pars + -- dumpStep (ind ++ " ") $ SplitStep dh [] +-- dumpES ind (SplitDistr dh' sched ss) = do +-- dumpStep ind (DistributeStep dh' sched []) +-- mapM_ (dumpES (ind++" ")) ss +dumpES ind (Expect i vs) = + putStrLn $ ind ++ "Expect " ++ show i ++ " " ++ show vs +dumpES ind (Yield vs) = + putStrLn $ ind ++ "Yield " ++ show vs + + +dumpTreeWith :: (String -> a -> IO ()) -> ActorTree a -> IO () +dumpTreeWith out = go "" + where + go off (ActorTree a m) = do + out off a + let off' = " " ++ off + forM_ (HM.toList m) $ \(i,a') -> do + putStrLn $ off' ++ "[Key=" ++ show i ++ "]" + go off' a' + +dumpTree :: Show a => ActorTree a -> IO () +dumpTree = dumpTreeWith (\off a -> putStr off >> print a) + + +main :: IO () +main = do + -- Peter's part + let size = 1000000 + -- strat = dpStrat size + strat = ddpStrat size + steps = runStrategy strat + putStrLn "----------------------------------------------------------------" + dumpSteps strat + -- Transformation + let ast0@(ActorTree ss acts) = makeActorTree steps + ast1 = findInOut ast0 + ast2 = addCommands ast1 + ast3 = compileProgram ast2 + putStrLn "\n-- Actor split -------------------------------------------------" + dumpTreeWith (\off -> mapM_ (dumpES off) . snd) ast0 + putStrLn "\n-- Used --------------------------------------------------------" + dumpTree $ fmap (varsUsed . fst . snd) ast1 + putStrLn "-- Produced ----------------------------------------------------" + dumpTree $ fmap (varsProd . fst . snd) ast1 + putStrLn "-- Missing -----------------------------------------------------" + dumpTree $ fmap (varsMissing . fst . snd) ast1 + -- + putStrLn "\n-- Annotated ---------------------------------------------------" + dumpTreeWith (\off (ty,(_,as))-> do putStr off >> putStr ">> " >> print ty + mapM_ (dumpES off) as + ) ast2 + -- Dump AST + putStrLn "\n-- AST ---------------------------------------------------------" + let renderAST = \case + MainActor aa -> putStrLn $ render $ prettyprint aa + RemoteActor lam -> putStrLn $ render $ prettyprintLam lam + + case ast3 of + (a,hm) -> do renderAST a + forM_ (HM.toList hm) $ \(i,aa) -> do + putStrLn $ "== " ++ show i ++ " ================" + renderAST aa + -- Run program + case ast3 of + (MainActor dna, amap) -> + let (rtable,prog) = interpretAST amap dna + in do return () + -- let m = unsafeCoerce $ rtable initRemoteTable :: Map.Map String Dynamic + -- mapM_ print $ Map.toList $ fmap dynTypeRep m + dnaRun rtable $ do + logMessage "START" + void prog + logMessage "END"