Permalink
Browse files

Rewrite from scratch using new multi-process architecture.

It turned out that using a single-process design was not
going to work well enough in the long-run.  Offloading process
management to the front-end is against the goals of Scion, so
the new architecture fully embraces the need for multiple
processes.

See docs/Architecture.markdown for details.
  • Loading branch information...
1 parent e0cd3e5 commit dc0193c32ec32dc8db8f08877e4f670edc008dcd @nominolo committed Apr 20, 2011
View
@@ -1,75 +1,31 @@
-.PHONY: default clean install-lib install-deps setup
-
-default: all
-all: build
-
-include config.mk
-
-# If not set in custom config.mk, use the default versions
-HC ?= ghc
-PKG ?= ghc-pkg
-HADDOCK ?= haddock
-
-DIST = dist
-DIST_LIB = $(DIST)/lib
-DIST_SERVER = $(DIST)/server
-SETUP_DIST = setup-dist
-SETUP = $(SETUP_DIST)/Setup
-
-DOTDOTSETUP = cabal
-
-CABAL_INSTALL_OPTS += --ghc --with-compiler=$(HC) --with-hc-pkg=$(PKG)
-CABAL_FLAGS ?=
-# -ftesting
-
-$(DIST)/setup-config: $(SETUP) scion.cabal $(DIST)
- $(SETUP) configure -v --builddir=$(DIST) \
- --with-compiler=$(HC) --with-hc-pkg=$(PKG) \
- --user $(CABAL_FLAGS) > $(DIST)/lib-config-log
-
-$(DIST)/build/libHSscion-0.1.a: $(SETUP) $(DIST)/setup-config $(wildcard lib/**/*.hs lib/**/**/*.hs server/**/*.hs)
- @echo === Building scion ===
- $(SETUP) build --builddir=$(DIST)
-
-$(DIST):
- mkdir $(DIST)
-
-$(SETUP): Setup.hs $(SETUP_DIST)
- $(HC) --make $< -o $@
-
-$(SETUP_DIST):
- mkdir $@
-
-setup: $(SETUP)
-
-build: $(DIST)/build/libHSscion-0.1.a
-
-# TODO: dodgy
-install: $(DIST)/build/libHSscion-0.1.a
- cabal install
-
-# test: build
-# echo main | $(HC) --interactive -package ghc -DDEBUG -isrc -idist/build tests/RunTests.hs
-# # ./dist/build/test_get_imports/test_get_imports $(GHC_PATH)/compiler dist-stage2 +RTS -s -RTS
-
-clean:
- $(SETUP) clean --builddir=$(DIST) || rm -rf $(DIST)
-
-distclean: clean
- rm -rf $(SETUP_DIST)
-
-# doc: configure
-# $(SETUP) haddock --with-haddock=$(HADDOCK)
-
-printvars:
- @echo "UseInplaceGhc = $(UseInplaceGhc)"
- @echo "GHC_PATH = $(GHC_PATH)"
- @echo "HC = $(HC)"
- @echo "PKG = $(PKG)"
- @echo "HADDOCK = $(HADDOCK)"
- @echo "CABAL_INSTALL = $(CABAL_INSTALL)"
- @echo " ..._OPTS = $(CABAL_INSTALL_OPTS)"
- @echo "CABAL_FLAGS = $(CABAL_FLAGS)"
- @echo "---------------------------------------------------------------"
- @echo "DIST_LIB = $(DIST_LIB)"
- @echo "SETUP_DIST = $(SETUP_DIST)"
+default: inplace
+
+TOP := $(shell pwd)
+DIST = $(HOME)/tmp/dist-devel/scion-0.4/
+HC ?= ghc
+RUNHC ?= runghc
+
+#HC = ghc-6.12.1
+#RUNHC = runghc -f$(HC)
+
+boot:
+ mkdir -p $(DIST)
+
+.PHONY: inplace
+inplace:
+ $(HC) --make -outputdir $(DIST) -isrc -package ghc Scion.Session
+ $(HC) --make -outputdir $(DIST) -isrc -package ghc Scion.Worker.Main
+ $(HC) --make -outputdir $(DIST) -isrc -package ghc src/Worker.hs -o $(DIST)/scion-worker
+# cp src/Worker.hs $(DIST)/Worker.hs
+ echo "#!/bin/sh\n$(DIST)/scion-worker \$${1+\"\$$@\"}" > inplace/scion-worker
+ chmod +x inplace/scion-worker
+ echo "#!/bin/sh\n$(RUNHC) -i\"$(TOP)/src\" -package --ghc-arg=ghc -i\"$(DIST)\" \"$(TOP)/src/Server.hs\"" > inplace/scion-server
+ chmod +x inplace/scion-server
+
+.PHONY: install
+install:
+ cabal -v install --builddir=$(DIST)/cabal
+
+.PHONY: test
+test:
+ runghc test/TestSuite.hs
@@ -0,0 +1,62 @@
+Since version 0.3 Scion uses a multi-process architecture. The Scion
+library starts one or more `scion-worker` processes which do the
+actual work. The Scion library just manages these processes (and
+caches some of their state). This solves the following problems:
+
+ - *Static Flags*. Some of GHC's command line flags can only be set
+ on start-up. This is important mainly for flags that control the
+ kind of compilation (profiled, threaded).
+
+ - *Other write-once state*. GHC only reads the package database once
+ on startup. If new packages have been installed since startup
+ they will not be visible. Changing the database by force while a
+ session is running is likely to cause problems.
+
+ - *Caches*. There are a few caches in GHC that cannot be flushed.
+ These include the name cache, and the package DB cache.
+
+ - *Multiple Compiler Versions*. It is not possible to link to two
+ different versions of GHC from within the same program. If we
+ want to make sure a program compiles with multiple versions of GHC
+ (or multiple combinations of its dependencies) we need to use
+ multiple processes.
+
+The downside of a multi-process architecture is of course the
+additional context switches and communication overhead. To reduce
+this, we:
+
+ - use a binary protocol,
+
+ - cache some information on the library side, and
+
+ - avoid sending too much data between library and worker.
+
+Non-Haskell front-ends use a scion-server that takes the place of the
+library.
+
+The architecture therefore looks as follows:
+
+ +-----------------------+
+ | Non-Haskell frontend |
+ | (Eclipse, Emacs, Vim) |
+ +-----------------------+
+ ^
+ | front-end specific protocol
+ | (e.g., json, s-exprs)
+ v
+ +-----------------+
+ | Scion server / |
+ | Scion library |
+ +-----------------+
+ ^ ^ ^
+ | | | binary protocol
+ v v v
+ +--------------+ +--------------+
+ | Scion worker | ... | Scion worker |
+ +--------------+ +--------------+
+
+If the front-end is written in Haskell, it will take the part of the
+Scion library. The Scion server, in turn, translates between a
+front-end-specific serialisation format to Scion library API calls.
+
+The library-worker protocol is defined in `src/Scion/Types/Commands`.
View
@@ -1,5 +1,5 @@
name: scion
-version: 0.1.0.2
+version: 0.3
license: BSD3
license-file: LICENSE
author: Thomas Schilling <nominolo@googlemail.com>
@@ -19,59 +19,61 @@ description:
category: Development
stability: provisional
build-type: Simple
-cabal-version: >= 1.6
+cabal-version: >= 1.10
extra-source-files: README.markdown
-data-files:
- emacs/*.el
- vim_runtime_path/autoload/*.vim
- vim_runtime_path/ftplugin/*.vim
- vim_runtime_path/plugin/*.vim
+--data-files:
+-- emacs/*.el
+-- vim_runtime_path/autoload/*.vim
+-- vim_runtime_path/ftplugin/*.vim
+-- vim_runtime_path/plugin/*.vim
-flag testing
- description: Enable Debugging things like QuickCheck properties, etc.
- default: False
+--flag testing
+-- description: Enable Debugging things like QuickCheck properties, etc.
+-- default: False
-flag server
- description: Install the scion-server.
- default: True
+--flag server
+-- description: Install the scion-server.
+-- default: True
library
+ default-language: Haskell2010
build-depends:
- base == 4.*,
- Cabal >= 1.5 && < 1.7,
- containers == 0.2.*,
- directory == 1.0.*,
- filepath == 1.1.*,
- ghc >= 6.10 && < 6.12,
+ base >= 4.2 && < 4.4,
+ Cabal >= 1.8 && < 1.12,
+ containers >= 0.3 && < 0.5,
+ directory >= 1.0 && < 1.2,
+ filepath >= 1.1 && < 1.3,
+ ghc >= 6.12 && < 7.2,
ghc-paths == 0.1.*,
- ghc-syb == 0.1.*,
- hslogger == 1.0.*,
- json == 0.4.*,
- multiset == 0.1.*,
- time == 1.1.*,
- uniplate == 1.2.*
-
- hs-source-dirs: lib
- extensions: CPP, PatternGuards
+ multiset >= 0.1 && < 0.3,
+ time >= 1.1 && < 1.3,
+ text >= 0.11 && < 0.12,
+ process >= 1.0 && < 1.1,
+ unix-compat >= 0.2 && < 0.3,
+ bytestring >= 0.9 && < 0.10,
+ binary >= 0.5 && < 0.6,
+ old-locale >= 1.0 && < 1.1,
+ network >= 2.3 && < 2.4
+
+ hs-source-dirs: src
+ default-extensions: CPP, PatternGuards
exposed-modules:
- Scion.Types,
- Scion.Types.ExtraInstances,
- Scion.Types.Notes,
- Scion.Inspect,
- Scion.Inspect.Find,
- Scion.Inspect.TypeOf,
- Scion.Inspect.DefinitionSite,
- Scion.Utils,
+ Scion.Ghc,
Scion.Session,
- Scion.Configure,
- Scion
-
- if flag(testing)
- build-depends: QuickCheck == 2.*
- cpp-options: -DDEBUG
+ Scion.Types.Compiler,
+ Scion.Types.Commands,
+ Scion.Types.Monad,
+ Scion.Types.Note,
+ Scion.Types.Session,
+ Scion.Types.Worker,
+ Scion.Utils.Convert,
+ Scion.Utils.IO,
+ Scion.Worker.Commands,
+ Scion.Worker.Main
+ other-modules:
+ Paths_scion
- if impl(ghc > 6.11)
- cpp-options: -DHAVE_PACKAGE_DB_MODULES
+ cpp-options: -DHAVE_PACKAGE_DB_MODULES
-- TODO: drop after 6.10.2 is out
if impl(ghc >= 6.11.20081113) || impl(ghc >= 6.10.2 && < 6.11)
@@ -82,65 +84,10 @@ library
ghc-options: -Wall
-executable scion-server
- if !flag(server)
- buildable: False
-
- main-is: Main.hs
- hs-source-dirs: lib server
-
+executable scion-worker
+ main-is: Worker.hs
+ hs-source-dirs: src-execs
+ default-language: Haskell2010
build-depends:
- -- From the library:
- base == 4.*,
- Cabal >= 1.5 && < 1.7,
- containers == 0.2.*,
- directory == 1.0.*,
- filepath == 1.1.*,
- ghc >= 6.10 && < 6.12,
- ghc-paths == 0.1.*,
- ghc-syb == 0.1.*,
- hslogger == 1.0.*,
- json == 0.4.*,
- multiset == 0.1.*,
- time == 1.1.*
-
- if flag(server)
- build-depends:
- -- Server only
- bytestring == 0.9.*,
- network >= 2.1 && < 2.3,
- network-bytestring == 0.1.*,
- utf8-string == 0.3.*
-
- other-modules:
- Scion
- Scion.Configure
- Scion.Inspect
- Scion.Inspect.DefinitionSite
- Scion.Session
- Scion.Types
- Scion.Types.Notes
- Scion.Utils
-
- Scion.Server.Commands
- Scion.Server.ConnectionIO
- Scion.Server.Generic
- Scion.Server.Protocol
-
- ghc-options: -Wall
- extensions: CPP, PatternGuards
-
- if flag(testing)
- build-depends: QuickCheck == 2.*
- cpp-options: -DDEBUG
-
- if impl(ghc > 6.11)
- cpp-options: -DHAVE_PACKAGE_DB_MODULES
-
- -- TODO: drop after 6.10.2 is out
- if impl(ghc >= 6.11.20081113) || impl(ghc >= 6.10.2 && < 6.11)
- cpp-options: -DRECOMPILE_BUG_FIXED
-
- if impl(ghc == 6.10.*)
- cpp-options: -DWPINLINE
-
+ scion,
+ base >= 4.2 && < 4.4
View
@@ -0,0 +1,6 @@
+module Main where
+
+import Scion.Worker.Main ( workerMain )
+
+main = workerMain 42
+--main = soloWorkerMain
Oops, something went wrong. Retry.

0 comments on commit dc0193c

Please sign in to comment.