data/iran_election_2009.csv @@ -40,7 +40,7 @@ For a great introduction to programming in Clojure, read "Clojure - Functional P h2. Getting started with Incanter -Start the Clojure shell (aka the REPL) from the Incanter directory: @bin/clj@ (or @bin\clj.bat@ on Windows). +Start the Clojure shell (aka the REPL) from the Incanter directory: @bin/clj@ or @bin\clj.bat@ on Windows (note: if you want to start the Clojure REPL from a directory other than the top level Incanter directory, change the INCANTER_HOME variable in the clj script so that it uses an absolute path instead the ./ directory). From the Clojure shell, load the Incanter libraries: <pre><code>(use '(incanter core stats charts))</code></pre> README.textile @@ -1917,7 +1917,7 @@ setLinkToggleText('library-contents-toggle-incanter.censored', '+ dist (DoubleUniform. min max (DoubleMersenneTwister.))] (if (coll? x) (map #(.cdf dist %) x) - (.cdf dist x)))))</pre></div></div></dd></dl></li><li class="library-member"><a name="member-incanter.stats-chisq-test"/><dl class="library-member-table"><dt class="library-member-name">chisq-test</dt><dd><div class="library-member-info"><span class="library-member-type">fn</span> <span class="library-member-arglists">([& options])</span></div><div class="library-member-docs"><div class="library-member-doc-line"> </div><div class="library-member-doc-line"> Performs chi-squared contingency table tests and goodness-of-fit tests.</div><div class="library-member-doc-line"> </div><div class="library-member-doc-line"> If the optional argument :y is not provided then a goodness-of-fit test</div><div class="library-member-doc-line"> is performed. In this case, the hypothesis tested is whether the </div><div class="library-member-doc-line"> population probabilities equal those in :probs, or are all equal if </div><div class="library-member-doc-line"> :probs is not given.</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> If :y is provided, it must be a sequence of integers that is the</div><div class="library-member-doc-line"> same length as x. A contingency table is computed from x and :y. </div><div class="library-member-doc-line"> Then, Pearson's chi-squared test of the null hypothesis that the joint</div><div class="library-member-doc-line"> distribution of the cell counts in a 2-dimensional contingency</div><div class="library-member-doc-line"> table is the product of the row and column marginals is performed.</div><div class="library-member-doc-line"> By default the Yates' continuity correction for 2x2 contingency</div><div class="library-member-doc-line"> tables is performed, this can be disabled by setting the :correct </div><div class="library-member-doc-line"> option to false.</div><div class="library-member-doc-line"> </div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> Options:</div><div class="library-member-doc-line"> :x -- a sequence of numbers.</div><div class="library-member-doc-line"> :y -- a sequence of numbers</div><div class="library-member-doc-line"> :table -- a contigency table</div><div class="library-member-doc-line"> :probs (when (nil? y) -- (repeat n-levels (/ n-levels)))</div><div class="library-member-doc-line"> :correct (default true) -- use Yates' correction for continuity</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> Returns:</div><div class="library-member-doc-line"> :X-sq -- the Pearson X-squared test statistics</div><div class="library-member-doc-line"> :p-value -- the p-value for the test statistic</div><div class="library-member-doc-line"> :df -- the degress of freedom</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> Examples:</div><div class="library-member-doc-line"> (use '(incanter core stats))</div><div class="library-member-doc-line"> (chisq-test :x [1 2 3 2 3 2 4 3 5]) ;; 2.6667</div><div class="library-member-doc-line"> (chisq-test :x [1 0 0 0 1 1 1 0 0 1 0 0 1 1 1 1]) ;; 0.25</div><div class="library-member-doc-line"> </div><div class="library-member-doc-line"> (use '(incanter core stats datasets))</div><div class="library-member-doc-line"> (def math-prog (to-matrix (get-dataset :math-prog)))</div><div class="library-member-doc-line"> (def x (sel math-prog :cols 1))</div><div class="library-member-doc-line"> (def y (sel math-prog :cols 2))</div><div class="library-member-doc-line"> (chisq-test :x x :y y) ;; X-sq = 1.24145, df=1, p-value = 0.26519</div><div class="library-member-doc-line"> (chisq-test :x x :y y :correct false) ;; X-sq = 2.01094, df=1, p-value = 0.15617</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> (def table (matrix [[31 12] [9 8]]))</div><div class="library-member-doc-line"> (chisq-test :table table) ;; X-sq = 1.24145, df=1, p-value = 0.26519</div><div class="library-member-doc-line"> (chisq-test :table table :correct false) ;; X-sq = 2.01094, df=1, p-value = 0.15617</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> ;; look at the hair-eye-color data</div><div class="library-member-doc-line"> ;; turn the count data for males into a contigency table</div><div class="library-member-doc-line"> (def male (matrix (sel (get-dataset :hair-eye-color) :cols 3 :rows (range 16)) 4))</div><div class="library-member-doc-line"> (chisq-test :table male) ;; X-sq = 41.280, df = 9, p-value = 4.44E-6</div><div class="library-member-doc-line"> ;; turn the count data for females into a contigency table</div><div class="library-member-doc-line"> (def female (matrix (sel (get-dataset :hair-eye-color) :cols 3 :rows (range 16 32)) 4))</div><div class="library-member-doc-line"> (chisq-test :table female) ;; X-sq = 106.664, df = 9, p-value = 7.014E-19,</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> (def detab (detabulate :table table))</div><div class="library-member-doc-line"> (chisq-test :x (sel detab :cols 0) :y (sel detab :cols 1))</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> References:</div><div class="library-member-doc-line"> http://www.itl.nist.gov/div898/handbook/eda/section3/eda35f.htm</div><div class="library-member-doc-line"> http://en.wikipedia.org/wiki/Pearson's_chi-square_test</div><div class="library-member-doc-line"> http://en.wikipedia.org/wiki/Yates'_chi-square_test</div></div><div class="library-member-source-section"><div class="library-member-source-toggle">[ <a href="javascript:toggleSource('membersource-incanter.stats-chisq-test')" id="linkto-membersource-incanter.stats-chisq-test">Show Source</a> ]</div><div class="library-member-source" id="membersource-incanter.stats-chisq-test"><pre>(defn chisq-test + (.cdf dist x)))))</pre></div></div></dd></dl></li><li class="library-member"><a name="member-incanter.stats-chisq-test"/><dl class="library-member-table"><dt class="library-member-name">chisq-test</dt><dd><div class="library-member-info"><span class="library-member-type">fn</span> <span class="library-member-arglists">([& options])</span></div><div class="library-member-docs"><div class="library-member-doc-line"> </div><div class="library-member-doc-line"> Performs chi-squared contingency table tests and goodness-of-fit tests.</div><div class="library-member-doc-line"> </div><div class="library-member-doc-line"> If the optional argument :y is not provided then a goodness-of-fit test</div><div class="library-member-doc-line"> is performed. In this case, the hypothesis tested is whether the </div><div class="library-member-doc-line"> population probabilities equal those in :probs, or are all equal if </div><div class="library-member-doc-line"> :probs is not given.</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> If :y is provided, it must be a sequence of integers that is the</div><div class="library-member-doc-line"> same length as x. A contingency table is computed from x and :y. </div><div class="library-member-doc-line"> Then, Pearson's chi-squared test of the null hypothesis that the joint</div><div class="library-member-doc-line"> distribution of the cell counts in a 2-dimensional contingency</div><div class="library-member-doc-line"> table is the product of the row and column marginals is performed.</div><div class="library-member-doc-line"> By default the Yates' continuity correction for 2x2 contingency</div><div class="library-member-doc-line"> tables is performed, this can be disabled by setting the :correct </div><div class="library-member-doc-line"> option to false.</div><div class="library-member-doc-line"> </div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> Options:</div><div class="library-member-doc-line"> :x -- a sequence of numbers.</div><div class="library-member-doc-line"> :y -- a sequence of numbers</div><div class="library-member-doc-line"> :table -- a contigency table. If one dimensional, the test is a goodness-of-fit</div><div class="library-member-doc-line"> :probs (when (nil? y) -- (repeat n-levels (/ n-levels)))</div><div class="library-member-doc-line"> :freq (default nil) -- if given, these are rescaled to probabilities</div><div class="library-member-doc-line"> :correct (default true) -- use Yates' correction for continuity for 2x2 contingency tables</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> Returns:</div><div class="library-member-doc-line"> :X-sq -- the Pearson X-squared test statistics</div><div class="library-member-doc-line"> :p-value -- the p-value for the test statistic</div><div class="library-member-doc-line"> :df -- the degress of freedom</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> Examples:</div><div class="library-member-doc-line"> (use '(incanter core stats))</div><div class="library-member-doc-line"> (chisq-test :x [1 2 3 2 3 2 4 3 5]) ;; X-sq 2.6667</div><div class="library-member-doc-line"> ;; create a one-dimensional table of this data</div><div class="library-member-doc-line"> (def table (matrix [1 3 3 1 1]))</div><div class="library-member-doc-line"> (chisq-test :table table) ;; X-sq 2.6667</div><div class="library-member-doc-line"> (chisq-test :table (trans table)) ;; throws exception</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> (chisq-test :x [1 0 0 0 1 1 1 0 0 1 0 0 1 1 1 1]) ;; 0.25</div><div class="library-member-doc-line"> </div><div class="library-member-doc-line"> (use '(incanter core stats datasets))</div><div class="library-member-doc-line"> (def math-prog (to-matrix (get-dataset :math-prog)))</div><div class="library-member-doc-line"> (def x (sel math-prog :cols 1))</div><div class="library-member-doc-line"> (def y (sel math-prog :cols 2))</div><div class="library-member-doc-line"> (chisq-test :x x :y y) ;; X-sq = 1.24145, df=1, p-value = 0.26519</div><div class="library-member-doc-line"> (chisq-test :x x :y y :correct false) ;; X-sq = 2.01094, df=1, p-value = 0.15617</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> (def table (matrix [[31 12] [9 8]]))</div><div class="library-member-doc-line"> (chisq-test :table table) ;; X-sq = 1.24145, df=1, p-value = 0.26519</div><div class="library-member-doc-line"> (chisq-test :table table :correct false) ;; X-sq = 2.01094, df=1, p-value = 0.15617</div><div class="library-member-doc-line"> ;; use the detabulate function to create data rows corresponding to the table</div><div class="library-member-doc-line"> (def detab (detabulate :table table))</div><div class="library-member-doc-line"> (chisq-test :x (sel detab :cols 0) :y (sel detab :cols 1))</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> ;; look at the hair-eye-color data</div><div class="library-member-doc-line"> ;; turn the count data for males into a contigency table</div><div class="library-member-doc-line"> (def male (matrix (sel (get-dataset :hair-eye-color) :cols 3 :rows (range 16)) 4))</div><div class="library-member-doc-line"> (chisq-test :table male) ;; X-sq = 41.280, df = 9, p-value = 4.44E-6</div><div class="library-member-doc-line"> ;; turn the count data for females into a contigency table</div><div class="library-member-doc-line"> (def female (matrix (sel (get-dataset :hair-eye-color) :cols 3 :rows (range 16 32)) 4))</div><div class="library-member-doc-line"> (chisq-test :table female) ;; X-sq = 106.664, df = 9, p-value = 7.014E-19,</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> ;; supply probabilities to goodness-of-fit test</div><div class="library-member-doc-line"> (def table [89 37 30 28 2])</div><div class="library-member-doc-line"> (def probs [0.40 0.20 0.20 0.19 0.01])</div><div class="library-member-doc-line"> (chisq-test :table table :probs probs) ;; X-sq = 5.7947, df = 4, p-value = 0.215</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> ;; use frequencies instead of probabilities </div><div class="library-member-doc-line"> (def freq [40 20 20 15 5])</div><div class="library-member-doc-line"> (chisq-test :table table :freq freq) ;; X-sq = 9.9901, df = 4, p-value = 0.04059</div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"><span class="library-member-doc-whitespace"> </span></div><div class="library-member-doc-line"> References:</div><div class="library-member-doc-line"> http://www.itl.nist.gov/div898/handbook/eda/section3/eda35f.htm</div><div class="library-member-doc-line"> http://en.wikipedia.org/wiki/Pearson's_chi-square_test</div><div class="library-member-doc-line"> http://en.wikipedia.org/wiki/Yates'_chi-square_test</div></div><div class="library-member-source-section"><div class="library-member-source-toggle">[ <a href="javascript:toggleSource('membersource-incanter.stats-chisq-test')" id="linkto-membersource-incanter.stats-chisq-test">Show Source</a> ]</div><div class="library-member-source" id="membersource-incanter.stats-chisq-test"><pre>(defn chisq-test " ..." ([& options] (let [opts (if options (apply assoc {} options) nil) @@ -1945,25 +1945,32 @@ setLinkToggleText('library-contents-toggle-incanter.censored', '+ (range (ncol table)) (second (:levels xtab))) r-margins (if table? - (apply hash-map (interleave r-levels (map sum (trans table)))) + (if two-samp? + (apply hash-map (interleave r-levels (map sum (trans table)))) + (if (> (nrow table) 1) + (to-list table) + (throw (Exception. "One dimensional tables must have only a single column")))) (second (:margins xtab))) c-margins (if table? - (apply hash-map (interleave c-levels (map sum table))) + (if two-samp? + (apply hash-map (interleave c-levels (map sum table))) + 0) (first (:margins xtab))) - counts (vectorize table) - ;(if table? - ; (vectorize table) - ;(vals (:counts xtab))) ;; BAD + counts (if two-samp? (vectorize table) table) N (if table? (sum counts) (:N xtab)) n (when (not two-samp?) (count r-levels)) df (if two-samp? (* (dec (nrow table)) (dec (ncol table))) (dec n)) probs (when (not two-samp?) - (if (:probs opts) + (cond (:probs opts) - (repeat n (/ n)))) + (:probs opts) + (:freq opts) + (div (:freq opts) (sum (:freq opts))) + :else + (repeat n (/ n)))) E (if two-samp? (for [r r-levels c c-levels] (/ (* (c-margins c) (r-margins r)) N)) docs/api/index.html incanter.jar lib/clojure.jar @@ -2007,9 +2007,10 @@ Options: :x -- a sequence of numbers. :y -- a sequence of numbers - :table -- a contigency table + :table -- a contigency table. If one dimensional, the test is a goodness-of-fit :probs (when (nil? y) -- (repeat n-levels (/ n-levels))) - :correct (default true) -- use Yates' correction for continuity + :freq (default nil) -- if given, these are rescaled to probabilities + :correct (default true) -- use Yates' correction for continuity for 2x2 contingency tables Returns: @@ -2020,7 +2021,12 @@ Examples: (use '(incanter core stats)) - (chisq-test :x [1 2 3 2 3 2 4 3 5]) ;; 2.6667 + (chisq-test :x [1 2 3 2 3 2 4 3 5]) ;; X-sq 2.6667 + ;; create a one-dimensional table of this data + (def table (matrix [1 3 3 1 1])) + (chisq-test :table table) ;; X-sq 2.6667 + (chisq-test :table (trans table)) ;; throws exception + (chisq-test :x [1 0 0 0 1 1 1 0 0 1 0 0 1 1 1 1]) ;; 0.25 (use '(incanter core stats datasets)) @@ -2033,6 +2039,9 @@ (def table (matrix [[31 12] [9 8]])) (chisq-test :table table) ;; X-sq = 1.24145, df=1, p-value = 0.26519 (chisq-test :table table :correct false) ;; X-sq = 2.01094, df=1, p-value = 0.15617 + ;; use the detabulate function to create data rows corresponding to the table + (def detab (detabulate :table table)) + (chisq-test :x (sel detab :cols 0) :y (sel detab :cols 1)) ;; look at the hair-eye-color data ;; turn the count data for males into a contigency table @@ -2043,9 +2052,14 @@ (chisq-test :table female) ;; X-sq = 106.664, df = 9, p-value = 7.014E-19, - (def detab (detabulate :table table)) - (chisq-test :x (sel detab :cols 0) :y (sel detab :cols 1)) + ;; supply probabilities to goodness-of-fit test + (def table [89 37 30 28 2]) + (def probs [0.40 0.20 0.20 0.19 0.01]) + (chisq-test :table table :probs probs) ;; X-sq = 5.7947, df = 4, p-value = 0.215 + ;; use frequencies instead of probabilities + (def freq [40 20 20 15 5]) + (chisq-test :table table :freq freq) ;; X-sq = 9.9901, df = 4, p-value = 0.04059 @@ -2081,25 +2095,32 @@ (range (ncol table)) (second (:levels xtab))) r-margins (if table? - (apply hash-map (interleave r-levels (map sum (trans table)))) + (if two-samp? + (apply hash-map (interleave r-levels (map sum (trans table)))) + (if (> (nrow table) 1) + (to-list table) + (throw (Exception. "One dimensional tables must have only a single column")))) (second (:margins xtab))) c-margins (if table? - (apply hash-map (interleave c-levels (map sum table))) + (if two-samp? + (apply hash-map (interleave c-levels (map sum table))) + 0) (first (:margins xtab))) - counts (vectorize table) - ;(if table? - ; (vectorize table) - ;(vals (:counts xtab))) ;; BAD + counts (if two-samp? (vectorize table) table) N (if table? (sum counts) (:N xtab)) n (when (not two-samp?) (count r-levels)) df (if two-samp? (* (dec (nrow table)) (dec (ncol table))) (dec n)) probs (when (not two-samp?) - (if (:probs opts) + (cond (:probs opts) - (repeat n (/ n)))) + (:probs opts) + (:freq opts) + (div (:freq opts) (sum (:freq opts))) + :else + (repeat n (/ n)))) E (if two-samp? (for [r r-levels c c-levels] (/ (* (c-margins c) (r-margins r)) N)) src/incanter/stats.clj db974d086732e2362753dbeba0ccf139c84ffa4f David Edgar Liebke liebke@gmail.com http://github.com/liebke/incanter/commit/005a408a9ab2b0c669f156bde65fe1b3c32e258d 005a408a9ab2b0c669f156bde65fe1b3c32e258d

2009-06-21T16:38:27-07:00

added goodness-of-fit test to chisq-test function 1c2fa0abaf259a62564fe00f2ee9f8fa6ea04465 David Edgar Liebke liebke@gmail.com