From b0611627c150a473a838e4d8d011be8ed5035be2 Mon Sep 17 00:00:00 2001 From: Lingxiao Jiang Date: Sun, 17 Oct 2010 11:33:26 +0800 Subject: [PATCH] Release 1.2 --- LICENSE | 30 + README | 259 ++ .../sample-clone-report_100_0_allg_1.0_50 | 15 + samples/config | 70 + samples/src/AbstractAsyncTableRendering.java | 3065 ++++++++++++++ samples/src/AbstractTableRendering.java | 3740 +++++++++++++++++ samples/vectors/sample-vectors_100_0 | 188 + scripts/bugdetect/bugcounting | 66 + scripts/bugdetect/bugfiltering | 92 + scripts/bugdetect/bugmerging | 165 + scripts/bugdetect/bugordering | 101 + scripts/bugdetect/bugtypecounting | 153 + scripts/bugdetect/mergecomments | 178 + scripts/clonedetect/cd_coverage | 68 + scripts/clonedetect/config-sample | 102 + scripts/clonedetect/configure | 132 + scripts/clonedetect/deckard.sh | 133 + scripts/clonedetect/generateparam | 48 + scripts/clonedetect/paramsetting | 96 + scripts/clonedetect/post_process_groupfile | 248 ++ scripts/clonedetect/vdbgen | 159 + scripts/clonedetect/vertical-param-batch | 434 ++ src/examples/GridLayout.java | 742 ++++ src/examples/swim3.c | 1155 +++++ src/include/ptree.h | 295 ++ src/lsh/Makefile | 79 + src/lsh/README | 85 + src/lsh/bin/compile | 32 + src/lsh/bin/exact | 18 + src/lsh/bin/lsh | 42 + src/lsh/bin/lsh_computeParams | 38 + src/lsh/bin/lsh_fromParams | 27 + src/lsh/manual.ps | Bin 0 -> 233549 bytes src/lsh/sources/BasicDefinitions.h | 200 + src/lsh/sources/BucketHashing.cpp | 698 +++ src/lsh/sources/BucketHashing.h | 214 + src/lsh/sources/Geometry.cpp | 39 + src/lsh/sources/Geometry.h | 46 + src/lsh/sources/GlobalVars.cpp | 19 + src/lsh/sources/GlobalVars.h | 62 + src/lsh/sources/LSHMain.cpp | 362 ++ src/lsh/sources/LocalitySensitiveHashing.cpp | 762 ++++ src/lsh/sources/LocalitySensitiveHashing.h | 162 + src/lsh/sources/Makefile | 49 + src/lsh/sources/NearNeighbors.cpp | 199 + src/lsh/sources/NearNeighbors.h | 27 + src/lsh/sources/Random.cpp | 76 + src/lsh/sources/Random.h | 31 + src/lsh/sources/SelfTuning.cpp | 574 +++ src/lsh/sources/SelfTuning.h | 47 + src/lsh/sources/Util.cpp | 88 + src/lsh/sources/Util.h | 35 + src/lsh/sources/compareOutputs.cpp | 126 + src/lsh/sources/enumBuckets.cpp | 733 ++++ src/lsh/sources/exactNNs.cpp | 186 + src/lsh/sources/exploreBuckets.cpp | 431 ++ src/lsh/sources/genDS.cpp | 75 + src/lsh/sources/genPlantedDS.cpp | 81 + src/lsh/sources/headers.h | 41 + src/lsh/sources/testFloat.cpp | 24 + src/main/Makefile | 118 + src/main/bugmain.cc | 172 + src/main/build.sh | 89 + src/main/clean.sh | 59 + src/main/main.cc | 296 ++ src/main/out2html.C | 86 + src/main/out2xml.C | 95 + src/main/parseTreeMain.cc | 141 + src/main/ptree.cc | 423 ++ src/ptgen/Makefile | 47 + src/ptgen/YaccLexer.py | 483 +++ src/ptgen/YaccParser.py | 269 ++ src/ptgen/antlr.py | 2833 +++++++++++++ src/ptgen/gcc/Makefile | 60 + src/ptgen/gcc/c.l | 408 ++ src/ptgen/gcc/c.y | 2233 ++++++++++ src/ptgen/gcc/c.y.foot | 12 + src/ptgen/gcc/c.y.head | 97 + src/ptgen/gcc/catomicNodes.h | 34 + src/ptgen/gcc/ccontextualNodes.h | 42 + src/ptgen/gcc/cparentNodes.h | 37 + src/ptgen/gcc/crelevantNodes.h | 129 + src/ptgen/gcc/main.cc | 69 + src/ptgen/gcc/mainc.py | 184 + src/ptgen/java/Makefile | 60 + src/ptgen/java/j.l | 225 + src/ptgen/java/j.y | 2076 +++++++++ src/ptgen/java/j.y.foot | 12 + src/ptgen/java/j.y.head | 54 + src/ptgen/java/jatomicNodes.h | 34 + src/ptgen/java/jcontextualNodes.h | 55 + src/ptgen/java/jparentNodes.h | 34 + src/ptgen/java/jrelevantNodes.h | 175 + src/ptgen/java/main.cc | 66 + src/ptgen/java/mainj.py | 187 + src/ptgen/php5/Makefile | 61 + src/ptgen/php5/mainphp.py | 187 + src/ptgen/php5/phpatomicNodes.h | 97 + src/ptgen/php5/phpcontextualNodes.h | 45 + src/ptgen/php5/phpparentNodes.h | 40 + src/ptgen/php5/phprelevantNodes.h | 242 ++ src/ptgen/php5/zend_language_parser.y | 778 ++++ src/ptgen/php5/zend_language_parser.y.foot | 12 + src/ptgen/php5/zend_language_parser.y.head | 97 + src/ptgen/php5/zend_language_scanner.l | 941 +++++ src/ptgen/simple/Makefile | 54 + src/ptgen/simple/c.l | 450 ++ src/ptgen/simple/c.y | 419 ++ src/ptgen/simple/c.y.foot | 13 + src/ptgen/simple/c.y.head | 14 + src/ptgen/simple/main.cc | 60 + src/ptgen/simple/tokid.h | 179 + src/ptgen/yacc.g | 156 + src/vgen/samplevec | 21 + src/vgen/tra-gen-test.C | 45 + src/vgen/treeTra/Makefile | 76 + src/vgen/treeTra/clone-context-php.C | 417 ++ src/vgen/treeTra/clone-context-php.h | 59 + src/vgen/treeTra/node-vec-gen.C | 119 + src/vgen/treeTra/node-vec-gen.h | 66 + src/vgen/treeTra/sq-tree.C | 213 + src/vgen/treeTra/sq-tree.h | 93 + src/vgen/treeTra/token-counter.C | 205 + src/vgen/treeTra/token-counter.h | 90 + src/vgen/treeTra/token-tree-map.C | 1414 +++++++ src/vgen/treeTra/token-tree-map.h | 201 + src/vgen/treeTra/tra-gen.C | 215 + src/vgen/treeTra/tra-gen.h | 70 + src/vgen/treeTra/tree-accessor.C | 342 ++ src/vgen/treeTra/tree-accessor.h | 66 + src/vgen/treeTra/tree-traversal.C | 157 + src/vgen/treeTra/tree-vector.C | 397 ++ src/vgen/treeTra/tree-vector.h | 89 + src/vgen/treeTra/vector-merger.C | 670 +++ src/vgen/treeTra/vector-merger.h | 131 + src/vgen/treeTra/vector-output.C | 168 + src/vgen/treeTra/vector-output.h | 77 + src/vgen/treeTra/vgen-config.C | 192 + src/vgen/treeTra/vgen-config.h | 88 + src/vgen/treeTra/vgen-utils.c | 59 + src/vgen/treeTra/vgen-utils.h | 57 + src/vgen/vgrouping/Makefile | 45 + src/vgen/vgrouping/computeranges.c | 145 + src/vgen/vgrouping/dispatchvectors.c | 414 ++ src/vgen/vgrouping/rundispatch | 97 + src/vgen/vgrouping/rundispatchonefile | 97 + src/vgen/vgrouping/runsplit | 74 + src/vgen/vgrouping/runvectorsort | 74 + src/vgen/vgrouping/split.c | 490 +++ src/vgen/vgrouping/vectorsort.c | 411 ++ 150 files changed, 40325 insertions(+) create mode 100644 LICENSE create mode 100755 README create mode 100644 samples/clusters/sample-clone-report_100_0_allg_1.0_50 create mode 100644 samples/config create mode 100644 samples/src/AbstractAsyncTableRendering.java create mode 100644 samples/src/AbstractTableRendering.java create mode 100644 samples/vectors/sample-vectors_100_0 create mode 100644 scripts/bugdetect/bugcounting create mode 100644 scripts/bugdetect/bugfiltering create mode 100644 scripts/bugdetect/bugmerging create mode 100644 scripts/bugdetect/bugordering create mode 100644 scripts/bugdetect/bugtypecounting create mode 100644 scripts/bugdetect/mergecomments create mode 100644 scripts/clonedetect/cd_coverage create mode 100644 scripts/clonedetect/config-sample create mode 100644 scripts/clonedetect/configure create mode 100644 scripts/clonedetect/deckard.sh create mode 100644 scripts/clonedetect/generateparam create mode 100644 scripts/clonedetect/paramsetting create mode 100644 scripts/clonedetect/post_process_groupfile create mode 100644 scripts/clonedetect/vdbgen create mode 100644 scripts/clonedetect/vertical-param-batch create mode 100644 src/examples/GridLayout.java create mode 100644 src/examples/swim3.c create mode 100755 src/include/ptree.h create mode 100755 src/lsh/Makefile create mode 100755 src/lsh/README create mode 100755 src/lsh/bin/compile create mode 100755 src/lsh/bin/exact create mode 100755 src/lsh/bin/lsh create mode 100755 src/lsh/bin/lsh_computeParams create mode 100755 src/lsh/bin/lsh_fromParams create mode 100755 src/lsh/manual.ps create mode 100755 src/lsh/sources/BasicDefinitions.h create mode 100755 src/lsh/sources/BucketHashing.cpp create mode 100755 src/lsh/sources/BucketHashing.h create mode 100755 src/lsh/sources/Geometry.cpp create mode 100755 src/lsh/sources/Geometry.h create mode 100755 src/lsh/sources/GlobalVars.cpp create mode 100755 src/lsh/sources/GlobalVars.h create mode 100755 src/lsh/sources/LSHMain.cpp create mode 100755 src/lsh/sources/LocalitySensitiveHashing.cpp create mode 100755 src/lsh/sources/LocalitySensitiveHashing.h create mode 100755 src/lsh/sources/Makefile create mode 100755 src/lsh/sources/NearNeighbors.cpp create mode 100755 src/lsh/sources/NearNeighbors.h create mode 100755 src/lsh/sources/Random.cpp create mode 100755 src/lsh/sources/Random.h create mode 100755 src/lsh/sources/SelfTuning.cpp create mode 100755 src/lsh/sources/SelfTuning.h create mode 100755 src/lsh/sources/Util.cpp create mode 100755 src/lsh/sources/Util.h create mode 100755 src/lsh/sources/compareOutputs.cpp create mode 100755 src/lsh/sources/enumBuckets.cpp create mode 100755 src/lsh/sources/exactNNs.cpp create mode 100755 src/lsh/sources/exploreBuckets.cpp create mode 100755 src/lsh/sources/genDS.cpp create mode 100755 src/lsh/sources/genPlantedDS.cpp create mode 100755 src/lsh/sources/headers.h create mode 100755 src/lsh/sources/testFloat.cpp create mode 100755 src/main/Makefile create mode 100755 src/main/bugmain.cc create mode 100755 src/main/build.sh create mode 100644 src/main/clean.sh create mode 100755 src/main/main.cc create mode 100755 src/main/out2html.C create mode 100755 src/main/out2xml.C create mode 100755 src/main/parseTreeMain.cc create mode 100755 src/main/ptree.cc create mode 100644 src/ptgen/Makefile create mode 100755 src/ptgen/YaccLexer.py create mode 100755 src/ptgen/YaccParser.py create mode 100755 src/ptgen/antlr.py create mode 100755 src/ptgen/gcc/Makefile create mode 100755 src/ptgen/gcc/c.l create mode 100755 src/ptgen/gcc/c.y create mode 100755 src/ptgen/gcc/c.y.foot create mode 100755 src/ptgen/gcc/c.y.head create mode 100644 src/ptgen/gcc/catomicNodes.h create mode 100644 src/ptgen/gcc/ccontextualNodes.h create mode 100644 src/ptgen/gcc/cparentNodes.h create mode 100644 src/ptgen/gcc/crelevantNodes.h create mode 100755 src/ptgen/gcc/main.cc create mode 100644 src/ptgen/gcc/mainc.py create mode 100755 src/ptgen/java/Makefile create mode 100755 src/ptgen/java/j.l create mode 100755 src/ptgen/java/j.y create mode 100755 src/ptgen/java/j.y.foot create mode 100755 src/ptgen/java/j.y.head create mode 100644 src/ptgen/java/jatomicNodes.h create mode 100644 src/ptgen/java/jcontextualNodes.h create mode 100644 src/ptgen/java/jparentNodes.h create mode 100644 src/ptgen/java/jrelevantNodes.h create mode 100755 src/ptgen/java/main.cc create mode 100644 src/ptgen/java/mainj.py create mode 100755 src/ptgen/php5/Makefile create mode 100644 src/ptgen/php5/mainphp.py create mode 100755 src/ptgen/php5/phpatomicNodes.h create mode 100755 src/ptgen/php5/phpcontextualNodes.h create mode 100755 src/ptgen/php5/phpparentNodes.h create mode 100755 src/ptgen/php5/phprelevantNodes.h create mode 100755 src/ptgen/php5/zend_language_parser.y create mode 100755 src/ptgen/php5/zend_language_parser.y.foot create mode 100755 src/ptgen/php5/zend_language_parser.y.head create mode 100755 src/ptgen/php5/zend_language_scanner.l create mode 100755 src/ptgen/simple/Makefile create mode 100755 src/ptgen/simple/c.l create mode 100755 src/ptgen/simple/c.y create mode 100644 src/ptgen/simple/c.y.foot create mode 100644 src/ptgen/simple/c.y.head create mode 100755 src/ptgen/simple/main.cc create mode 100755 src/ptgen/simple/tokid.h create mode 100755 src/ptgen/yacc.g create mode 100755 src/vgen/samplevec create mode 100755 src/vgen/tra-gen-test.C create mode 100755 src/vgen/treeTra/Makefile create mode 100755 src/vgen/treeTra/clone-context-php.C create mode 100755 src/vgen/treeTra/clone-context-php.h create mode 100755 src/vgen/treeTra/node-vec-gen.C create mode 100755 src/vgen/treeTra/node-vec-gen.h create mode 100755 src/vgen/treeTra/sq-tree.C create mode 100755 src/vgen/treeTra/sq-tree.h create mode 100755 src/vgen/treeTra/token-counter.C create mode 100755 src/vgen/treeTra/token-counter.h create mode 100755 src/vgen/treeTra/token-tree-map.C create mode 100755 src/vgen/treeTra/token-tree-map.h create mode 100755 src/vgen/treeTra/tra-gen.C create mode 100755 src/vgen/treeTra/tra-gen.h create mode 100755 src/vgen/treeTra/tree-accessor.C create mode 100755 src/vgen/treeTra/tree-accessor.h create mode 100755 src/vgen/treeTra/tree-traversal.C create mode 100755 src/vgen/treeTra/tree-vector.C create mode 100755 src/vgen/treeTra/tree-vector.h create mode 100755 src/vgen/treeTra/vector-merger.C create mode 100755 src/vgen/treeTra/vector-merger.h create mode 100755 src/vgen/treeTra/vector-output.C create mode 100755 src/vgen/treeTra/vector-output.h create mode 100755 src/vgen/treeTra/vgen-config.C create mode 100755 src/vgen/treeTra/vgen-config.h create mode 100755 src/vgen/treeTra/vgen-utils.c create mode 100755 src/vgen/treeTra/vgen-utils.h create mode 100755 src/vgen/vgrouping/Makefile create mode 100644 src/vgen/vgrouping/computeranges.c create mode 100644 src/vgen/vgrouping/dispatchvectors.c create mode 100644 src/vgen/vgrouping/rundispatch create mode 100644 src/vgen/vgrouping/rundispatchonefile create mode 100755 src/vgen/vgrouping/runsplit create mode 100755 src/vgen/vgrouping/runvectorsort create mode 100755 src/vgen/vgrouping/split.c create mode 100755 src/vgen/vgrouping/vectorsort.c diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4c77d6c --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ + +Copyright (c) 2007-2010, + Lingxiao Jiang + Ghassan Misherghi + Zhendong Su + Stephane Glondu +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/README b/README new file mode 100755 index 0000000..641c8a8 --- /dev/null +++ b/README @@ -0,0 +1,259 @@ + +This is a release package of Deckard -- a tree-based, scalable, and accurate +code clone detection tool. It is also capable of reporting clone-related bugs. + + +********************************************************************** +* +* LICENSE +* +********************************************************************** + +Copyright (c) 2007-2010, University of California + Lingxiao Jiang + Ghassan Misherghi + Zhendong Su + Stephane Glondu +All rights reserved. +Three-clause BSD licence + + +********************************************************************** +* +* Version 1.2 +* Oct 17, 2010 +* +********************************************************************** + + +********************************************************************** +* +* Installation +* +********************************************************************** + +In bash shell or cygwin, run the build script: + +/path/to/src/main/build.sh + +For convenience, add "src/main" into $PATH. + +NOTE: Deckard's built-in parser for Java cannot handle Java 1.5 or later +features, which means when Deckard processes a Java 1.5 file, it is very likely +there will be no vector generated. + +NOTE: The compiled executables may not be "executable" (showing "Permission +Denied") on Windows Vista/7 due to false alarms of UAC rules (based on file +path/hash of a .exe). A simple (but may not be desirable) workaround is to run +cygwin shell with elevated privileges before invoking the above scripts. Also, +Deckard's performance may be tens of times slower when executed in cygwin than +on Linux due to slow I/O operations. + +To uninstall, simply + +/path/to/src/main/clean.sh + + +********************************************************************* +* +* Usage +* +********************************************************************* + +1. For clone detection (suppose the source code of your application is +in /path/to/app/src): + + - Specify the location of your source code, say /path/to/app/src. + + - Create a "config" file in /path/to/app/, following the sample "config" in + samples/ or the template "config-sample" in scripts/clonedetect/. Make sure + all paths are valid and the programming language is specified correctly. + + - (Optional) create other three directories in /path/to/app/ for storing + outputs (see what's in samples/). These directories may be automatically + created if specified in 'config'. + + - Batch mode run of clone detection (no bug detection by default): + + "/path/to/scripts/clonedetect/deckard.sh" + + An optional parameter to the script is 'clean', 'clean_all', or 'overwrite' + + - Instead of running 'deckard.sh', you may also run the scripts called in + 'deckard.sh' step-by-step by yourself: + + -- Vector generation: from where "config" is, run + + "/path/to/scripts/clonedetect/vdbgen" + + An optional parameter to the script is 'clean', 'clean_all', or 'overwrite' + + -- Vector clustering (i.e., clone detection): from where "config" is, run + + "/path/to/scripts/clonedetect/vertical-param-batch" + + An optional parameter to the script is 'clean', 'clean_all', or 'overwrite' + +2. Vector generation for parts of a file: + + - Identify the source file name, say /path/to/src/filename.java and the range + [s, e] of line numbers you'd like to have a vector generated + + - Run "src/main/jvecgen [options] /path/to/src/filename.java --start-line-number s --end-line-number e" + Run "jvecgen -h" for more options. Note that different vecgen (cvecgen, + jvecgen, phpvecgen) should be used for files in different languages. + +This vecgen command will generate a vector representing the code between Line 's' and +'e' in the source file, and store the vector in "filename.java.vec" by default. + +3. Detection of clone-related bugs: + + - Invoke 'bugfiltering' on a clone report file with a specified language, e.g., + + /path/to/scripts/bugdetect/bugfiltering cluster_result c > bug_result + + - Optionally transform 'bug_result' to a html file for easier inspection of + the reported potentially buggy clones in a web browser: + + /path/to/src/main/out2html bug_result > bug_result.html + + - See 'deckard.sh' for how to run it in a batch mode (not enabled by default). + + +********************************************************************* +* +* What are in the package +* +********************************************************************* + +1. Organization + +The whole package is organized according to the several components in Deckard: + + - Parse tree generation + -- src/include/ : a generic interface for trees + -- src/ptgen/ : ANTLR parser generator + -- src/ptgen/gcc : a grammar for C (GNU C extensions) and its parse tree generator + -- src/ptgen/java : a grammar for Java (<=1.4) and its parse tree generator + -- src/ptgen/php5 : a grammar for php5 and its parse tree generator + + - Vector generation + -- src/vgen/treeTra/ : a generic tree traversal framework based on the + generic tree interface in src/include, and vector generation based on tree + traversal, mostly C++. + -- src/vgen/vgrouping/ : code for vector grouping (mix of C,C++,python,bash) + + - Vector clustering + -- src/lsh/ : the LSH package and an interface for Deckard to use + (src/lsh/source/enumBuckets.cpp). + + - Main entrances + -- src/main/ptree.cc : an implementation of the tree interface + -- src/main/main.cc : entrance for vector generation + -- src/main/parseTreeMain.cc : entrance for parse tree dumping, can be + useful for inspecting detected clones, bugs, and their related parse trees + -- src/main/bugmain.cc : entrance for bug filtering + -- src/main/out2html.C : entrance for adding html tags into clone/bug reports + + - Scripts gluing things together + -- scripts/clonedetect/ : bash and python scripts + --- deckard.sh : batch-mode clone detection + --- vdbgen : batch-mode vector generation + --- vertical-param-batch : batch-mode vector clustering + -- scripts/bugdetect/ : bash and python scripts + -- various auxiliary scripts for simple statistics + + - Others + -- README + -- LICENSE + +2. Details about the clone/bug detection algorithms can be found in these two +papers: + + - DECKARD: Scalable and Accurate Tree-based Detection of Code Clones, by + Lingxiao JIANG, Ghassan MISHERGHI, Zhendong SU, and Stephane GLONDU. In the + proceedings of 29th International Conference on Software Engineering (ICSE + '07), Minneapolis, Minnesota, USA, 2007. + + - Context-Based Detection of Clone-Related Bugs, by Lingxiao JIANG, Zhendong + SU, and Edwin CHIU. In the proceedings of the 6th joint meeting of the + European Software Engineering Conference and the ACM SIGSOFT Symposium on the + Foundations of Software Engineering (ESEC/FSE'07), Dubrovnik, Croatia, 2007. + + +********************************************************************** +* +* How to programmably use the vectors and the clone reports? +* +********************************************************************** + +1. How to get the subtree representing each clone? + +Each clone in the reports has a TBID and a TEID, in addition to the file name, +and line numbers. The TBID and TEID uniquely identify the IDs of the first +token and the last token in the clone from the original file (possibly +containing parsing errors). To maintain consistent counting of the IDs, you +should leave the work to "yyparse()" and Deckard's TokenCounter for how the IDs +are calculated (see TraGenMain::run() for implementation details). + +The following are the main steps for getting the subtree for a clone (please +refer to "src/vgen/treeTra/token-tree-map.h" for more implementation details): + + - Given a line from the clone report file, parse it to get file name, line + numbers, TBID, and TEID, etc. C.f. the function: + + bool parse(char * line, regex_t patterns[], int dim=ENUM_CLONE_THE_END) + + - Call the following function (which calls "yyparse()" and a token counter) to + get a whole parse tree for the source file and the token IDs for every node: + + ParseTree* TokenTreeMap::parseFile(const char * filename) + + - Call the following function to get the smallest tree that contains all + tokens between TBID and TEID: + + Tree* tokenRange2Tree2(std::pair tokenrange, ParseTree* pt) + + - Then do whatever you'd like with the returned tree. Note that vectors are + NOT generated for this tree yet. If vectors are needed, do the following: + + -- Create a new object of type TraGenMain and call "TraGenMain::run(0, 0)" + (c.f., src/main/main.cc) + + -- Retrieve the vector for the tree: + + TreeVector* tv = TreeAccessor::get_node_vector(Tree* tree_node_pointer) + + -- If you also want some merged vectors from the child nodes of this tree, + that would require calls to TraGenMain::run() with different parameters + or adjust the internals of TraGenMain::run(), depending on how you want + the vectors to be presented to you. Feel free to improve the vector + generation, both the core and its interface/APIs. + +2. How to get the vector for a line or a sequence of lines from a file? + + - Option 1: See above: Use "vector generation for parts of a file" with your + scripts. + + - Option 2: Given the parse tree for a file (produced by + TokenTreeMap::parseFile() and yyparse()) and the starting and ending line + numbers, do the following: + + -- (If not done before,) Call Deckard's vector generator on the parse tree + through TraGenMain::run, same as above. Please refer to src/main/main.cc, + TraGenMain::run(int startln, int endln), and + VecGenerator::traverse(Tree* root, Tree* init). + + -- Call the following function (c.f. src/include/ptree.h, src/main/ptree.cc) to + return the smallest tree enclosing all elements from these lines: + + Tree* ParseTree::line2Tree(int startln, int endln) + + -- Then retrieve the vector (the actual vector generation is done beforehand): + + TreeVector* tv = TreeAccessor::get_node_vector(tree_node_pointer) + + +Enjoy and Feedback :=) +@Deckard : Am I a clone? + diff --git a/samples/clusters/sample-clone-report_100_0_allg_1.0_50 b/samples/clusters/sample-clone-report_100_0_allg_1.0_50 new file mode 100644 index 0000000..f06c0b6 --- /dev/null +++ b/samples/clusters/sample-clone-report_100_0_allg_1.0_50 @@ -0,0 +1,15 @@ +000000000 dist:0.0 FILE src/AbstractAsyncTableRendering.java LINE:845:46 NODE_KIND:103 nVARs:26 NUM_NODE:592 TBID:4144 TEID:4354 +000000001 dist:0.0 FILE src/AbstractTableRendering.java LINE:900:46 NODE_KIND:103 nVARs:26 NUM_NODE:592 TBID:3928 TEID:4138 + +000000000 dist:0.0 FILE src/AbstractAsyncTableRendering.java LINE:1187:28 NODE_KIND:103 nVARs:16 NUM_NODE:364 TBID:5638 TEID:5757 +000000001 dist:0.0 FILE src/AbstractTableRendering.java LINE:960:28 NODE_KIND:103 nVARs:16 NUM_NODE:364 TBID:4188 TEID:4307 + +000000000 dist:0.0 FILE src/AbstractAsyncTableRendering.java LINE:1226:28 NODE_KIND:103 nVARs:16 NUM_NODE:384 TBID:5806 TEID:5932 +000000001 dist:0.0 FILE src/AbstractTableRendering.java LINE:999:28 NODE_KIND:103 nVARs:16 NUM_NODE:384 TBID:4356 TEID:4482 + +000000000 dist:0.0 FILE src/AbstractAsyncTableRendering.java LINE:1756:26 NODE_KIND:103 nVARs:17 NUM_NODE:380 TBID:7812 TEID:7951 +000000002 dist:0.0 FILE src/AbstractTableRendering.java LINE:1938:26 NODE_KIND:103 nVARs:17 NUM_NODE:380 TBID:7830 TEID:7969 + +000000000 dist:0.0 FILE src/AbstractAsyncTableRendering.java LINE:2707:8 NODE_KIND:103 nVARs:15 NUM_NODE:332 TBID:11300 TEID:11407 +000000001 dist:0.0 FILE src/AbstractTableRendering.java LINE:3559:8 NODE_KIND:103 nVARs:15 NUM_NODE:332 TBID:13928 TEID:14035 + diff --git a/samples/config b/samples/config new file mode 100644 index 0000000..513c0f0 --- /dev/null +++ b/samples/config @@ -0,0 +1,70 @@ +################################################################### +# Configuration file for clone detection. +# + +################################################################### +# Often, need to change these common parameters: +# - FILE_PATTERN : for source files in different languages +# - SRC_DIR : the root directory containing the source files +# - DECKARD_DIR : Where is the home directory of DECKARD +# - clone detection parameters: c.f. DECKARD's paper +# -- MIN_TOKENS +# -- STRIDE +# -- SIMILARITY +# +# java, c, or php? +FILE_PATTERN='*.java' # used for the 'find' command +# where are the source files? +SRC_DIR='src' +# where is Deckard? +DECKARD_DIR=".." +# clone parameters; refer to paper. +MIN_TOKENS='50 100' +STRIDE='2 0' +#DISTANCE='2.236 0.70711 1.58114' +SIMILARITY='1.0 0.95' + +################################################################### +# Where to store result files? +# +# where to output generated vectors? +VECTOR_DIR='vectors' +# where to output detected clone clusters? +CLUSTER_DIR='clusters' +# where to output timing/debugging info? +TIME_DIR='times' + +################################################################### +# where are several programs we need? +# +# where is the vector generator? +VGEN_EXEC="$DECKARD_DIR/src/main" +case $FILE_PATTERN in + *.java ) + VGEN_EXEC="$VGEN_EXEC/jvecgen" ;; + *.php ) + VGEN_EXEC="$VGEN_EXEC/phpvecgen" ;; + *.c | *.h ) + VGEN_EXEC="$VGEN_EXEC/cvecgen" ;; + * ) + echo "Error: invalid FILE_PATTERN: $FILE_PATTERN" + VGEN_EXEC="$VGEN_EXEC/invalidvecgen" ;; +esac +# how to divide the vectors into groups? It's just the directory name that matters +GROUPING_EXEC="$DECKARD_DIR/src/vgen/vgrouping/runvectorsort" +# where is the lsh? +CLUSTER_EXEC="$DECKARD_DIR/src/lsh/bin/enumBuckets" +# how to post process clone groups? +POSTPRO_EXEC="$DECKARD_DIR/scripts/clonedetect/post_process_groupfile" +# how to transform source code html? +SRC2HTM_EXEC=source-highlight +SRC2HTM_OPTS=--line-number-ref + +################################################################### +# Some additional, internal parameters; can be ignored +# +# the maximal vector size for the first group; not really useful +GROUPING_S='50' +#GROUPING_D +#GROUPING_C + diff --git a/samples/src/AbstractAsyncTableRendering.java b/samples/src/AbstractAsyncTableRendering.java new file mode 100644 index 0000000..315f061 --- /dev/null +++ b/samples/src/AbstractAsyncTableRendering.java @@ -0,0 +1,3065 @@ +/******************************************************************************* + * Copyright (c) 2006 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ + +package org.eclipse.debug.internal.ui.memory.provisional; + + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Iterator; + +import org.eclipse.core.commands.AbstractHandler; +import org.eclipse.core.commands.Command; +import org.eclipse.core.commands.ExecutionEvent; +import org.eclipse.core.commands.ExecutionException; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.runtime.jobs.ISchedulingRule; +import org.eclipse.core.runtime.jobs.Job; +import org.eclipse.debug.core.DebugException; +import org.eclipse.debug.core.model.IMemoryBlock; +import org.eclipse.debug.core.model.IMemoryBlockExtension; +import org.eclipse.debug.core.model.MemoryByte; +import org.eclipse.debug.internal.ui.DebugUIMessages; +import org.eclipse.debug.internal.ui.DebugUIPlugin; +import org.eclipse.debug.internal.ui.IInternalDebugUIConstants; +import org.eclipse.debug.internal.ui.memory.IPersistableDebugElement; +import org.eclipse.debug.internal.ui.preferences.IDebugPreferenceConstants; +import org.eclipse.debug.internal.ui.viewers.model.provisional.IModelChangedListener; +import org.eclipse.debug.internal.ui.viewers.model.provisional.IModelDelta; +import org.eclipse.debug.internal.ui.viewers.model.provisional.IModelProxy; +import org.eclipse.debug.internal.ui.viewers.model.provisional.IStatusMonitor; +import org.eclipse.debug.internal.ui.views.memory.MemoryViewUtil; +import org.eclipse.debug.internal.ui.views.memory.renderings.AbstractBaseTableRendering; +import org.eclipse.debug.internal.ui.views.memory.renderings.AbstractVirtualContentTableModel; +import org.eclipse.debug.internal.ui.views.memory.renderings.AsyncCopyTableRenderingAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.AsyncPrintTableRenderingAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.AsyncTableRenderingCellModifier; +import org.eclipse.debug.internal.ui.views.memory.renderings.AsyncTableRenderingViewer; +import org.eclipse.debug.internal.ui.views.memory.renderings.AsyncVirtualContentTableViewer; +import org.eclipse.debug.internal.ui.views.memory.renderings.CopyTableRenderingToClipboardAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.FormatTableRenderingAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.FormatTableRenderingDialog; +import org.eclipse.debug.internal.ui.views.memory.renderings.GoToAddressAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.GoToAddressComposite; +import org.eclipse.debug.internal.ui.views.memory.renderings.IPresentationErrorListener; +import org.eclipse.debug.internal.ui.views.memory.renderings.IVirtualContentListener; +import org.eclipse.debug.internal.ui.views.memory.renderings.MemorySegment; +import org.eclipse.debug.internal.ui.views.memory.renderings.PendingPropertyChanges; +import org.eclipse.debug.internal.ui.views.memory.renderings.PrintTableRenderingAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.ReformatAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.ResetToBaseAddressAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingContentDescriptor; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingLine; +import org.eclipse.debug.ui.DebugUITools; +import org.eclipse.debug.ui.IDebugUIConstants; +import org.eclipse.debug.ui.memory.AbstractTableRendering; +import org.eclipse.debug.ui.memory.IMemoryBlockTablePresentation; +import org.eclipse.debug.ui.memory.IMemoryRendering; +import org.eclipse.debug.ui.memory.IMemoryRenderingContainer; +import org.eclipse.debug.ui.memory.IMemoryRenderingSite; +import org.eclipse.debug.ui.memory.IMemoryRenderingSynchronizationService; +import org.eclipse.debug.ui.memory.IMemoryRenderingType; +import org.eclipse.debug.ui.memory.IResettableMemoryRendering; +import org.eclipse.jface.action.Action; +import org.eclipse.jface.action.IMenuListener; +import org.eclipse.jface.action.IMenuManager; +import org.eclipse.jface.action.MenuManager; +import org.eclipse.jface.action.Separator; +import org.eclipse.jface.dialogs.IDialogConstants; +import org.eclipse.jface.preference.IPreferenceStore; +import org.eclipse.jface.resource.ImageDescriptor; +import org.eclipse.jface.resource.JFaceResources; +import org.eclipse.jface.text.Document; +import org.eclipse.jface.text.TextViewer; +import org.eclipse.jface.util.IPropertyChangeListener; +import org.eclipse.jface.util.PropertyChangeEvent; +import org.eclipse.jface.viewers.CellEditor; +import org.eclipse.jface.viewers.IBasicPropertyConstants; +import org.eclipse.jface.viewers.ICellModifier; +import org.eclipse.jface.viewers.IColorProvider; +import org.eclipse.jface.viewers.IFontProvider; +import org.eclipse.jface.viewers.ILabelProvider; +import org.eclipse.jface.viewers.ISelectionChangedListener; +import org.eclipse.jface.viewers.ISelectionProvider; +import org.eclipse.jface.viewers.SelectionChangedEvent; +import org.eclipse.jface.viewers.StructuredViewer; +import org.eclipse.jface.viewers.TextCellEditor; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.SashForm; +import org.eclipse.swt.custom.StyledText; +import org.eclipse.swt.events.ControlEvent; +import org.eclipse.swt.events.ControlListener; +import org.eclipse.swt.events.KeyAdapter; +import org.eclipse.swt.events.KeyEvent; +import org.eclipse.swt.events.MouseEvent; +import org.eclipse.swt.events.MouseTrackAdapter; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.graphics.Font; +import org.eclipse.swt.graphics.Point; +import org.eclipse.swt.graphics.Rectangle; +import org.eclipse.swt.layout.GridData; +import org.eclipse.swt.layout.GridLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Menu; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Table; +import org.eclipse.swt.widgets.TableColumn; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swt.widgets.Text; +import org.eclipse.ui.IWorkbench; +import org.eclipse.ui.IWorkbenchActionConstants; +import org.eclipse.ui.PlatformUI; +import org.eclipse.ui.commands.ICommandService; +import org.eclipse.ui.contexts.IContextService; +import org.eclipse.ui.dialogs.PropertyDialogAction; +import org.eclipse.ui.model.IWorkbenchAdapter; +import org.eclipse.ui.part.PageBook; +import org.eclipse.ui.progress.UIJob; + +/** + * Abstract implementation of a table rendering. + *

+ * Clients should subclass from this class if they wish to provide a + * table rendering. + *

+ *

+ * + * The label of the rendering is constructed by retrieving the expression from + * IMemoryBlockExtension. For IMemoryBlock, the label is constructed + * using the memory block's start address. + * + * This rendering manages the change states of its memory bytes if the memory + * block does not opt to manage the change states. For IMemoryBlockExtension, if + * the memory block returns false when #supportsChangeManagement() is called, this + * rendering will calculate the change state for each byte when its content is updated. + * Clients may manages the change states of its memory block by returning true when + * #supportsChangeManagement() is called. This will cause this rendering to stop + * calculating the change states of the memory block. Instead it would rely on the + * attributes returned in the MemoryByte array to determine if a byte has changed. + * For IMemoryBlock, this rendering will manage the change states its content. + * + * When firing change event, be aware of the following: + * - whenever a change event is fired, the content provider for Memory View + * view checks to see if memory has actually changed. + * - If memory has actually changed, a refresh will commence. Changes to the memory block + * will be computed and will be shown with the delta icons. + * - If memory has not changed, content will not be refreshed. However, previous delta information + * will be erased. The screen will be refreshed to show that no memory has been changed. (All + * delta icons will be removed.) + * + * Please note that these APIs will be called multiple times by the Memory View. + * To improve performance, debug adapters need to cache the content of its memory block and only + * retrieve updated data when necessary. + *

+ + * @since 3.2 + */ +public abstract class AbstractAsyncTableRendering extends AbstractBaseTableRendering implements IPropertyChangeListener, IResettableMemoryRendering { + + /** + * Property identifier for the selected address in a table rendering + * This property is used for synchronization between renderings. + */ + public static final String PROPERTY_SELECTED_ADDRESS = AbstractTableRendering.PROPERTY_SELECTED_ADDRESS; + + /** + * Property identifier for the column size in a table rendering + * This property is used for synchronization between renderings. + */ + public static final String PROPERTY_COL_SIZE = AbstractTableRendering.PROPERTY_COL_SIZE; + + /** + * Property identifier for the top row address in a table rendering. + * This property is used for synchronization between renderings. + */ + public static final String PROPERTY_TOP_ADDRESS = AbstractTableRendering.PROPERTY_TOP_ADDRESS; + + private static final String ID_ASYNC_TABLE_RENDERING_CONTEXT = "org.eclipse.debug.ui.memory.abstractasynctablerendering"; //$NON-NLS-1$ + private static final String ID_GO_TO_ADDRESS_COMMAND = "org.eclipse.debug.ui.command.gotoaddress"; //$NON-NLS-1$ + + /** + * Property identifier for the row size in a table rendering + * This property is used for synchronization between renderings. + */ + public static final String PROPERTY_ROW_SIZE = AbstractTableRendering.PROPERTY_ROW_SIZE; + + private boolean fActivated = false; + +// TODO: review use of MemorySegment, need to be careful to ensure flexible hierarchy + + private class ToggleAddressColumnAction extends Action { + + public ToggleAddressColumnAction() { + super(); + PlatformUI.getWorkbench().getHelpSystem().setHelp(this, IDebugUIConstants.PLUGIN_ID + + ".ShowAddressColumnAction_context"); //$NON-NLS-1$ + updateActionLabel(); + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.jface.action.IAction#run() + */ + public void run() { + fIsShowAddressColumn = !fIsShowAddressColumn; + if (!fIsShowAddressColumn) + { + fTableViewer.getTable().getColumn(0).setWidth(0); + } + else + { + fTableViewer.getTable().getColumn(0).pack(); + } + updateActionLabel(); + } + + /** + * + */ + private void updateActionLabel() { + if (fIsShowAddressColumn) { + setText(DebugUIMessages.ShowAddressColumnAction_0); + } else { + setText(DebugUIMessages.ShowAddressColumnAction_1); + } + } + } + + private class NextPageAction extends Action + { + private NextPageAction() + { + super(); + setText(DebugUIMessages.AbstractTableRendering_4); + PlatformUI.getWorkbench().getHelpSystem().setHelp(this, IDebugUIConstants.PLUGIN_ID + ".NextPageAction_context"); //$NON-NLS-1$ + } + + public void run() { + BigInteger address = fContentDescriptor.getLoadAddress(); + address = address.add(BigInteger.valueOf(getPageSizeInUnits())); + handlePageStartAddressChanged(address); + } + } + + private class PrevPageAction extends Action + { + private PrevPageAction() + { + super(); + setText(DebugUIMessages.AbstractTableRendering_6); + PlatformUI.getWorkbench().getHelpSystem().setHelp(this, IDebugUIConstants.PLUGIN_ID + ".PrevPageAction_context"); //$NON-NLS-1$ + } + + public void run() { + BigInteger address = fContentDescriptor.getLoadAddress(); + address = address.subtract(BigInteger.valueOf(getPageSizeInUnits())); + handlePageStartAddressChanged(address); + } + } + + private class RenderingGoToAddressAction extends GoToAddressAction + { + public RenderingGoToAddressAction(AbstractBaseTableRendering rendering) { + super(rendering); + } + + public void run() { + showGoToAddressComposite(); + } + } + + private class SwitchPageJob extends UIJob { + private Object fLock = new Object(); + private boolean fShowMessagePage = false; + private String fMessage = ""; //$NON-NLS-1$ + + private SwitchPageJob() { + super("SwitchPageJob");//$NON-NLS-1$ + setSystem(true); + } + + private void setShowMessagePage(boolean showMsg) { + synchronized(fLock) + { + fShowMessagePage = showMsg; + } + } + + private void setMessage(String message) { + synchronized(fLock) + { + fMessage = message; + } + } + + public IStatus runInUIThread(IProgressMonitor monitor) { + + if (fPageBook.isDisposed()) + return Status.OK_STATUS; + + String msgToShow = null; + boolean showMsgPage = false; + synchronized (fLock) { + msgToShow = fMessage; + showMsgPage = fShowMessagePage; + } + + if (showMsgPage) { + StyledText styleText = null; + fShowMessage = true; + + styleText = fTextViewer.getTextWidget(); + + if (styleText != null) + styleText.setText(msgToShow); + fPageBook.showPage(fTextViewer.getControl()); + } else { + fShowMessage = false; + fPageBook.showPage(fTableViewer.getControl().getParent()); + } + return Status.OK_STATUS; + } + } + + + private class SerialByObjectRule implements ISchedulingRule + { + private Object fObject = null; + + public SerialByObjectRule(Object lock) { + fObject = lock; + } + + /* (non-Javadoc) + * @see org.eclipse.core.runtime.jobs.ISchedulingRule#contains(org.eclipse.core.runtime.jobs.ISchedulingRule) + */ + public boolean contains(ISchedulingRule rule) { + return rule == this; + } + + /* (non-Javadoc) + * @see org.eclipse.core.runtime.jobs.ISchedulingRule#isConflicting(org.eclipse.core.runtime.jobs.ISchedulingRule) + */ + public boolean isConflicting(ISchedulingRule rule) { + if (rule instanceof SerialByObjectRule) { + SerialByObjectRule rRule = (SerialByObjectRule) rule; + return fObject == rRule.fObject; + } + return false; + } + } + + private PageBook fPageBook; + private AsyncTableRenderingViewer fTableViewer; + private TextViewer fTextViewer; + private Shell fToolTipShell; + private MemoryViewPresentationContext fPresentationContext; + private int fAddressableSize; + private TableRenderingContentDescriptor fContentDescriptor; + private int fBytePerLine; + private int fColumnSize; + private boolean fShowMessage = false; + private String fLabel; + private IWorkbenchAdapter fWorkbenchAdapter; + private int fPageSize; + private SashForm fSashForm; + private GoToAddressComposite fGoToAddressComposite; + + // actions + private GoToAddressAction fGoToAddressAction; + private PrintTableRenderingAction fPrintViewTabAction; + private CopyTableRenderingToClipboardAction fCopyToClipboardAction; + private FormatTableRenderingAction fFormatRenderingAction; + private ReformatAction fReformatAction; + private ToggleAddressColumnAction fToggleAddressColumnAction; + private ResetToBaseAddressAction fResetMemoryBlockAction; + private PropertyDialogAction fPropertiesDialogAction; + private NextPageAction fNextAction; + private PrevPageAction fPrevAction; + + private ArrayList fContext = new ArrayList(); + private AbstractHandler fGoToAddressHandler; + + private boolean fIsCreated = false; + private boolean fIsDisposed = false; + private boolean fIsShowAddressColumn = true; + + private SwitchPageJob fSwitchPageJob = new SwitchPageJob(); + private boolean fError = false; + + private PendingPropertyChanges fPendingSyncProperties; + + // list of menu listeners for popupMenuMgr. + private ArrayList fMenuListeners; + private MenuManager fMenuMgr; + + private ISchedulingRule serialByRenderingRule = new SerialByObjectRule(this); + + /** + * Identifier for an empty group preceding all context menu actions + * (value "popUpBegin"). + */ + public static final String EMPTY_MEMORY_GROUP = "popUpBegin"; //$NON-NLS-1$ + + /** + * Identifier for an empty group following navigation actions in the rendering + * (value navigationGroup). + */ + public static final String EMPTY_NAVIGATION_GROUP = "navigationGroup"; //$NON-NLS-1$ + + /** + * Identifier for an empty group following actions that are only applicable in + * non-auto loading mode + * (value nonAutoLoadGroup). + */ + public static final String EMPTY_NON_AUTO_LOAD_GROUP = "nonAutoLoadGroup"; //$NON-NLS-1$ + + /** + * Identifier for an empty group following properties actions + * (value propertyGroup). + */ + public static final String EMPTY_PROPERTY_GROUP = "propertyGroup"; //$NON-NLS-1$ + + private ISelectionChangedListener fViewerSelectionChangedListener = new ISelectionChangedListener() { + public void selectionChanged(SelectionChangedEvent event) { + updateSyncTopAddress(getTopVisibleAddress()); + updateSyncSelectedAddress(getSelectedAddress()); + } + }; + + private SelectionAdapter fScrollBarSelectionListener = new SelectionAdapter() { + public void widgetSelected(SelectionEvent e) { + updateSyncTopAddress(getTopVisibleAddress()); + } + }; + + private IModelChangedListener fModelChangedListener = new IModelChangedListener() { + public void modelChanged(IModelDelta delta, IModelProxy proxy) { + if (delta.getElement() == getMemoryBlock()) + { + showTable(); + updateRenderingLabel(isVisible()); + } + }}; + + private IVirtualContentListener fViewerListener = new IVirtualContentListener() { + + public void handledAtBufferStart() { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + if (isDynamicLoad()) + { + BigInteger address = getTopVisibleAddress(); + if (address != null && !isAtTopLimit()) + reloadTable(address); + } + } + } + + public void handleAtBufferEnd() { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + if (isDynamicLoad()) + { + BigInteger address = getTopVisibleAddress(); + if (address != null && !isAtBottomLimit()) + reloadTable(address); + } + } + } + + public int getThreshold() { + return 3; + }}; + + private IPresentationErrorListener fPresentationErrorListener = new IPresentationErrorListener() { + public void handlePresentationFailure(IStatusMonitor monitor, IStatus status) { + showMessage(status.getMessage()); + }}; + + + /** + * Constructs a new table rendering of the specified type. + * + * @param renderingId memory rendering type identifier + */ + public AbstractAsyncTableRendering(String renderingId) { + super(renderingId); + } + + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IResettableMemoryRendering#resetRendering() + */ + public void resetRendering() throws DebugException { + BigInteger baseAddress = fContentDescriptor.getContentBaseAddress(); + goToAddress(baseAddress); + fTableViewer.setSelection(baseAddress); + fTableViewer.setTopIndex(baseAddress); + + updateSyncTopAddress(baseAddress); + updateSyncSelectedAddress(baseAddress); + } + + public Control createControl(Composite parent) { + + fPageBook = new PageBook(parent, SWT.NONE); + createMessagePage(fPageBook); + createTableViewer(fPageBook); + addListeners(); + + return fPageBook; + } + + /** + * Create the error page of this rendering + * @param parent + */ + private void createMessagePage(Composite parent) + { + if (fTextViewer == null) + { + fTextViewer = new TextViewer(parent, SWT.WRAP); + fTextViewer.setDocument(new Document()); + StyledText styleText = fTextViewer.getTextWidget(); + styleText.setEditable(false); + styleText.setEnabled(false); + } + } + + /** + * @param parent + */ + private void createTableViewer(final Composite parent) + { + StringBuffer buffer = new StringBuffer(); + IMemoryRenderingType type = DebugUITools.getMemoryRenderingManager().getRenderingType(getRenderingId()); + buffer.append(type.getLabel()); + buffer.append(": "); //$NON-NLS-1$ + buffer.append(DebugUIMessages.AbstractAsyncTableRendering_2); + + Job job = new Job(buffer.toString()) { + + protected IStatus run(IProgressMonitor monitor) { + + // gather information from memory block + initAddressableSize(); + final BigInteger topVisibleAddress = getInitialTopVisibleAddress(); + BigInteger mbBaseAddress = null; + try { + mbBaseAddress = getMemoryBlockBaseAddress(); + } catch (DebugException e) { + fError = true; + showMessage(e.getMessage()); + } + + // if it takes too long to get the base address, and user has canceled + // remove this rendering. + if (monitor.isCanceled()) + { + getMemoryRenderingContainer().removeMemoryRendering(AbstractAsyncTableRendering.this); + return Status.CANCEL_STATUS; + } + + final BigInteger finalMbBaseAddress = mbBaseAddress; + final BigInteger initialSelectedAddress = getInitialSelectedAddress(); + + if (monitor.isCanceled()) + { + getMemoryRenderingContainer().removeMemoryRendering(AbstractAsyncTableRendering.this); + return Status.CANCEL_STATUS; + } + + createContentDescriptor(topVisibleAddress); + + // if it takes too long to get other information, and user has canceled + // remove this rendering. + if (monitor.isCanceled()) + { + getMemoryRenderingContainer().removeMemoryRendering(AbstractAsyncTableRendering.this); + return Status.CANCEL_STATUS; + } + + // batch update on UI thread + UIJob uiJob = new UIJob("Create Table Viewer UI Job"){ //$NON-NLS-1$ + public IStatus runInUIThread(IProgressMonitor progressMonitor) { + + if (fPageBook.isDisposed()) + return Status.OK_STATUS; + + fSashForm = new SashForm(parent, SWT.VERTICAL); + fTableViewer = new AsyncTableRenderingViewer(AbstractAsyncTableRendering.this, fSashForm, SWT.VIRTUAL | SWT.SINGLE | SWT.H_SCROLL | SWT.V_SCROLL | SWT.HIDE_SELECTION | SWT.BORDER); + + GridData data = new GridData(GridData.FILL_BOTH); + fTableViewer.getControl().setLayoutData(data); + + createGoToAddressComposite(fSashForm); + hideGotoAddressComposite(); + + IMemoryRenderingSite site = getMemoryRenderingContainer().getMemoryRenderingSite(); + IMemoryRenderingContainer container = getMemoryRenderingContainer(); + fPresentationContext = new MemoryViewPresentationContext(site, container, AbstractAsyncTableRendering.this); + fTableViewer.setContext(fPresentationContext); + + // must call this after the context is created as the information is stored in the context + getDynamicLoadFromPreference(); + getPageSizeFromPreference(); + + int numberOfLines = getNumLinesToLoad(); + fContentDescriptor.setNumLines(numberOfLines); + + BigInteger baseAddress = finalMbBaseAddress; + if (baseAddress == null) + baseAddress = BigInteger.ZERO; + + + if (!(getMemoryBlock() instanceof IMemoryBlockExtension) || !isDynamicLoad()) + { + // If not extended memory block, do not create any buffer + // no scrolling + fContentDescriptor.setPreBuffer(0); + fContentDescriptor.setPostBuffer(0); + } + + setupInitialFormat(); + fTableViewer.setCellModifier(createCellModifier()); + fTableViewer.getTable().setHeaderVisible(true); + fTableViewer.getTable().setLinesVisible(true); + fTableViewer.addPresentationErrorListener(fPresentationErrorListener); + fTableViewer.setInput(getMemoryBlock()); + fTableViewer.resizeColumnsToPreferredSize(); + fTableViewer.setTopIndex(topVisibleAddress); + + fTableViewer.setSelection(initialSelectedAddress); + + // SET UP FONT + // set to a non-proportional font + fTableViewer.getTable().setFont(JFaceResources.getFont(IInternalDebugUIConstants.FONT_NAME)); + + if (!fError) + showTable(); + + fTableViewer.addVirtualContentListener(fViewerListener); + + // create context menu + // create pop up menu for the rendering + createActions(); + IMenuListener menuListener = new IMenuListener() { + public void menuAboutToShow(IMenuManager mgr) { + fillContextMenu(mgr); + } + }; + createPopupMenu(fTableViewer.getControl(), menuListener); + createPopupMenu(fTableViewer.getCursor(), menuListener); + + fTableViewer.addSelectionChangedListener(fViewerSelectionChangedListener); + fTableViewer.getTable().getVerticalBar().addSelectionListener(fScrollBarSelectionListener); + + createToolTip(); + + // now the rendering is successfully created + fIsCreated = true; + + return Status.OK_STATUS; + }}; + uiJob.setSystem(true); + uiJob.schedule(); + + return Status.OK_STATUS; + + }}; + + job.schedule(); + } + + /** + * Create popup menu for this rendering + * @param control - control to create the popup menu for + * @param menuListener - listener to notify when popup menu is about to show + */ + private void createPopupMenu(Control control, IMenuListener menuListener) + { + IMemoryRenderingContainer container = getMemoryRenderingContainer(); + if (fMenuMgr == null) + { + fMenuMgr = new MenuManager("#PopupMenu"); //$NON-NLS-1$ + fMenuMgr.setRemoveAllWhenShown(true); + IMemoryRenderingSite site = container.getMemoryRenderingSite(); + String menuId = container.getId(); + + ISelectionProvider selProvider = site.getSite().getSelectionProvider(); + + addMenuListener(menuListener); + + site.getSite().registerContextMenu(menuId, fMenuMgr, selProvider); + } + + addMenuListener(menuListener); + + Menu popupMenu = fMenuMgr.createContextMenu(control); + control.setMenu(popupMenu); + } + + + private void addMenuListener(IMenuListener menuListener) { + if (fMenuListeners == null) + fMenuListeners = new ArrayList(); + + if (!fMenuListeners.contains(menuListener)) + { + fMenuMgr.addMenuListener(menuListener); + fMenuListeners.add(menuListener); + } + } + + private BigInteger getInitialSelectedAddress() { + // figure out selected address + BigInteger selectedAddress = (BigInteger) getSynchronizedProperty(AbstractAsyncTableRendering.PROPERTY_SELECTED_ADDRESS); + if (selectedAddress == null) + { + if (getMemoryBlock() instanceof IMemoryBlockExtension) { + try { + selectedAddress = ((IMemoryBlockExtension) getMemoryBlock()).getBigBaseAddress(); + } catch (DebugException e) { + selectedAddress = BigInteger.ZERO; + } + + if (selectedAddress == null) { + selectedAddress =BigInteger.ZERO; + } + + } else { + long address = getMemoryBlock().getStartAddress(); + selectedAddress = BigInteger.valueOf(address); + } + } + return selectedAddress; + } + + private void addListeners() + { + DebugUIPlugin.getDefault().getPreferenceStore().addPropertyChangeListener(this); + addRenderingToSyncService(); + JFaceResources.getFontRegistry().addListener(this); + } + + private void removeListeners() + { + DebugUIPlugin.getDefault().getPreferenceStore().removePropertyChangeListener(this); + removeRenderingFromSyncService(); + JFaceResources.getFontRegistry().removeListener(this); + + Iterator iter = fMenuListeners.iterator(); + while (iter.hasNext()) + { + fMenuMgr.removeMenuListener((IMenuListener)iter.next()); + } + + fMenuListeners.clear(); + } + + private void addRenderingToSyncService() + { + IMemoryRenderingSynchronizationService syncService = getMemoryRenderingContainer().getMemoryRenderingSite().getSynchronizationService(); + + if (syncService == null) + return; + + syncService.addPropertyChangeListener(this, null); + } + + private void removeRenderingFromSyncService() + { + IMemoryRenderingSynchronizationService syncService = getMemoryRenderingContainer().getMemoryRenderingSite().getSynchronizationService(); + + if (syncService == null) + return; + + syncService.removePropertyChangeListener(this); + } + + private void initAddressableSize() + { + // set up addressable size and figure out number of bytes required per line + fAddressableSize = -1; + try { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + fAddressableSize = ((IMemoryBlockExtension)getMemoryBlock()).getAddressableSize(); + else + fAddressableSize = 1; + } catch (DebugException e1) { + DebugUIPlugin.log(e1); + // log error and default to 1 + fAddressableSize = 1; + return; + + } + if (fAddressableSize < 1) + { + DebugUIPlugin.logErrorMessage("Invalid addressable size"); //$NON-NLS-1$ + fAddressableSize = 1; + } + } + + private BigInteger getInitialTopVisibleAddress() { + BigInteger topVisibleAddress = (BigInteger) getSynchronizedProperty(AbstractAsyncTableRendering.PROPERTY_TOP_ADDRESS); + if (topVisibleAddress == null) + { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + try { + topVisibleAddress = ((IMemoryBlockExtension)getMemoryBlock()).getBigBaseAddress(); + } catch (DebugException e1) { + topVisibleAddress = new BigInteger("0"); //$NON-NLS-1$ + } + } + else + { + topVisibleAddress = BigInteger.valueOf(getMemoryBlock().getStartAddress()); + } + } + return topVisibleAddress; + } + + private void setupInitialFormat() { + + boolean validated = validateInitialFormat(); + + if (!validated) + { + // pop up dialog to ask user for default values + StringBuffer msgBuffer = new StringBuffer(DebugUIMessages.AbstractTableRendering_20); + msgBuffer.append(" "); //$NON-NLS-1$ + msgBuffer.append(this.getLabel()); + msgBuffer.append("\n\n"); //$NON-NLS-1$ + msgBuffer.append(DebugUIMessages.AbstractTableRendering_16); + msgBuffer.append("\n"); //$NON-NLS-1$ + msgBuffer.append(DebugUIMessages.AbstractTableRendering_18); + msgBuffer.append("\n\n"); //$NON-NLS-1$ + + int bytePerLine = fBytePerLine; + int columnSize = fColumnSize; + + // initialize this value to populate the dialog properly + fBytePerLine = getDefaultRowSize() / getAddressableSize(); + fColumnSize = getDefaultColumnSize() / getAddressableSize(); + + FormatTableRenderingDialog dialog = new FormatTableRenderingDialog(this, DebugUIPlugin.getShell()); + dialog.openError(msgBuffer.toString()); + + // restore to original value before formatting + fBytePerLine = bytePerLine; + fColumnSize = columnSize; + + bytePerLine = dialog.getRowSize() * getAddressableSize(); + columnSize = dialog.getColumnSize() * getAddressableSize(); + + format(bytePerLine, columnSize); + } + else + { + // Row size is stored as number of addressable units in preference store + int bytePerLine = getDefaultRowSize(); + // column size is now stored as number of addressable units + int columnSize = getDefaultColumnSize(); + + // format memory block with specified "bytesPerLine" and "columnSize" + boolean ok = format(bytePerLine, columnSize); + + if (!ok) + { + // this is to ensure that the rest of the rendering can be created + // and we can recover from a format error + format(bytePerLine, bytePerLine); + } + } + } + + private boolean validateInitialFormat() + { + int rowSize = getDefaultRowSize(); + int columnSize = getDefaultColumnSize(); + + if (rowSize < columnSize || rowSize % columnSize != 0 || rowSize == 0 || columnSize == 0) + { + return false; + } + return true; + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#getControl() + */ + public Control getControl() { + return fPageBook.getParent(); + } + + /* (non-Javadoc) + * @see org.eclipse.jface.util.IPropertyChangeListener#propertyChange(org.eclipse.jface.util.PropertyChangeEvent) + */ + public void propertyChange(PropertyChangeEvent event) { + + if (!fIsCreated) + return; + + // if memory view table font has changed + if (event.getProperty().equals(IInternalDebugUIConstants.FONT_NAME)) + { + if (!fIsDisposed) + { + Font memoryViewFont = JFaceResources.getFont(IInternalDebugUIConstants.FONT_NAME); + setFont(memoryViewFont); + } + return; + } + + Object evtSrc = event.getSource(); + + if (event.getProperty().equals(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE)) { + // always update page size, only refresh if the table is visible + getPageSizeFromPreference(); + } + + // do not handle event if the rendering is displaying an error + // or if it's not visible + if (isDisplayingError() || !isVisible()) + { + handlePropertiesChangeWhenHidden(event); + return; + } + + + if (event.getProperty().equals(IDebugUIConstants.PREF_PADDED_STR) || + event.getProperty().equals(IDebugUIConstants.PREF_CHANGED_DEBUG_ELEMENT_COLOR) || + event.getProperty().equals(IDebugUIConstants.PREF_MEMORY_HISTORY_KNOWN_COLOR) || + event.getProperty().equals(IDebugUIConstants.PREF_MEMORY_HISTORY_UNKNOWN_COLOR)) + { + if (!fIsDisposed) + { + fTableViewer.refresh(false); + } + return; + } + + if (event.getProperty().equals(IDebugPreferenceConstants.PREF_DYNAMIC_LOAD_MEM)) { + handleDyanicLoadChanged(); + return; + } + + if (event.getProperty().equals(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE)) { + if (!isDynamicLoad()) + { + int pageSize = DebugUIPlugin.getDefault().getPreferenceStore().getInt(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE); + handlePageSizeChanged(pageSize); + } + return; + } + + if (evtSrc == this) + return; + + if (evtSrc instanceof IMemoryRendering) + { + IMemoryRendering rendering = (IMemoryRendering)evtSrc; + IMemoryBlock memoryBlock = rendering.getMemoryBlock(); + + // do not handle event from renderings displaying other memory blocks + if (memoryBlock != getMemoryBlock()) + return; + } + + String propertyName = event.getProperty(); + Object value = event.getNewValue(); + + if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_SELECTED_ADDRESS) && value instanceof BigInteger) + { + selectedAddressChanged((BigInteger)value); + } + else if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_COL_SIZE) && value instanceof Integer) + { + columnSizeChanged(((Integer)value).intValue()); + } + else if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_ROW_SIZE) && value instanceof Integer) + { + rowSizeChanged(((Integer)value).intValue()); + } + else if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_TOP_ADDRESS) && value instanceof BigInteger) + { + topVisibleAddressChanged((BigInteger)value); + } + else if (propertyName.equals(IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS) && value instanceof BigInteger) + { + handlePageStartAddressChanged((BigInteger)value); + } + } + + /** + * + */ + private void handlePageSizeChanged(int pageSize) { + fPageSize = pageSize; + // only refresh if in non-auto-load mode + fContentDescriptor.setNumLines(pageSize); + refresh(); + } + + private void handlePropertiesChangeWhenHidden(PropertyChangeEvent event) + { + if (fPendingSyncProperties == null) + return; + + String propertyName = event.getProperty(); + Object value = event.getNewValue(); + + if (event.getSource() instanceof IMemoryRendering) + { + IMemoryRendering rendering = (IMemoryRendering)event.getSource(); + if (rendering == this || rendering.getMemoryBlock() != getMemoryBlock()) + { + return; + } + } + + if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_COL_SIZE) && value instanceof Integer) + { + fPendingSyncProperties.setColumnSize(((Integer)value).intValue()); + } + else if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_ROW_SIZE) && value instanceof Integer) + { + fPendingSyncProperties.setRowSize(((Integer)value).intValue()); + } + + else if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_SELECTED_ADDRESS) && value instanceof BigInteger) + { + fPendingSyncProperties.setSelectedAddress((BigInteger)value); + } + else if (propertyName.equals(AbstractAsyncTableRendering.PROPERTY_TOP_ADDRESS) && value instanceof BigInteger) + { + fPendingSyncProperties.setTopVisibleAddress((BigInteger)value); + } + else if (propertyName.equals(IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS) && value instanceof BigInteger) + { + fPendingSyncProperties.setPageStartAddress((BigInteger)value); + } + else if (event.getProperty().equals(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE)) { + int pageSize = DebugUIPlugin.getDefault().getPreferenceStore().getInt(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE); + fPendingSyncProperties.setPageSize(pageSize); + } + } + + private void topVisibleAddressChanged(final BigInteger address) + { + final Runnable runnable = new Runnable() { + public void run() { + if (fTableViewer.getTable().isDisposed()) + return; + + doTopVisibleAddressChanged(address); + }}; + runOnUIThread(runnable); + } + + /** + * @param address + */ + private void doTopVisibleAddressChanged(final BigInteger address) { + if (fIsDisposed) + return; + + if (!isDynamicLoad()) + { + fTableViewer.setTopIndex(address); + fTableViewer.topIndexChanged(); + return; + } + + if (!isAtTopBuffer(address) && !isAtBottomBuffer(address)) + { + fTableViewer.setTopIndex(address); + fTableViewer.topIndexChanged(); + } + else + { + reloadTable(address); + } + } + + private boolean isAtBottomBuffer(BigInteger address) + { + int idx = fTableViewer.indexOf(address); + if (idx < 0) + return true; + + int bottomIdx = idx + getNumberOfVisibleLines(); + int elementsCnt = fTableViewer.getVirtualContentModel().getElements().length; + int numLinesLeft = elementsCnt - bottomIdx; + + if (numLinesLeft <= 3) + return true; + + return false; + } + + private boolean isAtTopBuffer(BigInteger address) + { + int topIdx = fTableViewer.indexOf(address); + if (topIdx <= 3) + return true; + + return false; + } + + private void runOnUIThread(final Runnable runnable) + { + if (Display.getCurrent() != null) + { + runnable.run(); + } + else + { + UIJob job = new UIJob("Async Table Rendering UI Job"){ //$NON-NLS-1$ + + public IStatus runInUIThread(IProgressMonitor monitor) { + runnable.run(); + return Status.OK_STATUS; + }}; + job.setSystem(true); + job.schedule(); + } + } + + private void selectedAddressChanged(final BigInteger address) + { + Runnable runnable = new Runnable() { + + public void run() { + + if (fTableViewer.getTable().isDisposed()) + return; + + // call this to make the table viewer to reload when needed + int i = fTableViewer.indexOf(address); + if (i < 0) + { + topVisibleAddressChanged(address); + } + fTableViewer.setSelection(address); + } + }; + + runOnUIThread(runnable); + } + + private void setFont(Font font) + { + // set font + fTableViewer.getTable().setFont(font); + fTableViewer.getCursor().setFont(font); + } + + private int getDefaultColumnSize() { + + // default to global preference store + IPreferenceStore prefStore = DebugUITools.getPreferenceStore(); + int columnSize = prefStore.getInt(IDebugPreferenceConstants.PREF_COLUMN_SIZE); + // actual column size is number of addressable units * size of the addressable unit + columnSize = columnSize * getAddressableSize(); + + // check synchronized col size + Integer colSize = (Integer)getSynchronizedProperty(AbstractAsyncTableRendering.PROPERTY_COL_SIZE); + if (colSize != null) + { + // column size is stored as actual number of bytes in synchronizer + int syncColSize = colSize.intValue(); + if (syncColSize > 0) + { + columnSize = syncColSize; + } + } + else + { + IPersistableDebugElement elmt = (IPersistableDebugElement)getMemoryBlock().getAdapter(IPersistableDebugElement.class); + int defaultColSize = -1; + + if (elmt != null) + { + if (elmt.supportsProperty(this, IDebugPreferenceConstants.PREF_COL_SIZE_BY_MODEL)) + defaultColSize = getDefaultFromPersistableElement(IDebugPreferenceConstants.PREF_COL_SIZE_BY_MODEL); + } + + if (defaultColSize <= 0) + { + // if not provided, get default by model + defaultColSize = getDefaultColumnSizeByModel(getMemoryBlock().getModelIdentifier()); + } + + if (defaultColSize > 0) + columnSize = defaultColSize * getAddressableSize(); + } + return columnSize; + } + + private int getDefaultRowSize() { + + int rowSize = DebugUITools.getPreferenceStore().getInt(IDebugPreferenceConstants.PREF_ROW_SIZE); + int bytePerLine = rowSize * getAddressableSize(); + + // check synchronized row size + Integer size = (Integer)getSynchronizedProperty(AbstractAsyncTableRendering.PROPERTY_ROW_SIZE); + if (size != null) + { + // row size is stored as actual number of bytes in synchronizer + int syncRowSize = size.intValue(); + if (syncRowSize > 0) + { + bytePerLine = syncRowSize; + } + } + else + { + int defaultRowSize = -1; + IPersistableDebugElement elmt = (IPersistableDebugElement)getMemoryBlock().getAdapter(IPersistableDebugElement.class); + if (elmt != null) + { + if (elmt.supportsProperty(this, IDebugPreferenceConstants.PREF_ROW_SIZE_BY_MODEL)) + { + defaultRowSize = getDefaultFromPersistableElement(IDebugPreferenceConstants.PREF_ROW_SIZE_BY_MODEL); + return defaultRowSize * getAddressableSize(); + } + } + + if (defaultRowSize <= 0) + // no synchronized property, ask preference store by id + defaultRowSize = getDefaultRowSizeByModel(getMemoryBlock().getModelIdentifier()); + + if (defaultRowSize > 0) + bytePerLine = defaultRowSize * getAddressableSize(); + } + return bytePerLine; + } + + /** + * Returns the addressable size of this rendering's memory block in bytes. + * + * @return the addressable size of this rendering's memory block in bytes + */ + public int getAddressableSize() { + return fAddressableSize; + } + + private Object getSynchronizedProperty(String propertyId) + { + IMemoryRenderingSynchronizationService syncService = getMemoryRenderingContainer().getMemoryRenderingSite().getSynchronizationService(); + + if (syncService == null) + return null; + + return syncService.getProperty(getMemoryBlock(), propertyId); + } + + private int getDefaultFromPersistableElement(String propertyId) { + int defaultValue = -1; + IPersistableDebugElement elmt = (IPersistableDebugElement)getMemoryBlock().getAdapter(IPersistableDebugElement.class); + if (elmt != null) + { + try { + Object valueMB = elmt.getProperty(this, propertyId); + if (valueMB != null && !(valueMB instanceof Integer)) + { + IStatus status = DebugUIPlugin.newErrorStatus("Model returned invalid type on " + propertyId, null); //$NON-NLS-1$ + DebugUIPlugin.log(status); + } + + if (valueMB != null) + { + Integer value = (Integer)valueMB; + defaultValue = value.intValue(); + } + } catch (CoreException e) { + DebugUIPlugin.log(e); + } + } + return defaultValue; + } + + /** + * @param modelId + * @return default number of addressable units per line for the model + */ + private int getDefaultRowSizeByModel(String modelId) + { + int row = DebugUITools.getPreferenceStore().getInt(getRowPrefId(modelId)); + if (row == 0) + { + DebugUITools.getPreferenceStore().setValue(getRowPrefId(modelId), IDebugPreferenceConstants.PREF_ROW_SIZE_DEFAULT); + } + + row = DebugUITools.getPreferenceStore().getInt(getRowPrefId(modelId)); + return row; + + } + + /** + * @param modelId + * @return default number of addressable units per column for the model + */ + private int getDefaultColumnSizeByModel(String modelId) + { + int col = DebugUITools.getPreferenceStore().getInt(getColumnPrefId(modelId)); + if (col == 0) + { + DebugUITools.getPreferenceStore().setValue(getColumnPrefId(modelId), IDebugPreferenceConstants.PREF_COLUMN_SIZE_DEFAULT); + } + + col = DebugUITools.getPreferenceStore().getInt(getColumnPrefId(modelId)); + return col; + } + + + private String getRowPrefId(String modelId) { + String rowPrefId = IDebugPreferenceConstants.PREF_ROW_SIZE + ":" + modelId; //$NON-NLS-1$ + return rowPrefId; + } + + private String getColumnPrefId(String modelId) { + String colPrefId = IDebugPreferenceConstants.PREF_COLUMN_SIZE + ":" + modelId; //$NON-NLS-1$ + return colPrefId; + } + + /** + * Format view tab based on the bytes per line and column. + * + * @param bytesPerLine - number of bytes per line, possible values: (1 / 2 / 4 / 8 / 16) * addressableSize + * @param columnSize - number of bytes per column, possible values: (1 / 2 / 4 / 8 / 16) * addressableSize + * @return true if format is successful, false, otherwise + */ + public boolean format(int bytesPerLine, int columnSize) + { + + // bytes per cell must be divisible to bytesPerLine + if (bytesPerLine % columnSize != 0) + { + return false; + } + + if (bytesPerLine < columnSize) + { + return false; + } + + // do not format if the view tab is already in that format + if(fBytePerLine == bytesPerLine && fColumnSize == columnSize){ + return false; + } + + fBytePerLine = bytesPerLine; + fColumnSize = columnSize; + formatViewer(); + + updateSyncRowSize(); + updateSyncColSize(); + + return true; + } + + + /** + * Returns the number of addressable units per row. + * + * @return number of addressable units per row + */ + public int getAddressableUnitPerLine() { + return fBytePerLine / getAddressableSize(); + } + + /** + * Returns the number of addressable units per column. + * + * @return number of addressable units per column + */ + public int getAddressableUnitPerColumn() { + return fColumnSize / getAddressableSize(); + } + + + /** + * This method estimates the number of visible lines in the rendering + * table. + * @return estimated number of visible lines in the table + */ + private int getNumberOfVisibleLines() + { + if(fTableViewer == null) + return -1; + + Table table = fTableViewer.getTable(); + int height = fTableViewer.getTable().getSize().y; + + // when table is not yet created, height is zero + if (height == 0) + { + // make use of the table viewer to estimate table size + height = fTableViewer.getTable().getParent().getSize().y; + } + + // height of border + int border = fTableViewer.getTable().getHeaderHeight(); + + // height of scroll bar + int scroll = fTableViewer.getTable().getHorizontalBar().getSize().y; + + // height of table is table's area minus border and scroll bar height + height = height-border-scroll; + + // calculate number of visible lines + int lineHeight = getMinTableItemHeight(table); + + int numberOfLines = height/lineHeight; + + if (numberOfLines <= 0) + return 20; + + return numberOfLines; + } + + private int getMinTableItemHeight(Table table){ + + // Hack to get around Linux GTK problem. + // On Linux GTK, table items have variable item height as + // carriage returns are actually shown in a cell. Some rows will be + // taller than others. When calculating number of visible lines, we + // need to find the smallest table item height. Otherwise, the rendering + // underestimates the number of visible lines. As a result the rendering + // will not be able to get more memory as needed. + if (MemoryViewUtil.isLinuxGTK()) + { + // check each of the items and find the minimum + TableItem[] items = table.getItems(); + int minHeight = table.getItemHeight(); + for (int i=0; i topIndex) + { + MemorySegment topItem = (MemorySegment)table.getItem(topIndex).getData(); + if (topItem != null) + { + return topItem.getAddress(); + } + } + return null; + } + + private synchronized void reloadTable(final BigInteger topAddress) { + + if (AsyncVirtualContentTableViewer.DEBUG_DYNAMIC_LOADING) + System.out.println(this + " reload at: " + topAddress.toString(16)); //$NON-NLS-1$ + + fContentDescriptor.setLoadAddress(topAddress); + fContentDescriptor.setNumLines(getNumLinesToLoad()); + fTableViewer.setTopIndex(topAddress); + fTableViewer.refresh(); + + } + + private boolean isAtTopLimit() + { + BigInteger startAddress = fContentDescriptor.getStartAddress(); + startAddress = MemoryViewUtil.alignToBoundary(startAddress, getAddressableUnitPerLine()); + AbstractVirtualContentTableModel model = fTableViewer.getVirtualContentModel(); + + if (model != null) + { + Object key = model.getKey(0); + if (key instanceof BigInteger) + { + BigInteger startBufferAddress = (BigInteger)key; + startBufferAddress = MemoryViewUtil.alignToBoundary(startBufferAddress, getAddressableUnitPerLine()); + + if (startAddress.compareTo(startBufferAddress) == 0) + return true; + } + } + return false; + } + + private boolean isAtBottomLimit() + { + BigInteger endAddress = fContentDescriptor.getEndAddress(); + endAddress = MemoryViewUtil.alignToBoundary(endAddress, getAddressableUnitPerLine()); + + AbstractVirtualContentTableModel model = fTableViewer.getVirtualContentModel(); + if (model != null) + { + int numElements = model.getElements().length; + Object key = model.getKey(numElements-1); + if (key instanceof BigInteger) + { + BigInteger endBufferAddress = (BigInteger)key; + endBufferAddress = MemoryViewUtil.alignToBoundary(endBufferAddress, getAddressableUnitPerLine()); + + if (endAddress.compareTo(endBufferAddress) == 0) + return true; + } + } + + return false; + } + + private void formatViewer() { + + fTableViewer.disposeColumns(); + fTableViewer.disposeCellEditors(); + doFormatTable(); + fTableViewer.setColumnHeaders(getColumnProperties()); + fTableViewer.showColumnHeader(true); + fTableViewer.setCellEditors(createCellEditors(fTableViewer.getTable())); + + fTableViewer.formatViewer(); + + // This resize needs to happen after the viewer has finished + // getting the labels. + // This fix is a hack to delay the resize until the viewer has a chance to get + // the setData event from the UI thread. Otherwise, the columns will be + // squeezed together. + UIJob job = new UIJob("resize to fit"){ //$NON-NLS-1$ + public IStatus runInUIThread(IProgressMonitor monitor) { + resizeColumnsToPreferredSize(); + return Status.OK_STATUS; + }}; + + job.setSystem(true); + job.schedule(); + } + + private void doFormatTable() { + int bytesPerLine = getBytesPerLine(); + int columnSize = getBytesPerColumn(); + int numColumns = bytesPerLine/columnSize; + + Table table = fTableViewer.getTable(); + TableColumn column0 = new TableColumn(table,SWT.LEFT,0); + column0.setText(DebugUIMessages.AbstractTableRendering_2); + + // create new byte columns + TableColumn [] byteColumns = new TableColumn[numColumns]; + for (int i=0;i"; //$NON-NLS-1$ //$NON-NLS-2$ + + return decorateLabel(label); + } + + /* Returns the label of this rendering. + * + * @return label of this rendering + */ + public String getLabel() { + + if (fLabel == null) + { + fLabel = DebugUIMessages.AbstractAsyncTableRendering_1; + updateRenderingLabel(isVisible()); + } + + return fLabel; + } + + /* (non-Javadoc) + * @see org.eclipse.core.runtime.PlatformObject#getAdapter(java.lang.Class) + */ + public Object getAdapter(Class adapter) { + + if (adapter == IColorProvider.class) + return getColorProviderAdapter(); + + if (adapter == ILabelProvider.class) + return getLabelProviderAdapter(); + + if (adapter == IFontProvider.class) + return getFontProviderAdapter(); + + if (adapter == IModelChangedListener.class) + { + return fModelChangedListener; + } + + if (adapter == IWorkbenchAdapter.class) + { + // needed workbench adapter to fill the title of property page + if (fWorkbenchAdapter == null) { + fWorkbenchAdapter = new IWorkbenchAdapter() { + public Object[] getChildren(Object o) { + return new Object[0]; + } + + public ImageDescriptor getImageDescriptor(Object object) { + return null; + } + + public String getLabel(Object o) { + return AbstractAsyncTableRendering.this.getLabel(); + } + + public Object getParent(Object o) { + return null; + } + }; + } + return fWorkbenchAdapter; + } + + if (adapter == TableRenderingContentDescriptor.class) + return getContentDescriptor(); + + return super.getAdapter(adapter); + } + + /** + * Returns the number of characters a byte will convert to + * or -1 if unknown. + * + * @return the number of characters a byte will convert to + * or -1 if unknown + */ + public int getNumCharsPerByte() + { + return -1; + } + + /** + * Create actions for this rendering + */ + protected void createActions() { + + fCopyToClipboardAction = new AsyncCopyTableRenderingAction(this, fTableViewer); + fGoToAddressAction = new RenderingGoToAddressAction(this); + fResetMemoryBlockAction = new ResetToBaseAddressAction(this); + + fPrintViewTabAction = new AsyncPrintTableRenderingAction(this, fTableViewer); + + fFormatRenderingAction = new FormatTableRenderingAction(this); + fReformatAction = new ReformatAction(this); + fToggleAddressColumnAction = new ToggleAddressColumnAction(); + + IMemoryRenderingSite site = getMemoryRenderingContainer().getMemoryRenderingSite(); + if (site.getSite().getSelectionProvider() != null) + { + fPropertiesDialogAction = new PropertyDialogAction(site.getSite(),site.getSite().getSelectionProvider()); + } + + fNextAction = new NextPageAction(); + fPrevAction = new PrevPageAction(); + } + + /** + * Returns the currently selected address in this rendering. + * + * @return the currently selected address in this rendering + */ + public BigInteger getSelectedAddress() { + Object key = fTableViewer.getSelectionKey(); + + if (key != null && key instanceof BigInteger) + return (BigInteger)key; + + return null; + } + + /** + * Returns the currently selected content in this rendering as MemoryByte. + * + * @return the currently selected content in array of MemoryByte. + * Returns an empty array if the selected address is out of buffered range. + */ + public MemoryByte[] getSelectedAsBytes() + { + if (getSelectedAddress() == null) + return new MemoryByte[0]; + + Object key = fTableViewer.getSelectionKey(); + AbstractVirtualContentTableModel model = fTableViewer.getVirtualContentModel(); + + if (model != null) + { + model = (AbstractVirtualContentTableModel)fTableViewer.getModel(); + int row = model.indexOfKey(key); + Object element = model.getElement(row); + int col = model.columnOf(element, key); + + // check precondition + if (col <= 0 || col > getBytesPerLine()/getBytesPerColumn()) + { + return new MemoryByte[0]; + } + + if (!(element instanceof MemorySegment)) + return new MemoryByte[0]; + + MemorySegment line = (MemorySegment)element; + int offset = (col-1)*(getAddressableUnitPerColumn()*getAddressableSize()); + + // make a copy of the bytes to ensure that data cannot be changed + // by caller + MemoryByte[] bytes = line.getBytes(offset, getAddressableUnitPerColumn()*getAddressableSize()); + MemoryByte[] retBytes = new MemoryByte[bytes.length]; + + System.arraycopy(bytes, 0, retBytes, 0, bytes.length); + return retBytes; + } + return new MemoryByte[0]; + } + + /** + * Returns the currently selected content in this rendering as a String. + * + * @return the currently selected content in this rendering + */ + public String getSelectedAsString() { + + if (getSelectedAddress() == null) + return ""; //$NON-NLS-1$ + + MemoryByte[] bytes = getSelectedAsBytes(); + if (bytes.length > 0) + { + return getString(this.getRenderingId(), getSelectedAddress(), bytes); + } + else + return ""; //$NON-NLS-1$ + + } + + /** + * Moves the cursor to the specified address. + * Will load more memory if the address is not currently visible. + * + * @param address address to position cursor at + * @throws DebugException if an exception occurs + */ + public void goToAddress(BigInteger address) throws DebugException { + + if (fTableViewer.getVirtualContentModel() == null) + return; + + int i = fTableViewer.getVirtualContentModel().indexOfKey(address); + + if (i >= 0) + { + // address is within range, set cursor and reveal + fTableViewer.setSelection(address); + updateSyncTopAddress(getTopVisibleAddress()); + updateSyncSelectedAddress(address); + } + else + { + // if not extended memory block + // do not allow user to go to an address that's out of range + if (!(getMemoryBlock() instanceof IMemoryBlockExtension)) + { + Status stat = new Status( + IStatus.ERROR, DebugUIPlugin.getUniqueIdentifier(), + DebugException.NOT_SUPPORTED, DebugUIMessages.AbstractTableRendering_11, null + ); + DebugException e = new DebugException(stat); + throw e; + } + + BigInteger startAdd = fContentDescriptor.getStartAddress(); + BigInteger endAdd = fContentDescriptor.getEndAddress(); + + if (address.compareTo(startAdd) < 0 || + address.compareTo(endAdd) > 0) + { + Status stat = new Status( + IStatus.ERROR, DebugUIPlugin.getUniqueIdentifier(), + DebugException.NOT_SUPPORTED, DebugUIMessages.AbstractTableRendering_11, null + ); + DebugException e = new DebugException(stat); + throw e; + } + + // load at the address + fTableViewer.setSelection(address); + reloadTable(address); + + updateSyncSelectedAddress(address); + + if (!isDynamicLoad()) + { + updateSyncPageStartAddress(address); + } + + updateSyncTopAddress(address); + } + } + + /** + * Refresh the table viewer with the current top visible address. + * Update labels in the memory rendering. + */ + public void refresh() { + fTableViewer.refresh(); + } + + + /** + * Resize column to the preferred size. + */ + public void resizeColumnsToPreferredSize() { + fTableViewer.resizeColumnsToPreferredSize(); + if (!fIsShowAddressColumn) + { + final TableColumn column = fTableViewer.getTable().getColumn(0); + column.addControlListener(new ControlListener() { + + public void controlMoved(ControlEvent e) { + } + + public void controlResized(ControlEvent e) { + column.removeControlListener(this); + column.setWidth(0); + }}); + } + } + + /** + * Updates labels of this rendering. + */ + public void updateLabels() + { + UIJob job = new UIJob("updateLabels"){ //$NON-NLS-1$ + + public IStatus runInUIThread(IProgressMonitor monitor) { + + // do not handle if the rendering is already disposed + if (fPageBook.isDisposed()) + return Status.OK_STATUS; + + // update tab labels + updateRenderingLabel(true); + + if (fTableViewer != null) + { + // update column labels + setColumnHeadings(); + + // rebuild cache and force labels to be refreshed + fTableViewer.formatViewer(); + } + return Status.OK_STATUS; + }}; + job.setSystem(true); + job.schedule(); + } + + /** + * Fills the context menu for this rendering + * + * @param menu menu to fill + */ + protected void fillContextMenu(IMenuManager menu) { + + menu.add(new Separator(EMPTY_MEMORY_GROUP)); + menu.add(new Separator()); + menu.add(fResetMemoryBlockAction); + menu.add(fGoToAddressAction); + menu.add(new Separator(EMPTY_NAVIGATION_GROUP)); + + menu.add(new Separator()); + menu.add(fFormatRenderingAction); + + if (!isDynamicLoad() && getMemoryBlock() instanceof IMemoryBlockExtension) + { + menu.add(new Separator()); + menu.add(fPrevAction); + menu.add(fNextAction); + menu.add(new Separator(EMPTY_NON_AUTO_LOAD_GROUP)); + } + + menu.add(new Separator()); + menu.add(fReformatAction); + menu.add(fToggleAddressColumnAction); + menu.add(new Separator()); + menu.add(fCopyToClipboardAction); + menu.add(fPrintViewTabAction); + if (fPropertiesDialogAction != null) + { + menu.add(new Separator()); + menu.add(fPropertiesDialogAction); + menu.add(new Separator(EMPTY_PROPERTY_GROUP)); + } + + menu.add(new Separator(IWorkbenchActionConstants.MB_ADDITIONS)); + } + + private int getPageSizeInUnits() + { + return fPageSize * getAddressableUnitPerLine(); + } + + private void getPageSizeFromPreference() + { + fPageSize = DebugUIPlugin.getDefault().getPreferenceStore().getInt(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE); + } + + private void updateDynamicLoadProperty() { + + boolean value = DebugUIPlugin + .getDefault() + .getPreferenceStore() + .getBoolean(IDebugPreferenceConstants.PREF_DYNAMIC_LOAD_MEM); + + if (value != isDynamicLoad()) + { + setDynamicLoad(value); + + if (!fIsDisposed) { + if (isDynamicLoad()) { + fContentDescriptor.setPostBuffer(20); + fContentDescriptor.setPreBuffer(20); + fContentDescriptor.setNumLines(getNumberOfVisibleLines()); + + } else { + fContentDescriptor.setPostBuffer(0); + fContentDescriptor.setPreBuffer(0); + fContentDescriptor.setNumLines(fPageSize); + } + } + } + } + + private void getDynamicLoadFromPreference() + { + setDynamicLoad(DebugUIPlugin.getDefault().getPreferenceStore().getBoolean(IDebugPreferenceConstants.PREF_DYNAMIC_LOAD_MEM)); + } + + private boolean isDynamicLoad() + { + return fContentDescriptor.isDynamicLoad(); + } + + private int getPageSize() + { + return fPageSize; + } + + private int getNumLinesToLoad() { + int numberOfLines = -1; + + if (isDynamicLoad()) + numberOfLines = getNumberOfVisibleLines(); + else + numberOfLines = getPageSize(); + + return numberOfLines; + } + + private void setDynamicLoad(boolean load) + { + fContentDescriptor.setDynamicLoad(load); + } + + private void handlePageStartAddressChanged(BigInteger address) + { + // do not handle if in dynamic mode + if (isDynamicLoad()) + return; + + if (!(getMemoryBlock() instanceof IMemoryBlockExtension)) + return; + + // do not handle event if the base address of the memory + // block has changed, wait for debug event to update to + // new location + if (isMemoryBlockBaseAddressChanged()) + return; + + if(fTableViewer.getKey(0).equals(address)) + return; + + BigInteger start = fContentDescriptor.getStartAddress(); + BigInteger end = fContentDescriptor.getEndAddress(); + + // smaller than start address, load at start address + if (address.compareTo(start) < 0) + { + if (isAtTopLimit()) + return; + + address = start; + } + + // bigger than end address, no need to load, already at top + if (address.compareTo(end) > 0) + { + if (isAtBottomLimit()) + return; + + address = end.subtract(BigInteger.valueOf(getPageSizeInUnits())); + } + + fContentDescriptor.setLoadAddress(address); + final BigInteger finaladdress = address; + Runnable runnable = new Runnable() { + public void run() { + if (fTableViewer.getTable().isDisposed()) + return; + + fTableViewer.setTopIndex(finaladdress); + refresh(); + }}; + + runOnUIThread(runnable); + + updateSyncPageStartAddress(address); + updateSyncTopAddress(address); + } + private void handleDyanicLoadChanged() { + + // if currently in dynamic load mode, update page + // start address + BigInteger pageStart = getTopVisibleAddress(); + updateSyncPageStartAddress(pageStart); + + updateDynamicLoadProperty(); + if (isDynamicLoad()) + { + refresh(); + fTableViewer.setTopIndex(pageStart); + } + else + { + handlePageStartAddressChanged(pageStart); + } + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#becomesHidden() + */ + public void becomesHidden() { + // creates new object for storing potential changes in sync properties + fPendingSyncProperties = new PendingPropertyChanges(); + super.becomesHidden(); + + if (getMemoryBlock() instanceof IMemoryBlockExtension) + updateRenderingLabel(false); + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#becomesVisible() + */ + public void becomesVisible() { + + if (!fIsCreated) + { + // label can still be constructed even though the rendering has not finished being + // initialized + updateRenderingLabel(true); + super.becomesVisible(); + return; + } + + // do not do anything if already visible + if (isVisible() == true) + { + // super should always be called + super.becomesVisible(); + return; + } + + super.becomesVisible(); + + if (fPendingSyncProperties != null) + { + // deal with format + boolean format = false; + int rowSize = getBytesPerLine(); + if (fPendingSyncProperties.getRowSize() > 0) + { + format = true; + rowSize = fPendingSyncProperties.getRowSize(); + } + + int colSize = getBytesPerColumn(); + if (fPendingSyncProperties.getColumnSize() > 0) + { + format = true; + colSize = fPendingSyncProperties.getColumnSize(); + } + + if (format) + format(rowSize, colSize); + + BigInteger selectedAddress = fPendingSyncProperties.getSelectedAddress(); + if (selectedAddress != null) + fTableViewer.setSelection(selectedAddress); + + updateDynamicLoadProperty(); + + if (isDynamicLoad()) + { + BigInteger topVisibleAddress = fPendingSyncProperties.getTopVisibleAddress(); + if (topVisibleAddress != null) + { + fContentDescriptor.setLoadAddress(topVisibleAddress); + fTableViewer.setTopIndex(topVisibleAddress); + } + } + else if (!(getMemoryBlock() instanceof IMemoryBlockExtension)) + { + BigInteger topVisibleAddress = fPendingSyncProperties.getTopVisibleAddress(); + if (topVisibleAddress != null) + fTableViewer.setTopIndex(topVisibleAddress); + } + else + { + if (fPendingSyncProperties.getPageSize() > 0) + { + fPageSize = fPendingSyncProperties.getPageSize(); + fContentDescriptor.setNumLines(fPageSize); + } + + BigInteger pageStartAddress = fPendingSyncProperties.getPageStartAddress(); + if (pageStartAddress != null) + fContentDescriptor.setLoadAddress(pageStartAddress); + + fTableViewer.setTopIndex(pageStartAddress); + } + + showTable(); + refresh(); + } + + updateRenderingLabel(true); + + Job job = new Job("becomesVisible") //$NON-NLS-1$ + { + protected IStatus run(IProgressMonitor monitor) { + if (fIsDisposed) + return Status.OK_STATUS; + try { + fContentDescriptor.updateContentBaseAddress(); + } catch (DebugException e) { + showMessage(e.getMessage()); + } + return Status.OK_STATUS; + } + }; + job.setSystem(true); + job.schedule(); + + // discard these properties + fPendingSyncProperties = null; + } + + /** + * Handle column size changed event from synchronizer + * @param newColumnSize + */ + private void columnSizeChanged(final int newColumnSize) { + // ignore event if rendering is not visible + if (!isVisible()) + return; + + Display.getDefault().asyncExec(new Runnable() { + public void run() { + int rowSize = getBytesPerLine(); + if (rowSize < newColumnSize) + rowSize = newColumnSize; + + format(rowSize, newColumnSize); + } + }); + } + + /** + * @param newRowSize - new row size in number of bytes + */ + private void rowSizeChanged(final int newRowSize) + { + // ignore event if rendering is not visible + if (!isVisible()) + return; + + Display.getDefault().asyncExec(new Runnable() { + public void run() { + int colSize = getBytesPerColumn(); + if (newRowSize < colSize) + colSize = newRowSize; + + format(newRowSize, colSize); + } + }); + } + + /** + * update selected address in synchronizer if update is true. + */ + private void updateSyncSelectedAddress(BigInteger address) { + + if (!fIsCreated) + return; + PropertyChangeEvent event = new PropertyChangeEvent(this, AbstractAsyncTableRendering.PROPERTY_SELECTED_ADDRESS, null, address); + firePropertyChangedEvent(event); + } + + /** + * update column size in synchronizer + */ + private void updateSyncColSize() { + + if (!fIsCreated) + return; + + PropertyChangeEvent event = new PropertyChangeEvent(this, AbstractAsyncTableRendering.PROPERTY_COL_SIZE, null, new Integer(fColumnSize)); + firePropertyChangedEvent(event); + } + + /** + * update column size in synchronizer + */ + private void updateSyncRowSize() { + + if (!fIsCreated) + return; + + PropertyChangeEvent event = new PropertyChangeEvent(this, AbstractAsyncTableRendering.PROPERTY_ROW_SIZE, null, new Integer(fBytePerLine)); + firePropertyChangedEvent(event); + } + + /** + * update top visible address in synchronizer + */ + private void updateSyncTopAddress(BigInteger address) { + + if (!fIsCreated) + return; + + PropertyChangeEvent event = new PropertyChangeEvent(this, AbstractAsyncTableRendering.PROPERTY_TOP_ADDRESS, null, address); + firePropertyChangedEvent(event); + } + + private void updateSyncPageStartAddress(BigInteger address) { + + if (!fIsCreated) + return; + + if (isMemoryBlockBaseAddressChanged()) + return; + + PropertyChangeEvent event = new PropertyChangeEvent(this, IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS, null, address); + firePropertyChangedEvent(event); + } + + /** + * Returns the color provider for this rendering's memory block or + * null if none. + *

+ * By default a color provider is obtained by asking this rendering's + * memory block for its {@link IColorProvider} adapter. When the color + * provider is queried for color information, it is provided with a + * {@link MemoryRenderingElement} as an argument. + *

+ * @return the color provider for this rendering's memory block, + * or null + */ + protected IColorProvider getColorProviderAdapter() + { + return (IColorProvider)getMemoryBlock().getAdapter(IColorProvider.class); + } + + /** + * Returns the label provider for this rendering's memory block or + * null if none. + *

+ * By default a label provider is obtained by asking this rendering's + * memory block for its {@link ILabelProvider} adapter. When the label + * provider is queried for label information, it is provided with a + * {@link MemoryRenderingElement} as an argument. + *

+ * @return the label provider for this rendering's memory block, + * or null + */ + protected ILabelProvider getLabelProviderAdapter() + { + return (ILabelProvider)getMemoryBlock().getAdapter(ILabelProvider.class); + } + + /** + * Returns the font provider for this rendering's memory block or + * null if none. + *

+ * By default a font provider is obtained by asking this rendering's + * memory block for its {@link IFontProvider} adapter. When the font + * provider is queried for font information, it is provided with a + * {@link MemoryRenderingElement} as an argument. + *

+ * @return the font provider for this rendering's memory block, + * or null + */ + protected IFontProvider getFontProviderAdapter() + { + return (IFontProvider)getMemoryBlock().getAdapter(IFontProvider.class); + } + + /** + * Returns the table presentation for this rendering's memory block or + * null if none. + *

+ * By default a table presentation is obtained by asking this rendering's + * memory block for its {@link IMemoryBlockTablePresentation} adapter. + *

+ * @return the table presentation for this rendering's memory block, + * or null + */ + protected IMemoryBlockTablePresentation getTablePresentationAdapter() + { + return (IMemoryBlockTablePresentation)getMemoryBlock().getAdapter(IMemoryBlockTablePresentation.class); + } + + /** + * Setup the viewer so it supports hovers to show the offset of each field + */ + private void createToolTip() { + + fToolTipShell = new Shell(DebugUIPlugin.getShell(), SWT.ON_TOP | SWT.RESIZE ); + GridLayout gridLayout = new GridLayout(); + gridLayout.numColumns = 1; + gridLayout.marginWidth = 2; + gridLayout.marginHeight = 0; + fToolTipShell.setLayout(gridLayout); + fToolTipShell.setBackground(fTableViewer.getTable().getDisplay().getSystemColor(SWT.COLOR_INFO_BACKGROUND)); + + final Control toolTipControl = createToolTipControl(fToolTipShell); + + if (toolTipControl == null) + { + // if client decide not to use tooltip support + fToolTipShell.dispose(); + return; + } + + MouseTrackAdapter listener = new MouseTrackAdapter(){ + + private TableItem fTooltipItem = null; + private int fCol = -1; + + public void mouseExit(MouseEvent e){ + + if (!fToolTipShell.isDisposed()) + fToolTipShell.setVisible(false); + fTooltipItem = null; + } + + public void mouseHover(MouseEvent e){ + + Point hoverPoint = new Point(e.x, e.y); + Control control = null; + + if (e.widget instanceof Control) + control = (Control)e.widget; + + if (control == null) + return; + + hoverPoint = control.toDisplay(hoverPoint); + TableItem item = getItem(hoverPoint); + int column = getColumn(hoverPoint); + + //Only if there is a change in hover + if(this.fTooltipItem != item || fCol != column){ + + //Keep Track of the latest hover + fTooltipItem = item; + fCol = column; + + if(item != null){ + toolTipAboutToShow(toolTipControl, fTooltipItem, column); + + //Setting location of the tooltip + Rectangle shellBounds = fToolTipShell.getBounds(); + shellBounds.x = hoverPoint.x; + shellBounds.y = hoverPoint.y + item.getBounds(0).height; + + fToolTipShell.setBounds(shellBounds); + fToolTipShell.pack(); + + fToolTipShell.setVisible(true); + } + else { + fToolTipShell.setVisible(false); + } + } + } + }; + + fTableViewer.getTable().addMouseTrackListener(listener); + fTableViewer.getCursor().addMouseTrackListener(listener); + } + + /** + * Creates the control used to display tool tips for cells in this table. By default + * a label is used to display the address of the cell. Clients may override this + * method to create custom tooltip controls. + *

+ * Also see the methods getToolTipText(...) and + * toolTipAboutToShow(...). + *

+ * @param composite parent for the tooltip control + * @return the tooltip control to be displayed + * @since 3.2 + */ + protected Control createToolTipControl(Composite composite) { + Control fToolTipLabel = new Label(composite, SWT.NONE); + fToolTipLabel.setForeground(fTableViewer.getTable().getDisplay().getSystemColor(SWT.COLOR_INFO_FOREGROUND)); + fToolTipLabel.setBackground(fTableViewer.getTable().getDisplay().getSystemColor(SWT.COLOR_INFO_BACKGROUND)); + fToolTipLabel.setLayoutData(new GridData(GridData.FILL_HORIZONTAL | + GridData.VERTICAL_ALIGN_CENTER)); + return fToolTipLabel; + } + + /** + * Bug with table widget,BUG 113015, the widget is not able to return the correct + * table item if SWT.FULL_SELECTION is not on when the table is created. + * Created the following function to work around the problem. + * We can remove this method when the bug is fixed. + * @param point + * @return the table item where the point is located, return null if the item cannot be located. + */ + private TableItem getItem(Point point) + { + TableItem[] items = fTableViewer.getTable().getItems(); + for (int i=0; i point.x) + return i; + } + } + return -1; + } + + /** + * Called when the tool tip is about to show in this rendering. + * Clients who overrides createTooltipControl may need to + * also override this method to ensure that the tooltip shows up properly + * in their customized control. + *

+ * By default a text tooltip is displayed, and the contents for the tooltip + * are generated by the getToolTipText(...) method. + *

+ * @param toolTipControl - the control for displaying the tooltip + * @param item - the table item where the mouse is pointing. + * @param col - the column at which the mouse is pointing. + * @since 3.2 + */ + protected void toolTipAboutToShow(Control toolTipControl, TableItem item, + int col) { + if (toolTipControl instanceof Label) { + Object address = fTableViewer.getKey(fTableViewer.getTable().indexOf(item), col); + if (address != null && address instanceof BigInteger) { + Object data = item.getData(); + if (data instanceof MemorySegment) { + MemorySegment line = (MemorySegment) data; + + if (col > 0) { + int start = (col - 1) * getBytesPerColumn(); + int end = start + getBytesPerColumn(); + MemoryByte[] bytes = line.getBytes(start, end); + + String str = getToolTipText((BigInteger)address, bytes); + + if (str != null) + ((Label) toolTipControl).setText(str); + } else { + String str = getToolTipText((BigInteger)address, + new MemoryByte[] {}); + + if (str != null) + ((Label) toolTipControl).setText(str); + } + } + } + } + } + + /** + * Returns the text to display in a tool tip at the specified address + * for the specified bytes. By default the address of the bytes is displayed. + * Subclasses may override. + * + * @param address address of cell that tool tip is displayed for + * @param bytes the bytes in the cell + * @return the tooltip text for the memory bytes located at the specified + * address + * @since 3.2 + */ + protected String getToolTipText(BigInteger address, MemoryByte[] bytes) + { + StringBuffer buf = new StringBuffer("0x"); //$NON-NLS-1$ + buf.append(address.toString(16).toUpperCase()); + + return buf.toString(); + } + + private void setColumnHeadings() + { + String[] columnLabels = new String[0]; + + IMemoryBlockTablePresentation presentation = getTablePresentationAdapter(); + if (presentation != null) + { + columnLabels = presentation.getColumnLabels(getMemoryBlock(), getBytesPerLine(), getBytesPerLine()/getBytesPerColumn()); + } + + // check that column labels returned are not null + if (columnLabels == null) + columnLabels = new String[0]; + + int numByteColumns = fBytePerLine/fColumnSize; + + TableColumn[] columns = fTableViewer.getTable().getColumns(); + + int j=0; + for (int i=1; i= 4) + { + columns[i].setText(Integer.toHexString(j*addressableUnit).toUpperCase() + + " - " + Integer.toHexString(j*addressableUnit+addressableUnit-1).toUpperCase()); //$NON-NLS-1$ + } + else + { + columns[i].setText(Integer.toHexString(j*addressableUnit).toUpperCase()); + } + j++; + } + } + } + + /** + * + * Return this rendering's viewer + * @return this rendering's viewer + */ + public StructuredViewer getViewer() + { + return fTableViewer; + } + + private boolean isMemoryBlockBaseAddressChanged() + { + try { + BigInteger address = getMemoryBlockBaseAddress(); + BigInteger oldBaseAddress = fContentDescriptor.getContentBaseAddress(); + if (!oldBaseAddress.equals(address)) + return true; + } catch (DebugException e) { + // fail silently + } + return false; + } + + /** + * @param topVisibleAddress + */ + private void createContentDescriptor(final BigInteger topVisibleAddress) { + fContentDescriptor = new TableRenderingContentDescriptor(AbstractAsyncTableRendering.this); + fContentDescriptor.setPostBuffer(20); + fContentDescriptor.setPreBuffer(20); + fContentDescriptor.setLoadAddress(topVisibleAddress); + try { + fContentDescriptor.updateContentBaseAddress(); + + } catch (DebugException e) { + fError = true; + showMessage(e.getMessage()); + } + + fContentDescriptor.setAddressableSize(getAddressableSize()); + + try { + int addressSize = 4; + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + IMemoryBlockExtension extMb = (IMemoryBlockExtension)getMemoryBlock(); + addressSize = extMb.getAddressSize(); + + if (addressSize <= 0) + { + DebugUIPlugin.logErrorMessage("Invalid address Size: " + addressSize); //$NON-NLS-1$ + addressSize = 4; + } + fContentDescriptor.setAddressSize(addressSize); + } + fContentDescriptor.setAddressSize(addressSize); + } catch (DebugException e) { + fError = true; + showMessage(e.getMessage()); + } finally { + if (fContentDescriptor.getAddressSize() <= 0) + fContentDescriptor.setAddressSize(4); + } + + } + + private TableRenderingContentDescriptor getContentDescriptor() + { + return fContentDescriptor; + } + + private void createGoToAddressComposite(Composite parent) + { + fGoToAddressComposite = new GoToAddressComposite(); + fGoToAddressComposite.createControl(parent); + Button button = fGoToAddressComposite.getButton(IDialogConstants.OK_ID); + if (button != null) + { + button.addSelectionListener(new SelectionAdapter() { + + public void widgetSelected(SelectionEvent e) { + doGoToAddress(); + } + }); + + button = fGoToAddressComposite.getButton(IDialogConstants.CANCEL_ID); + if (button != null) + { + button.addSelectionListener(new SelectionAdapter() { + public void widgetSelected(SelectionEvent e) { + hideGotoAddressComposite(); + }}); + } + } + + fGoToAddressComposite.getExpressionWidget().addSelectionListener(new SelectionAdapter() { + public void widgetDefaultSelected(SelectionEvent e) { + doGoToAddress(); + }}); + + fGoToAddressComposite.getExpressionWidget().addKeyListener(new KeyAdapter() { + + public void keyPressed(KeyEvent e) { + if (e.keyCode == SWT.ESC) + hideGotoAddressComposite(); + super.keyPressed(e); + }}); + } + + private void showGoToAddressComposite() { + + String selectedStr = getSelectedAsString(); + Text text = fGoToAddressComposite.getExpressionWidget(); + text.setText(selectedStr); + text.setSelection(0, text.getCharCount()); + + double height = fGoToAddressComposite.getHeight(); + double canvasHeight = fSashForm.getParent().getClientArea().height; + double tableHeight = canvasHeight - height; + + double tableWeight = (tableHeight/canvasHeight) * 100; + double textWeight = (height / canvasHeight) * 100; + fSashForm.setWeights(new int[]{(int)tableWeight, (int)textWeight}); + fSashForm.setMaximizedControl(null); + + fGoToAddressComposite.getExpressionWidget().setFocus(); + } + + private void hideGotoAddressComposite() + { + fSashForm.setMaximizedControl(fTableViewer.getControl()); + if (isActivated()) + fTableViewer.getControl().setFocus(); + } + + /** + * + */ + private void doGoToAddress() { + try { + BigInteger address = fGoToAddressComposite.getGoToAddress(fContentDescriptor.getContentBaseAddress(), getSelectedAddress()); + fGoToAddressAction.doGoToAddress(address.toString(16)); + hideGotoAddressComposite(); + } catch (DebugException e1) { + MemoryViewUtil.openError(DebugUIMessages.GoToAddressAction_Go_to_address_failed, + DebugUIMessages.GoToAddressAction_Go_to_address_failed, e1); + } catch (NumberFormatException e1) + { + MemoryViewUtil.openError(DebugUIMessages.GoToAddressAction_Go_to_address_failed, + DebugUIMessages.GoToAddressAction_Address_is_invalid, e1); + } + } + + public void activated() { + super.activated(); + + fActivated = true; + IWorkbench workbench = PlatformUI.getWorkbench(); + ICommandService commandSupport = (ICommandService)workbench.getAdapter(ICommandService.class); + IContextService contextSupport = (IContextService)workbench.getAdapter(IContextService.class); + + if (commandSupport != null && contextSupport != null) + { + fContext.add(contextSupport.activateContext(ID_ASYNC_TABLE_RENDERING_CONTEXT)); + Command gotoCommand = commandSupport.getCommand(ID_GO_TO_ADDRESS_COMMAND); + + if (fGoToAddressHandler == null) + { + fGoToAddressHandler = new AbstractHandler() { + public Object execute(ExecutionEvent event) throws ExecutionException { + if (fSashForm.getMaximizedControl() != null) + fGoToAddressAction.run(); + else + hideGotoAddressComposite(); + return null; + }}; + } + gotoCommand.setHandler(fGoToAddressHandler); + } + + } + + + public void deactivated() { + + fActivated = false; + IWorkbench workbench = PlatformUI.getWorkbench(); + ICommandService commandSupport = (ICommandService)workbench.getAdapter(ICommandService.class); + IContextService contextSupport = (IContextService)workbench.getAdapter(IContextService.class); + + if (commandSupport != null && contextSupport != null) + { + // remove handler + Command command = commandSupport.getCommand(ID_GO_TO_ADDRESS_COMMAND); + command.setHandler(null); + + if (fContext != null) + contextSupport.deactivateContexts(fContext); + } + super.deactivated(); + } + + private boolean isActivated() + { + return fActivated; + } + + /** + * Returns text for the given memory bytes at the specified address for the specified + * rendering type. This is called by the label provider for. + * Subclasses must override. + * + * @param renderingTypeId rendering type identifier + * @param address address where the bytes belong to + * @param data the bytes + * @return a string to represent the memory. Cannot not return null. + * Returns a string to pad the cell if the memory cannot be converted + * successfully. + */ + abstract public String getString(String renderingTypeId, BigInteger address, MemoryByte[] data); + + /** + * Returns bytes for the given text corresponding to bytes at the given + * address for the specified rendering type. This is called by the cell modifier + * when modifying bytes in a memory block. + * Subclasses must convert the string value to an array of bytes. The bytes will + * be passed to the debug adapter for memory block modification. + * Returns null if the bytes cannot be formatted properly. + * + * @param renderingTypeId rendering type identifier + * @param address address the bytes begin at + * @param currentValues current values of the data in bytes format + * @param newValue the string to be converted to bytes + * @return the bytes converted from a string + */ + abstract public byte[] getBytes(String renderingTypeId, BigInteger address, MemoryByte[] currentValues, String newValue); +} diff --git a/samples/src/AbstractTableRendering.java b/samples/src/AbstractTableRendering.java new file mode 100644 index 0000000..6896937 --- /dev/null +++ b/samples/src/AbstractTableRendering.java @@ -0,0 +1,3740 @@ +/******************************************************************************* + * Copyright (c) 2004, 2006 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ + +package org.eclipse.debug.ui.memory; + +import java.math.BigInteger; + +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Status; +import org.eclipse.debug.core.DebugException; +import org.eclipse.debug.core.model.IMemoryBlock; +import org.eclipse.debug.core.model.IMemoryBlockExtension; +import org.eclipse.debug.core.model.MemoryByte; +import org.eclipse.debug.internal.ui.DebugUIMessages; +import org.eclipse.debug.internal.ui.DebugUIPlugin; +import org.eclipse.debug.internal.ui.IInternalDebugUIConstants; +import org.eclipse.debug.internal.ui.memory.IMemoryBlockConnection; +import org.eclipse.debug.internal.ui.memory.IPersistableDebugElement; +import org.eclipse.debug.internal.ui.preferences.IDebugPreferenceConstants; +import org.eclipse.debug.internal.ui.views.memory.MemoryViewUtil; +import org.eclipse.debug.internal.ui.views.memory.renderings.CopyTableRenderingToClipboardAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.FormatTableRenderingAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.FormatTableRenderingDialog; +import org.eclipse.debug.internal.ui.views.memory.renderings.GoToAddressAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.AbstractBaseTableRendering; +import org.eclipse.debug.internal.ui.views.memory.renderings.PrintTableRenderingAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.ReformatAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.ResetToBaseAddressAction; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingCellModifier; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingContentInput; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingContentProvider; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingLabelProvider; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingLabelProviderEx; +import org.eclipse.debug.internal.ui.views.memory.renderings.TableRenderingLine; +import org.eclipse.debug.ui.DebugUITools; +import org.eclipse.debug.ui.IDebugUIConstants; +import org.eclipse.jface.action.Action; +import org.eclipse.jface.action.IMenuListener; +import org.eclipse.jface.action.IMenuManager; +import org.eclipse.jface.action.Separator; +import org.eclipse.jface.preference.IPreferenceStore; +import org.eclipse.jface.resource.ImageDescriptor; +import org.eclipse.jface.resource.JFaceResources; +import org.eclipse.jface.text.Document; +import org.eclipse.jface.text.TextViewer; +import org.eclipse.jface.util.IPropertyChangeListener; +import org.eclipse.jface.util.PropertyChangeEvent; +import org.eclipse.jface.viewers.CellEditor; +import org.eclipse.jface.viewers.IBasicPropertyConstants; +import org.eclipse.jface.viewers.ICellModifier; +import org.eclipse.jface.viewers.IColorProvider; +import org.eclipse.jface.viewers.IFontProvider; +import org.eclipse.jface.viewers.ILabelProvider; +import org.eclipse.jface.viewers.TableViewer; +import org.eclipse.jface.viewers.TextCellEditor; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.StyledText; +import org.eclipse.swt.custom.TableCursor; +import org.eclipse.swt.custom.TableEditor; +import org.eclipse.swt.events.DisposeEvent; +import org.eclipse.swt.events.DisposeListener; +import org.eclipse.swt.events.FocusAdapter; +import org.eclipse.swt.events.FocusEvent; +import org.eclipse.swt.events.KeyAdapter; +import org.eclipse.swt.events.KeyEvent; +import org.eclipse.swt.events.MouseAdapter; +import org.eclipse.swt.events.MouseEvent; +import org.eclipse.swt.events.MouseTrackAdapter; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.events.TraverseEvent; +import org.eclipse.swt.events.TraverseListener; +import org.eclipse.swt.graphics.Font; +import org.eclipse.swt.graphics.Point; +import org.eclipse.swt.graphics.Rectangle; +import org.eclipse.swt.layout.GridData; +import org.eclipse.swt.layout.GridLayout; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.ScrollBar; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Table; +import org.eclipse.swt.widgets.TableColumn; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swt.widgets.Text; +import org.eclipse.ui.IWorkbenchActionConstants; +import org.eclipse.ui.PlatformUI; +import org.eclipse.ui.dialogs.PropertyDialogAction; +import org.eclipse.ui.model.IWorkbenchAdapter; +import org.eclipse.ui.part.PageBook; + +/** + * Abstract implementation of a table rendering. + *

+ * Clients should subclass from this class if they wish to provide a + * table rendering. + *

+ *

+ * + * The label of the rendering is constructed by retrieving the expression from + * IMemoryBlockExtension. For IMemoryBlock, the label is constructed + * using the memory block's start address. + * + * This rendering manages the change states of its memory bytes if the memory + * block does not opt to manage the change states. For IMemoryBlockExtension, if + * the memory block returns false when #supportsChangeManagement() is called, this + * rendering will calculate the change state for each byte when its content is updated. + * Clients may manages the change states of its memory block by returning true when + * #supportsChangeManagement() is called. This will cause this rendering to stop + * calculating the change states of the memory block. Instead it would rely on the + * attributes returned in the MemoryByte array to determine if a byte has changed. + * For IMemoryBlock, this rendering will manage the change states its content. + * + * When firing change event, be aware of the following: + * - whenever a change event is fired, the content provider for Memory View + * view checks to see if memory has actually changed. + * - If memory has actually changed, a refresh will commence. Changes to the memory block + * will be computed and will be shown with the delta icons. + * - If memory has not changed, content will not be refreshed. However, previous delta information + * will be erased. The screen will be refreshed to show that no memory has been changed. (All + * delta icons will be removed.) + * + * Please note that these APIs will be called multiple times by the Memory View. + * To improve performance, debug adapters need to cache the content of its memory block and only + * retrieve updated data when necessary. + *

+ + * @since 3.1 + */ +public abstract class AbstractTableRendering extends AbstractBaseTableRendering implements IPropertyChangeListener, IResettableMemoryRendering{ + + /** + * Property identifier for the selected address in a table rendering + * This property is used for synchronization between renderings. + */ + public static final String PROPERTY_SELECTED_ADDRESS = "selectedAddress"; //$NON-NLS-1$ + + /** + * Property identifier for the column size in a table rendering + * This property is used for synchronization between renderings. + */ + public static final String PROPERTY_COL_SIZE = "columnSize"; //$NON-NLS-1$ + + /** + * Property identifier for the top row address in a table rendering. + * This property is used for synchronization between renderings. + */ + public static final String PROPERTY_TOP_ADDRESS = "topAddress"; //$NON-NLS-1$ + + /** + * Property identifier for the row size in a table rendering + * This property is used for synchronization between renderings. + * @since 3.2 + */ + public static final String PROPERTY_ROW_SIZE = "rowSize"; //$NON-NLS-1$ + + private PageBook fPageBook; + private TableViewer fTableViewer; + private TextViewer fTextViewer; + + private int fBytePerLine; // number of bytes per line: 16 + private int fColumnSize; // number of bytes per column: 1,2,4,8 + private int fAddressableSize; + + private boolean fIsShowingErrorPage; + + private TableRenderingContentProvider fContentProvider; + private BigInteger fSelectedAddress; + private TableRenderingContentInput fContentInput; + private TableRenderingCellModifier fCellModifier; + private boolean fIsCreated; + private CellEditor[] fEditors; + private String fLabel; + private TableCursor fTableCursor; + private boolean fIsDisposed; + private TraverseListener fCursorTraverseListener; + private KeyAdapter fCursorKeyAdapter; + private BigInteger fTopRowAddress; + + private CopyTableRenderingToClipboardAction fCopyToClipboardAction; + private GoToAddressAction fGoToAddressAction; + private ResetToBaseAddressAction fResetMemoryBlockAction; + private PrintTableRenderingAction fPrintViewTabAction; + private ReformatAction fReformatAction; + private ToggleAddressColumnAction fToggleAddressColumnAction; + private EventHandleLock fEvtHandleLock = new EventHandleLock(); + private TableEditor fCursorEditor; + private FocusAdapter fEditorFocusListener; + private MouseAdapter fCursorMouseListener; + private KeyAdapter fEditorKeyListener; + private SelectionAdapter fCursorSelectionListener; + private IWorkbenchAdapter fWorkbenchAdapter; + private IMemoryBlockConnection fConnection; + + private boolean fIsShowAddressColumn = true; + private SelectionAdapter fScrollbarSelectionListener; + + private PropertyDialogAction fPropertiesAction; + + private int fPageSize; + private NextPageAction fNextAction; + private PrevPageAction fPrevAction; + + private Shell fToolTipShell; + private FormatTableRenderingAction fFormatRenderingAction; + + private IMenuListener fMenuListener; + + private class EventHandleLock + { + Object fOwner; + + public boolean acquireLock(Object client) + { + if (fOwner == null) + { + fOwner = client; + return true; + } + return false; + } + + public boolean releaseLock(Object client) + { + if (fOwner == client) + { + fOwner = null; + return true; + } + return false; + } + + public boolean isLocked() + { + return (fOwner != null); + } + } + + + private class ToggleAddressColumnAction extends Action { + + public ToggleAddressColumnAction() { + super(); + PlatformUI.getWorkbench().getHelpSystem().setHelp(this, IDebugUIConstants.PLUGIN_ID + + ".ShowAddressColumnAction_context"); //$NON-NLS-1$ + updateActionLabel(); + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.jface.action.IAction#run() + */ + public void run() { + fIsShowAddressColumn = !fIsShowAddressColumn; + resizeColumnsToPreferredSize(); + updateActionLabel(); + } + + /** + * + */ + private void updateActionLabel() { + if (fIsShowAddressColumn) { + setText(DebugUIMessages.ShowAddressColumnAction_0); + } else { + setText(DebugUIMessages.ShowAddressColumnAction_1); + } + } + } + + + private class NextPageAction extends Action + { + private NextPageAction() + { + super(); + setText(DebugUIMessages.AbstractTableRendering_4); + PlatformUI.getWorkbench().getHelpSystem().setHelp(this, IDebugUIConstants.PLUGIN_ID + ".NextPageAction_context"); //$NON-NLS-1$ + } + + public void run() { + BigInteger address = fContentInput.getLoadAddress(); + address = address.add(BigInteger.valueOf(getPageSizeInUnits())); + handlePageStartAddressChanged(address); + } + } + + private class PrevPageAction extends Action + { + private PrevPageAction() + { + super(); + setText(DebugUIMessages.AbstractTableRendering_6); + PlatformUI.getWorkbench().getHelpSystem().setHelp(this, IDebugUIConstants.PLUGIN_ID + ".PrevPageAction_context"); //$NON-NLS-1$ + } + + public void run() { + BigInteger address = fContentInput.getLoadAddress(); + address = address.subtract(BigInteger.valueOf(getPageSizeInUnits())); + handlePageStartAddressChanged(address); + } + } + + /** + * Constructs a new table rendering of the specified type. + * + * @param renderingId memory rendering type identifier + */ + public AbstractTableRendering(String renderingId) { + super(renderingId); + } + + /* (non-Javadoc) + * @see org.eclipse.jface.util.IPropertyChangeListener#propertyChange(org.eclipse.jface.util.PropertyChangeEvent) + */ + public void propertyChange(PropertyChangeEvent event) { + // if memory view table font has changed + if (event.getProperty().equals(IInternalDebugUIConstants.FONT_NAME)) + { + if (!fIsDisposed) + { + Font memoryViewFont = JFaceResources.getFont(IInternalDebugUIConstants.FONT_NAME); + setFont(memoryViewFont); + } + return; + } + + if (event.getProperty().equals(IDebugUIConstants.PREF_PADDED_STR) || + event.getProperty().equals(IDebugUIConstants.PREF_MEMORY_HISTORY_KNOWN_COLOR) || + event.getProperty().equals(IDebugUIConstants.PREF_MEMORY_HISTORY_UNKNOWN_COLOR)) + { + if (!fIsDisposed) + { + fTableViewer.refresh(); + fTableCursor.redraw(); + } + return; + } + + Object evtSrc = event.getSource(); + + if (event.getProperty().equals(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE)) { + // always update page size, only refresh if the table is visible + getPageSizeFromPreference(); + } + + // do not handle event if the rendering is displaying an error + if (isDisplayingError()) + return; + + // do not handle property change event if the rendering is not visible + if (!isVisible()) + return; + + if (event.getProperty().equals(IDebugPreferenceConstants.PREF_DYNAMIC_LOAD_MEM)) { + handleDyanicLoadChanged(); + return; + } + + if (event.getProperty().equals(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE)) { + if (!isDynamicLoad()) + { + // only refresh if in non-autoload mode + refresh(); + } + return; + } + + if (evtSrc == this) + return; + + if (!(evtSrc instanceof IMemoryRendering)) + return; + + IMemoryRendering rendering = (IMemoryRendering)evtSrc; + IMemoryBlock memoryBlock = rendering.getMemoryBlock(); + + // do not handle event from renderings displaying other memory blocks + if (memoryBlock != getMemoryBlock()) + return; + + String propertyName = event.getProperty(); + Object value = event.getNewValue(); + + if (propertyName.equals(AbstractTableRendering.PROPERTY_SELECTED_ADDRESS) && value instanceof BigInteger) + { + selectedAddressChanged((BigInteger)value); + } + else if (propertyName.equals(AbstractTableRendering.PROPERTY_COL_SIZE) && value instanceof Integer) + { + columnSizeChanged(((Integer)value).intValue()); + } + else if (propertyName.equals(AbstractTableRendering.PROPERTY_ROW_SIZE) && value instanceof Integer) + { + rowSizeChanged(((Integer)value).intValue()); + } + else if (propertyName.equals(AbstractTableRendering.PROPERTY_TOP_ADDRESS) && value instanceof BigInteger) + { + if (needMoreLines()) + { + if (isDynamicLoad()) + reloadTable(getTopVisibleAddress(), false); + } + topVisibleAddressChanged((BigInteger)value, false); + } + else if (propertyName.equals(IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS) && value instanceof BigInteger) + { + handlePageStartAddressChanged((BigInteger)value); + } + } + + private void handleDyanicLoadChanged() { + + // if currently in dynamic load mode, update page + // start address + updateSyncPageStartAddress(); + + updateDynamicLoadProperty(); + if (isDynamicLoad()) + { + refresh(); + } + else + { + BigInteger pageStart = (BigInteger)getSynchronizedProperty(IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS); + if (pageStart == null) + pageStart = fTopRowAddress; + handlePageStartAddressChanged(pageStart); + } + } + + private void updateDynamicLoadProperty() { + + boolean value = DebugUIPlugin + .getDefault() + .getPreferenceStore() + .getBoolean(IDebugPreferenceConstants.PREF_DYNAMIC_LOAD_MEM); + + if (value != isDynamicLoad()) + { + fContentProvider.setDynamicLoad(value); + + if (!fIsDisposed) { + if (isDynamicLoad()) { + fContentInput.setPostBuffer(20); + fContentInput.setPreBuffer(20); + fContentInput.setDefaultBufferSize(20); + fContentInput.setNumLines(getNumberOfVisibleLines()); + + } else { + fContentInput.setPostBuffer(0); + fContentInput.setPreBuffer(0); + fContentInput.setDefaultBufferSize(0); + fContentInput.setNumLines(fPageSize); + } + } + } + } + + /** + * Handle top visible address change event from synchronizer + * @param address + */ + private void topVisibleAddressChanged(final BigInteger address, boolean force) + { + // do not handle event if rendering is not visible + // continue to handle event if caller decides to force the rendering + // to move to the top visible address even when the rendering + // is not visible + if (!isVisible() && !force) + return; + + // do not handle event if the base address of the memory + // block has changed, wait for debug event to update to + // new location + if (isBaseAddressChanged()) + return; + + if (!address.equals(fTopRowAddress)) + { + fTopRowAddress = address; + updateSyncTopAddress(); + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + + handleTopAddressChangedforExtended(address); + } + else + { + handleTopAddressChangedForSimple(address); + } + } + } + + /** + * @param address + */ + private void handleTopAddressChangedForSimple(final BigInteger address) { + // IMemoryBlock support + int index = findAddressIndex(address); + Table table = fTableViewer.getTable(); + if (index >= 0) + { + setTopIndex(table, index); + } + + if (isAddressVisible(fSelectedAddress)) + fTableCursor.setVisible(true); + else + fTableCursor.setVisible(false); + + } + + /** + * @param address + */ + private void handleTopAddressChangedforExtended(final BigInteger address) { + + Object evtLockClient = new Object(); + try + { + if (!fEvtHandleLock.acquireLock(evtLockClient)) + return; + + if (!isAddressOutOfRange(address)) + { + Table table = fTableViewer.getTable(); + int index = findAddressIndex(address); + if (index >= 3 && table.getItemCount() - (index+getNumberOfVisibleLines()) >= 3) + { + // update cursor position + setTopIndex(table, index); + } + else + { + int numInBuffer = table.getItemCount(); + if (index < 3) + { + if(isAtTopLimit()) + { + setTopIndex(table, index); + } + else + { + if (isDynamicLoad()) + reloadTable(address, false); + else + setTopIndex(table, index); + } + } + else if ((numInBuffer-(index+getNumberOfVisibleLines())) < 3) + { + if (!isAtBottomLimit() && isDynamicLoad()) + reloadTable(address, false); + else + setTopIndex(table, index); + } + } + } + else + { + // approaching limit, reload table + reloadTable(address, false); + } + + if (isAddressVisible(fSelectedAddress)) + fTableCursor.setVisible(true); + else + fTableCursor.setVisible(false); + } + finally + { + fEvtHandleLock.releaseLock(evtLockClient); + } + } + + /** + * @param value + */ + private void selectedAddressChanged(BigInteger value) { + + // do not handle event if the base address of the memory + // block has changed, wait for debug event to update to + // new location + if (isBaseAddressChanged()) + return; + + try { + // do not handle event if the event is out of range and the + // rendering is in non-dynamic-load mode, otherwise, will + // cause rendering to continue to scroll when it shouldn't + if (isDynamicLoad()) + goToAddress(value); + else if (!isAddressOutOfRange(value)) + goToAddress(value); + } catch (DebugException e) { + // do nothing + } + } + + private void handlePageStartAddressChanged(BigInteger address) + { + // do not handle if in dynamic mode + if (isDynamicLoad()) + return; + + if (fContentInput == null) + return; + + if (!(getMemoryBlock() instanceof IMemoryBlockExtension)) + return; + + // do not handle event if the base address of the memory + // block has changed, wait for debug event to update to + // new location + if (isBaseAddressChanged()) + return; + + if(fContentProvider.getBufferTopAddress().equals(address)) + return; + + BigInteger start = fContentInput.getStartAddress(); + BigInteger end = fContentInput.getEndAddress(); + + // smaller than start address, load at start address + if (address.compareTo(start) < 0) + { + if (isAtTopLimit()) + return; + + address = start; + } + + // bigger than end address, no need to load, already at top + if (address.compareTo(end) > 0) + { + if (isAtBottomLimit()) + return; + + address = end.subtract(BigInteger.valueOf(getPageSizeInUnits())); + } + + fContentInput.setLoadAddress(address); + refresh(); + updateSyncPageStartAddress(); + setTopIndex(fTableViewer.getTable(), 0); + fTopRowAddress = address; + updateSyncTopAddress(); + + BigInteger selectedAddress = (BigInteger)getSynchronizedProperty(AbstractTableRendering.PROPERTY_SELECTED_ADDRESS); + if (selectedAddress != null) + { + fSelectedAddress = selectedAddress; + if (!isAddressOutOfRange(fSelectedAddress)) + { + setCursorAtAddress(fSelectedAddress); + fTableCursor.setVisible(true); + } + else + { + fTableCursor.setVisible(false); + } + } + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#createControl(org.eclipse.swt.widgets.Composite) + */ + public Control createControl(Composite parent) { + + fPageBook = new PageBook(parent, SWT.NONE); + createErrorPage(fPageBook); + createTableViewer(fPageBook); + + fTableViewer.getTable().redraw(); + createToolTip(); + + return fPageBook; + } + + /** + * Create the table viewer and other support controls + * for this rendering. + * + * @param parent parent composite + */ + private void createTableViewer(Composite parent) { + + fTableViewer= new TableViewer(parent, SWT.SINGLE | SWT.H_SCROLL | SWT.V_SCROLL | SWT.HIDE_SELECTION | SWT.BORDER); + + TableRenderingLabelProvider labelProvider; + if (hasCustomizedDecorations()) + labelProvider = new TableRenderingLabelProviderEx(this); + else + labelProvider = new TableRenderingLabelProvider(this); + + fTableViewer.setLabelProvider(labelProvider); + + fContentProvider = new TableRenderingContentProvider(); + fContentProvider.setDynamicLoad(DebugUIPlugin.getDefault().getPreferenceStore().getBoolean(IDebugPreferenceConstants.PREF_DYNAMIC_LOAD_MEM)); + + fTableViewer.setContentProvider(fContentProvider); + fContentProvider.setViewer(fTableViewer); + + ScrollBar scroll = ((Table)fTableViewer.getControl()).getVerticalBar(); + scroll.setMinimum(-100); + scroll.setMaximum(200); + + fTableViewer.getTable().setHeaderVisible(true); + fTableViewer.getTable().setLinesVisible(true); + + + // set up addressable size and figure out number of bytes required per line + fAddressableSize = -1; + try { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + fAddressableSize = ((IMemoryBlockExtension)getMemoryBlock()).getAddressableSize(); + } catch (DebugException e1) { + // log error and default to 1 + fAddressableSize = 1; + displayError(e1); + return; + + } + if (getAddressableSize() < 1) + fAddressableSize = 1; + +// set up initial format + setupInitialFormat(); + +// set up selected address + setupSelectedAddress(); + + // figure out top visible address + BigInteger topVisibleAddress = getInitialTopVisibleAddress(); + + getPageSizeFromPreference(); + + if (isDynamicLoad()) + fContentInput = new TableRenderingContentInput(this, 20, 20, 20, topVisibleAddress, getNumberOfVisibleLines(), false, null); + else + { + BigInteger addressToLoad = topVisibleAddress; + + // check synchronization service to see if we need to sync with another rendering + Object obj = getSynchronizedProperty(IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS); + if (obj != null && obj instanceof BigInteger) + { + addressToLoad = (BigInteger)obj; + } + fContentInput = new TableRenderingContentInput(this, 0, 0, 0, addressToLoad, fPageSize, false, null); + } + + fTableViewer.setInput(fContentInput); + + // set up cell modifier + fCellModifier = new TableRenderingCellModifier(this); + fTableViewer.setCellModifier(fCellModifier); + + // SET UP FONT + // set to a non-proportional font + fTableViewer.getTable().setFont(JFaceResources.getFont(IInternalDebugUIConstants.FONT_NAME)); + if (!(getMemoryBlock() instanceof IMemoryBlockExtension)) + { + // If not extended memory block, do not create any buffer + // no scrolling + fContentInput.setPreBuffer(0); + fContentInput.setPostBuffer(0); + fContentInput.setDefaultBufferSize(0); + } + + // set up table cursor + createCursor(fTableViewer.getTable(), fSelectedAddress); + fTableViewer.getTable().addMouseListener(new MouseAdapter() { + public void mouseDown(MouseEvent e) { + handleTableMouseEvent(e); + }}); + + // create pop up menu for the rendering + createActions(); + createPopupMenu(fTableViewer.getControl()); + createPopupMenu(fTableCursor); + + fMenuListener = new IMenuListener() { + public void menuAboutToShow(IMenuManager manager) { + fillContextMenu(manager); + manager.add(new Separator(IWorkbenchActionConstants.MB_ADDITIONS)); + }}; + getPopupMenuManager().addMenuListener(fMenuListener); + + // now the rendering is successfully created + fIsCreated = true; + + //synchronize + addRenderingToSyncService(); + synchronize(); + + fTopRowAddress = getTopVisibleAddress(); + // Need to resize column after content is filled in + // Pack function does not work unless content is not filled in + // since the table is not able to compute the preferred size. + resizeColumnsToPreferredSize(); + try { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + if(((IMemoryBlockExtension)getMemoryBlock()).getBigBaseAddress() == null) + { + DebugException e = new DebugException(DebugUIPlugin.newErrorStatus(DebugUIMessages.AbstractTableRendering_1, null)); + displayError(e); + } + } + } catch (DebugException e1) { + displayError(e1); + } + + // add font change listener and update font when the font has been changed + JFaceResources.getFontRegistry().addListener(this); + fScrollbarSelectionListener = new SelectionAdapter() { + + public void widgetSelected(SelectionEvent event) { + handleScrollBarSelection(); + + }}; + scroll.addSelectionListener(fScrollbarSelectionListener); + DebugUIPlugin.getDefault().getPreferenceStore().addPropertyChangeListener(this); + } + + private boolean validateInitialFormat() + { + int rowSize = getDefaultRowSize(); + int columnSize = getDefaultColumnSize(); + + if (rowSize < columnSize || rowSize % columnSize != 0 || rowSize == 0 || columnSize == 0) + { + return false; + } + return true; + } + + private BigInteger getInitialTopVisibleAddress() { + BigInteger topVisibleAddress = (BigInteger) getSynchronizedProperty(AbstractTableRendering.PROPERTY_TOP_ADDRESS); + if (topVisibleAddress == null) + { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + try { + topVisibleAddress = ((IMemoryBlockExtension)getMemoryBlock()).getBigBaseAddress(); + } catch (DebugException e1) { + topVisibleAddress = new BigInteger("0"); //$NON-NLS-1$ + } + } + else + { + topVisibleAddress = BigInteger.valueOf(getMemoryBlock().getStartAddress()); + } + } + return topVisibleAddress; + } + + private void setupSelectedAddress() { + // figure out selected address + BigInteger selectedAddress = (BigInteger) getSynchronizedProperty(AbstractTableRendering.PROPERTY_SELECTED_ADDRESS); + if (selectedAddress == null) + { + if (getMemoryBlock() instanceof IMemoryBlockExtension) { + try { + selectedAddress = ((IMemoryBlockExtension) getMemoryBlock()) + .getBigBaseAddress(); + } catch (DebugException e1) { + selectedAddress = new BigInteger("0"); //$NON-NLS-1$ + } + if (selectedAddress == null) { + selectedAddress = new BigInteger("0"); //$NON-NLS-1$ + } + + } else { + long address = getMemoryBlock().getStartAddress(); + selectedAddress = BigInteger.valueOf(address); + } + } + setSelectedAddress(selectedAddress); + } + + private void setupInitialFormat() { + + boolean validated = validateInitialFormat(); + + if (!validated) + { + // pop up dialog to ask user for default values + StringBuffer msgBuffer = new StringBuffer(DebugUIMessages.AbstractTableRendering_20); + msgBuffer.append(" "); //$NON-NLS-1$ + msgBuffer.append(this.getLabel()); + msgBuffer.append("\n\n"); //$NON-NLS-1$ + msgBuffer.append(DebugUIMessages.AbstractTableRendering_16); + msgBuffer.append("\n"); //$NON-NLS-1$ + msgBuffer.append(DebugUIMessages.AbstractTableRendering_18); + msgBuffer.append("\n\n"); //$NON-NLS-1$ + + int bytePerLine = fBytePerLine; + int columnSize = fColumnSize; + + // initialize this value to populate the dialog properly + fBytePerLine = getDefaultRowSize() / getAddressableSize(); + fColumnSize = getDefaultColumnSize() / getAddressableSize(); + + FormatTableRenderingDialog dialog = new FormatTableRenderingDialog(this, DebugUIPlugin.getShell()); + dialog.openError(msgBuffer.toString()); + + // restore to original value before formatting + fBytePerLine = bytePerLine; + fColumnSize = columnSize; + + bytePerLine = dialog.getRowSize() * getAddressableSize(); + columnSize = dialog.getColumnSize() * getAddressableSize(); + + format(bytePerLine, columnSize); + } + else + { + // Row size is stored as number of addressable units in preference store + int bytePerLine = getDefaultRowSize(); + // column size is now stored as number of addressable units + int columnSize = getDefaultColumnSize(); + + // format memory block with specified "bytesPerLine" and "columnSize" + boolean ok = format(bytePerLine, columnSize); + + if (!ok) + { + // this is to ensure that the rest of the rendering can be created + // and we can recover from a format error + format(bytePerLine, bytePerLine); + } + } + } + + private int getDefaultColumnSize() { + + // default to global preference store + IPreferenceStore prefStore = DebugUITools.getPreferenceStore(); + int columnSize = prefStore.getInt(IDebugPreferenceConstants.PREF_COLUMN_SIZE); + // actual column size is number of addressable units * size of the addressable unit + columnSize = columnSize * getAddressableSize(); + + // check synchronized column size + Integer colSize = (Integer)getSynchronizedProperty(AbstractTableRendering.PROPERTY_COL_SIZE); + if (colSize != null) + { + // column size is stored as actual number of bytes in synchronizer + int syncColSize = colSize.intValue(); + if (syncColSize > 0) + { + columnSize = syncColSize; + } + } + else + { + IPersistableDebugElement elmt = (IPersistableDebugElement)getMemoryBlock().getAdapter(IPersistableDebugElement.class); + int defaultColSize = -1; + + if (elmt != null) + { + if (elmt.supportsProperty(this, IDebugPreferenceConstants.PREF_COL_SIZE_BY_MODEL)) + defaultColSize = getDefaultFromPersistableElement(IDebugPreferenceConstants.PREF_COL_SIZE_BY_MODEL); + } + + if (defaultColSize <= 0) + { + // if not provided, get default by model + defaultColSize = getDefaultColumnSizeByModel(getMemoryBlock().getModelIdentifier()); + } + + if (defaultColSize > 0) + columnSize = defaultColSize * getAddressableSize(); + } + return columnSize; + } + + private int getDefaultRowSize() { + + int rowSize = DebugUITools.getPreferenceStore().getInt(IDebugPreferenceConstants.PREF_ROW_SIZE); + int bytePerLine = rowSize * getAddressableSize(); + + // check synchronized row size + Integer size = (Integer)getSynchronizedProperty(AbstractTableRendering.PROPERTY_ROW_SIZE); + if (size != null) + { + // row size is stored as actual number of bytes in synchronizer + int syncRowSize = size.intValue(); + if (syncRowSize > 0) + { + bytePerLine = syncRowSize; + } + } + else + { + int defaultRowSize = -1; + IPersistableDebugElement elmt = (IPersistableDebugElement)getMemoryBlock().getAdapter(IPersistableDebugElement.class); + if (elmt != null) + { + if (elmt.supportsProperty(this, IDebugPreferenceConstants.PREF_ROW_SIZE_BY_MODEL)) + { + defaultRowSize = getDefaultFromPersistableElement(IDebugPreferenceConstants.PREF_ROW_SIZE_BY_MODEL); + return defaultRowSize * getAddressableSize(); + } + } + + if (defaultRowSize <= 0) + // no synchronized property, ask preference store by id + defaultRowSize = getDefaultRowSizeByModel(getMemoryBlock().getModelIdentifier()); + + if (defaultRowSize > 0) + bytePerLine = defaultRowSize * getAddressableSize(); + } + return bytePerLine; + } + + private int getDefaultFromPersistableElement(String propertyId) { + int defaultValue = -1; + IPersistableDebugElement elmt = (IPersistableDebugElement)getMemoryBlock().getAdapter(IPersistableDebugElement.class); + if (elmt != null) + { + try { + Object valueMB = elmt.getProperty(this, propertyId); + if (valueMB != null && !(valueMB instanceof Integer)) + { + IStatus status = DebugUIPlugin.newErrorStatus("Model returned invalid type on " + propertyId, null); //$NON-NLS-1$ + DebugUIPlugin.log(status); + } + + if (valueMB != null) + { + Integer value = (Integer)valueMB; + defaultValue = value.intValue(); + } + } catch (CoreException e) { + DebugUIPlugin.log(e); + } + } + return defaultValue; + } + + private void getPageSizeFromPreference() + { + fPageSize = DebugUIPlugin.getDefault().getPreferenceStore().getInt(IDebugPreferenceConstants.PREF_TABLE_RENDERING_PAGE_SIZE); + } + + private void createCursor(Table table, BigInteger address) + { + fTableCursor = new TableCursor(table, SWT.NONE); + Display display = fTableCursor.getDisplay(); + + // set up cursor color + fTableCursor.setBackground(display.getSystemColor(SWT.COLOR_LIST_SELECTION)); + fTableCursor.setForeground(display.getSystemColor(SWT.COLOR_LIST_SELECTION_TEXT)); + + fTableCursor.setFont(JFaceResources.getFont(IInternalDebugUIConstants.FONT_NAME)); + fTableCursor.setVisible(true); + + fCursorKeyAdapter = new KeyAdapter() { + public void keyPressed(KeyEvent e) + { + handleCursorKeyPressed(e); + } + }; + fTableCursor.addKeyListener(fCursorKeyAdapter); + + fCursorTraverseListener = new TraverseListener() { + public void keyTraversed(TraverseEvent e) { + handleCursorTraverseEvt(e); + }}; + + fTableCursor.addTraverseListener(fCursorTraverseListener); + + fCursorMouseListener = new MouseAdapter() { + public void mouseDown(MouseEvent e) { + handleCursorMouseEvent(e); + }}; + fTableCursor.addMouseListener(fCursorMouseListener); + + // cursor may be disposed before disposed is called + // remove listeners whenever the cursor is disposed + fTableCursor.addDisposeListener(new DisposeListener() { + public void widgetDisposed(DisposeEvent e) { + if (fTableCursor == null) + return; + fTableCursor.removeTraverseListener(fCursorTraverseListener); + fTableCursor.removeKeyListener(fCursorKeyAdapter); + fTableCursor.removeMouseListener(fCursorMouseListener); + fTableCursor.removeSelectionListener(fCursorSelectionListener); + }}); + + fCursorSelectionListener = new SelectionAdapter() { + public void widgetSelected(SelectionEvent e) { + + if (!fEvtHandleLock.acquireLock(this)) + return; + + handleCursorMoved(); + + fEvtHandleLock.releaseLock(this); + + } + }; + fTableCursor.addSelectionListener(fCursorSelectionListener); + + + setCursorAtAddress(address); + + fCursorEditor = new TableEditor (fTableViewer.getTable()); + } + + private void handleCursorTraverseEvt(TraverseEvent e){ + + if (fTableCursor.getRow() == null) + return; + + Table table = (Table)fTableCursor.getParent(); + int row = table.indexOf(fTableCursor.getRow()); + int col = fTableCursor.getColumn(); + if (col == getNumCol() && e.keyCode == SWT.ARROW_RIGHT) + { + if (row + 1>= table.getItemCount()) + { + return; + } + + row = row +1; + col = 0; + fTableCursor.setSelection(row, col); + } + if (col <= 1 && e.keyCode == SWT.ARROW_LEFT) + { + if (row-1 < 0) + { + return; + } + + row = row - 1; + col = getNumCol()+1; + fTableCursor.setSelection(row, col); + } + + Object evtLockClient = new Object(); + if (!fEvtHandleLock.acquireLock(evtLockClient)) + return; + + handleCursorMoved(); + + fEvtHandleLock.releaseLock(evtLockClient); + + } + + /** + * Update selected address. + * Load more memory if required. + */ + private void handleCursorMoved() + { + if (fIsDisposed) + return; + + BigInteger selectedAddress = getSelectedAddressFromCursor(fTableCursor); + + // when the cursor is moved, the selected address is changed + if (selectedAddress != null && !selectedAddress.equals(fSelectedAddress)) + { + setSelectedAddress(selectedAddress); + updateSyncSelectedAddress(); + } + + // now check to see if the cursor is approaching buffer limit + TableItem item = fTableCursor.getRow(); + if (item == null) + return; + + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + int row = fTableViewer.getTable().indexOf(item); + + if (row < 3) + { + if (!isAtTopLimit()) + { + if (isDynamicLoad()) + { + refresh(); + setCursorAtAddress(fSelectedAddress); + } + } + } + else if (row >= fTableViewer.getTable().getItemCount() - 3) + { + if (!isAtBottomLimit()) + { + if (isDynamicLoad()) + { + refresh(); + setCursorAtAddress(fSelectedAddress); + } + } + } + } + + // if the cursor has moved, the top index of the table may change + // just update the synchronization service + BigInteger address = getTopVisibleAddress(); + if (!address.equals(fTopRowAddress)) + { + fTopRowAddress = address; + updateSyncTopAddress(); + } + } + + private void handleCursorKeyPressed(KeyEvent event) + { + // allow edit if user hits return + if (event.character == '\r' && event.getSource() instanceof TableCursor) + { + activateCellEditor(null); + return; + } + + if (MemoryViewUtil.isValidEditEvent(event.keyCode)) + { + // activate edit as soon as user types something at the cursor + if (event.getSource() instanceof TableCursor) + { + String initialValue = String.valueOf(event.character); + activateCellEditor(initialValue); + return; + } + } + } + + /** + * Calculate selected address based on cursor's current position + * @param cursor + * @return the selected address + */ + private BigInteger getSelectedAddressFromCursor(TableCursor cursor) + { + TableItem row = cursor.getRow(); + int col = cursor.getColumn(); + + return getAddressFromTableItem(row, col); + } + + private BigInteger getAddressFromTableItem(TableItem row, int col) { + if (row == null) + return null; + + // get row address + String temp = ((TableRenderingLine)row.getData()).getAddress(); + BigInteger rowAddress = new BigInteger(temp, 16); + + int offset; + if (col > 0) + { + // get address offset + int addressableUnit = getAddressableUnitPerColumn(); + offset = (col-1) * addressableUnit; + } + else + { + offset = 0; + } + + return rowAddress.add(BigInteger.valueOf(offset)); + } + + + /** + * Sets the cursor at the specified address + * @param address + * @return true if successful, false otherwise + */ + private boolean setCursorAtAddress(BigInteger address) + { + // selected address is out of range, simply return false + if (address.compareTo(fContentProvider.getBufferTopAddress()) < 0) + return false; + + // calculate selected row address + int addressableUnit = getAddressableUnitPerLine(); + int numOfRows = address.subtract(fContentProvider.getBufferTopAddress()).intValue()/addressableUnit; + BigInteger rowAddress = fContentProvider.getBufferTopAddress().add(BigInteger.valueOf(numOfRows * addressableUnit)); + + // try to find the row of the selected address + int row = findAddressIndex(address); + + if (row == -1) + { + return false; + } + + // calculate offset to the row address + BigInteger offset = address.subtract(rowAddress); + + // locate column + int colAddressableUnit = getAddressableUnitPerColumn(); + int col = ((offset.intValue()/colAddressableUnit)+1); + + if (col == 0) + col = 1; + + fTableCursor.setSelection(row, col); + + return true; + } + + + /** + * Format view tab based on the bytes per line and column. + * + * @param bytesPerLine - number of bytes per line, possible values: (1 / 2 / 4 / 8 / 16) * addressableSize + * @param columnSize - number of bytes per column, possible values: (1 / 2 / 4 / 8 / 16) * addressableSize + * @return true if format is successful, false, otherwise + */ + public boolean format(int bytesPerLine, int columnSize) + { + + // selected address gets changed as the cursor is moved + // during the reformat. + // Back up the address and restore it later. + BigInteger selectedAddress = fSelectedAddress; + + // bytes per cell must be divisible to bytesPerLine + if (bytesPerLine % columnSize != 0) + { + return false; + } + + if (bytesPerLine < columnSize) + { + return false; + } + + // do not format if the view tab is already in that format + if(fBytePerLine == bytesPerLine && fColumnSize == columnSize){ + return false; + } + + fBytePerLine = bytesPerLine; + fColumnSize = columnSize; + + Object evtLockClient = new Object(); + if (!fEvtHandleLock.acquireLock(evtLockClient)) + return false; + + // if the tab is already created and is being reformatted + if (fIsCreated) + { + if (fTableViewer == null) + return false; + + if (fTableViewer.getTable() == null) + return false; + + // clean up old columns + TableColumn[] oldColumns = fTableViewer.getTable().getColumns(); + + for (int i=0; i= 4) + { + column.setText(Integer.toHexString(i*addressableUnit).toUpperCase() + + " - " + Integer.toHexString(i*addressableUnit+addressableUnit-1).toUpperCase()); //$NON-NLS-1$ + } + else + { + column.setText(Integer.toHexString(i*addressableUnit).toUpperCase()); + } + } + } + + //Empty column for cursor navigation + TableColumn emptyCol = new TableColumn(fTableViewer.getTable(),SWT.LEFT,byteColumns.length+1); + emptyCol.setText(" "); //$NON-NLS-1$ + emptyCol.setWidth(1); + emptyCol.setResizable(false); + + // +2 to include properties for address and navigation column + String[] columnProperties = new String[byteColumns.length+2]; + columnProperties[0] = TableRenderingLine.P_ADDRESS; + + int addressableUnit = columnSize / getAddressableSize(); + + // use column beginning offset to the row address as properties + for (int i=1; i= 0) + setTopIndex(fTableViewer.getTable(), i); + + if (isAddressVisible(selectedAddress)) + // after refresh, make sure the cursor is at the correct position + setCursorAtAddress(selectedAddress); + } + + fEvtHandleLock.releaseLock(evtLockClient); + + return true; + } + + /** + * Create the error page for this rendering. + * The error page is used to report any error resulted from + * getting memory from a memory block. + * @param parent + */ + private void createErrorPage(Composite parent) + { + if (fTextViewer == null) + { + fTextViewer = new TextViewer(parent, SWT.WRAP); + fTextViewer.setDocument(new Document()); + StyledText styleText = fTextViewer.getTextWidget(); + styleText.setEditable(false); + styleText.setEnabled(false); + } + } + + /** + * Displays the content of the table viewer. + */ + public void displayTable() + { + fIsShowingErrorPage = false; + fPageBook.showPage(fTableViewer.getControl()); + } + + /** + * Displays an error message for the given exception. + * + * @param e exception to display + */ + public void displayError(DebugException e) + { + StyledText styleText = null; + fIsShowingErrorPage = true; + + styleText = fTextViewer.getTextWidget(); + + if (styleText != null) + styleText.setText(DebugUIMessages.AbstractTableRendering_3 + e.getMessage()); + fPageBook.showPage(fTextViewer.getControl()); + + // clear content cache if we need to display error + fContentProvider.clearContentCache(); + } + + /** + * Returns whether the error page is displayed. + * + * @return whether the error page is displayed + */ + public boolean isDisplayingError() + { + return fIsShowingErrorPage; + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#getControl() + */ + public Control getControl() { + return fPageBook; + } + + /** + * Returns the addressable size of this rendering's memory block in bytes. + * + * @return the addressable size of this rendering's memory block in bytes + */ + public int getAddressableSize() { + return fAddressableSize; + } + + private Object getSynchronizedProperty(String propertyId) + { + IMemoryRenderingSynchronizationService syncService = getMemoryRenderingContainer().getMemoryRenderingSite().getSynchronizationService(); + + if (syncService == null) + return null; + + return syncService.getProperty(getMemoryBlock(), propertyId); + } + + /** + * This method estimates the number of visible lines in the rendering + * table. + * @return estimated number of visible lines in the table + */ + private int getNumberOfVisibleLines() + { + if(fTableViewer == null) + return -1; + + Table table = fTableViewer.getTable(); + int height = fTableViewer.getTable().getSize().y; + + // when table is not yet created, height is zero + if (height == 0) + { + // make use of the table viewer to estimate table size + height = fTableViewer.getTable().getParent().getSize().y; + } + + // height of border + int border = fTableViewer.getTable().getHeaderHeight(); + + // height of scroll bar + int scroll = fTableViewer.getTable().getHorizontalBar().getSize().y; + + // height of table is table's area minus border and scroll bar height + height = height-border-scroll; + + // calculate number of visible lines + int lineHeight = getMinTableItemHeight(table); + + int numberOfLines = height/lineHeight; + + if (numberOfLines <= 0) + return 20; + + return numberOfLines; + } + + private static void setTopIndex(Table table, int index) + { + table.setTopIndex(index); + } + + private void addRenderingToSyncService() + { + IMemoryRenderingSynchronizationService syncService = getMemoryRenderingContainer().getMemoryRenderingSite().getSynchronizationService(); + + if (syncService == null) + return; + + syncService.addPropertyChangeListener(this, null); + + // we could be in a format error even though the error is not yet displayed + // do not update sync property in this case + if (!isDisplayingError()) + { + if (syncService.getSynchronizationProvider() == null) + syncService.setSynchronizationProvider(this); + + // check if there is already synchronization info available + Object selectedAddress =getSynchronizedProperty( AbstractTableRendering.PROPERTY_SELECTED_ADDRESS); + Object rowSize = getSynchronizedProperty(AbstractTableRendering.PROPERTY_ROW_SIZE); + Object colSize =getSynchronizedProperty( AbstractTableRendering.PROPERTY_COL_SIZE); + Object topAddress =getSynchronizedProperty( AbstractTableRendering.PROPERTY_TOP_ADDRESS); + + if (!isDynamicLoad()) + { + Object pageStartAddress = getSynchronizedProperty(IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS); + if (pageStartAddress == null) + updateSyncPageStartAddress(); + } + + // if info is available, some other view tab has already been + // created + // do not overwrite info in the synchronizer if that's the case + if (selectedAddress == null) { + updateSyncSelectedAddress(); + } + + if (rowSize == null) + { + updateSyncRowSize(); + } + + if (colSize == null) { + updateSyncColSize(); + } + if (topAddress == null) { + updateSyncTopAddress(); + } + } + } + + /** + * Get properties from synchronizer and synchronize settings + */ + private void synchronize() + { + if (!isDynamicLoad()) + { + BigInteger pageStart = (BigInteger)getSynchronizedProperty(IInternalDebugUIConstants.PROPERTY_PAGE_START_ADDRESS); + if (pageStart != null && fContentInput != null && fContentInput.getLoadAddress() != null) + { + if (!fContentInput.getLoadAddress().equals(pageStart)) + handlePageStartAddressChanged(pageStart); + } + else if (pageStart != null) + { + handlePageStartAddressChanged(pageStart); + } + } + + Integer rowSize = (Integer) getSynchronizedProperty(AbstractTableRendering.PROPERTY_ROW_SIZE); + Integer columnSize = (Integer) getSynchronizedProperty(AbstractTableRendering.PROPERTY_COL_SIZE); + BigInteger selectedAddress = (BigInteger)getSynchronizedProperty(AbstractTableRendering.PROPERTY_SELECTED_ADDRESS); + BigInteger topAddress = (BigInteger)getSynchronizedProperty(AbstractTableRendering.PROPERTY_TOP_ADDRESS); + + if (rowSize != null) + { + int rSize = rowSize.intValue(); + if (rSize > 0 && rSize != fBytePerLine) { + rowSizeChanged(rSize); + } + } + + if (columnSize != null) { + int colSize = columnSize.intValue(); + if (colSize > 0 && colSize != fColumnSize) { + columnSizeChanged(colSize); + } + } + if (topAddress != null) { + if (!topAddress.equals(getTopVisibleAddress())) { + if (selectedAddress != null) { + if (!fSelectedAddress.equals(selectedAddress)) { + selectedAddressChanged(selectedAddress); + } + } + topVisibleAddressChanged(topAddress, false); + } + } + if (selectedAddress != null) { + if (selectedAddress.compareTo(fSelectedAddress) != 0) { + selectedAddressChanged(selectedAddress); + } + } + } + + /** + * Resize column to the preferred size. + */ + public void resizeColumnsToPreferredSize() { + // pack columns + Table table = fTableViewer.getTable(); + TableColumn[] columns = table.getColumns(); + + for (int i=0 ;i"; //$NON-NLS-1$ //$NON-NLS-2$ + + return decorateLabel(label); + } + + private void setColumnHeadings() + { + String[] columnLabels = new String[0]; + + IMemoryBlockTablePresentation presentation = getTablePresentationAdapter(); + if (presentation != null) + { + columnLabels = presentation.getColumnLabels(getMemoryBlock(), fBytePerLine, getNumCol()); + } + + // check that column labels returned are not null + if (columnLabels == null) + columnLabels = new String[0]; + + int numByteColumns = fBytePerLine/fColumnSize; + + TableColumn[] columns = fTableViewer.getTable().getColumns(); + + int j=0; + for (int i=1; i= 4) + { + columns[i].setText(Integer.toHexString(j*fColumnSize).toUpperCase() + + " - " + Integer.toHexString(j*fColumnSize+fColumnSize-1).toUpperCase()); //$NON-NLS-1$ + } + else + { + columns[i].setText(Integer.toHexString(j*fColumnSize).toUpperCase()); + } + j++; + } + } + } + + /** + * Refresh the table viewer with the current top visible address. + * Update labels in the memory rendering. + */ + public void refresh() + { + // refresh at start address of this memory block + // address may change if expression is evaluated to a different value + IMemoryBlock mem = getMemoryBlock(); + BigInteger address; + + if (mem instanceof IMemoryBlockExtension) + { + try { + address = ((IMemoryBlockExtension)mem).getBigBaseAddress(); + if (address == null) + { + DebugException e = new DebugException(DebugUIPlugin.newErrorStatus(DebugUIMessages.AbstractTableRendering_10, null)); + displayError(e); + return; + } + updateRenderingLabel(true); + // base address has changed + if (address.compareTo(fContentProvider.getContentBaseAddress()) != 0) + { + // get to new address + setSelectedAddress(address); + updateSyncSelectedAddress(); + + reloadTable(address, true); + + if (!isDynamicLoad()) + { + updateSyncPageStartAddress(); + setTopIndex(fTableViewer.getTable(), 0); + } + + fTopRowAddress = getTopVisibleAddress(); + updateSyncTopAddress(); + + fContentInput.updateContentBaseAddress(); + } + else + { + // reload at top of table + if (isDynamicLoad()) + address = getTopVisibleAddress(); + else + address = fContentInput.getLoadAddress(); + reloadTable(address, true); + } + } catch (DebugException e) { + displayError(e); + return; + } + } + else + { + address = BigInteger.valueOf(mem.getStartAddress()); + reloadTable(address, true); + } + } + + synchronized private void reloadTable(BigInteger topAddress, boolean updateDelta){ + + if (fTableViewer == null) + return; + + try + { + Table table = (Table)fTableViewer.getControl(); + + TableRenderingContentInput input; + if (isDynamicLoad()) + input = new TableRenderingContentInput(this, fContentInput.getPreBuffer(), fContentInput.getPostBuffer(), fContentInput.getDefaultBufferSize(), topAddress, getNumberOfVisibleLines(), updateDelta, null); + else + input = new TableRenderingContentInput(this, fContentInput.getPreBuffer(), fContentInput.getPostBuffer(), fContentInput.getDefaultBufferSize(), topAddress, fPageSize, updateDelta, null); + + fContentInput = input; + fTableViewer.setInput(fContentInput); + + if (isDynamicLoad()) + { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + int topIdx = findAddressIndex(topAddress); + + if (topIdx != -1) + { + setTopIndex(table, topIdx); + } + } + + // cursor needs to be refreshed after reload + if (isAddressVisible(fSelectedAddress)) + setCursorAtAddress(fSelectedAddress); + } + else + { + if (!isAddressOutOfRange(fSelectedAddress)) + { + setCursorAtAddress(fSelectedAddress); + fTableCursor.setVisible(true); + } + else + { + fTableCursor.setVisible(false); + } + } + } + finally + { + } + } + + private BigInteger getTopVisibleAddress() { + + if (fTableViewer == null) + return BigInteger.valueOf(0); + + Table table = fTableViewer.getTable(); + int topIndex = getTopVisibleIndex(table); + + if (topIndex < 1) { topIndex = 0; } + + if (table.getItemCount() > topIndex) + { + TableRenderingLine topItem = (TableRenderingLine)table.getItem(topIndex).getData(); + + String calculatedAddress = null; + if (topItem == null) + { + calculatedAddress = table.getItem(topIndex).getText(); + } + else + { + calculatedAddress = topItem.getAddress(); + } + + BigInteger bigInt = new BigInteger(calculatedAddress, 16); + return bigInt; + } + return BigInteger.valueOf(0); + } + + private int findAddressIndex(BigInteger address) + { + TableItem items[] = fTableViewer.getTable().getItems(); + + for (int i=0; i 0) + { + return i; + } + } + } + + return -1; + } + + private static int getTopVisibleIndex(Table table) + { + int index = table.getTopIndex(); + + TableItem item = table.getItem(index); + int cnt = table.getItemCount(); + + while (item.getBounds(0).y < 0) + { + index++; + if (index >= cnt) + { + index--; + break; + } + item = table.getItem(index); + } + + return index; + } + + /** + * Returns this rendering's table viewer. + */ + public TableViewer getTableViewer() + { + return fTableViewer; + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#dispose() + */ + public void dispose() { + try { + // prevent rendering from being disposed again + if (fIsDisposed) + return; + + fIsDisposed = true; + + if (fContentProvider != null) + fContentProvider.dispose(); + + ScrollBar scroll = ((Table)fTableViewer.getControl()).getVerticalBar(); + if (scroll != null && !scroll.isDisposed()) + scroll.removeSelectionListener(fScrollbarSelectionListener); + + if (!fTableCursor.isDisposed()) + { + fTableCursor.removeTraverseListener(fCursorTraverseListener); + fTableCursor.removeKeyListener(fCursorKeyAdapter); + fTableCursor.removeMouseListener(fCursorMouseListener); + } + + fCursorEditor.dispose(); + + fTextViewer = null; + fTableViewer = null; + fTableCursor = null; + + // clean up cell editors + for (int i=0; i 0) + { + Status stat = new Status( + IStatus.ERROR, DebugUIPlugin.getUniqueIdentifier(), + DebugException.NOT_SUPPORTED, DebugUIMessages.AbstractTableRendering_11, null + ); + DebugException e = new DebugException(stat); + throw e; + } + + setSelectedAddress(address); + updateSyncSelectedAddress(); + + reloadTable(address, false); + + if (!isDynamicLoad()) + { + updateSyncPageStartAddress(); + } + + // if the table is reloaded, the top address is changed in this case + fTopRowAddress = address; + updateSyncTopAddress(); + + // set the cursor at the selected address after reload + setCursorAtAddress(address); + } + fTableCursor.setVisible(true); + } + catch (DebugException e) + { + throw e; + } + finally + { + fEvtHandleLock.releaseLock(evtLockClient); + } + } + + /** + * Check if address provided is out of buffered range + * @param address + * @return if address is out of buffered range + */ + private boolean isAddressOutOfRange(BigInteger address) + { + return fContentProvider.isAddressOutOfRange(address); + } + + /** + * Check if address is visible + * @param address + * @return if the given address is visible + */ + private boolean isAddressVisible(BigInteger address) + { + // if view tab is not yet created + // cursor should always be visible + if (!fIsCreated) + return true; + + BigInteger topVisible = getTopVisibleAddress(); + int addressableUnit = getAddressableUnitPerLine(); + BigInteger lastVisible = getTopVisibleAddress().add(BigInteger.valueOf((getNumberOfVisibleLines() * addressableUnit) + addressableUnit)); + + if (topVisible.compareTo(address) <= 0 && lastVisible.compareTo(address) > 0) + { + return true; + } + return false; + } + + /** + * Create actions for this rendering + */ + protected void createActions() { + fCopyToClipboardAction = new CopyTableRenderingToClipboardAction(this, fTableViewer); + fGoToAddressAction = new GoToAddressAction(this); + fResetMemoryBlockAction = new ResetToBaseAddressAction(this); + fPrintViewTabAction = new PrintTableRenderingAction(this, fTableViewer); + + fFormatRenderingAction = new FormatTableRenderingAction(this); + fReformatAction = new ReformatAction(this); + fToggleAddressColumnAction = new ToggleAddressColumnAction(); + + IMemoryRenderingSite site = getMemoryRenderingContainer().getMemoryRenderingSite(); + if (site.getSite().getSelectionProvider() != null) + { + fPropertiesAction = new PropertyDialogAction(site.getSite(),site.getSite().getSelectionProvider()); + } + + fNextAction = new NextPageAction(); + fPrevAction = new PrevPageAction(); + } + + /** + * Handle scrolling and reload table if necessary + * @param event + */ + private synchronized void handleScrollBarSelection() + { + Object evtLockClient = new Object(); + try + { + if (fIsDisposed) + return; + + BigInteger address = getTopVisibleAddress(); + + if (!fTopRowAddress.equals(address)) + { + fTopRowAddress = address; + updateSyncTopAddress(); + } + + if (!fEvtHandleLock.acquireLock(evtLockClient)) + return; + + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + + if (isDynamicLoad()) + { + if (!isAddressOutOfRange(address)) + { + Table table = fTableViewer.getTable(); + int numInBuffer = table.getItemCount(); + int index = findAddressIndex(address); + if (index < 3) + { + if (isAtTopLimit()) + { + setTopIndex(table, index); + } + else + { + reloadTable(address, false); + } + } + else if ((numInBuffer-(index+getNumberOfVisibleLines())) < 3) + { + if (!isAtBottomLimit()) + reloadTable(address, false); + } + } + else + { + // approaching limit, reload table + reloadTable(address, false); + } + } + + if (isAddressVisible(fSelectedAddress)) + fTableCursor.setVisible(true); + else + fTableCursor.setVisible(false); + } + } + finally + { + fEvtHandleLock.releaseLock(evtLockClient); + } + } + + + private boolean isAtTopLimit() + { + BigInteger startAddress = fContentInput.getStartAddress(); + startAddress = MemoryViewUtil.alignToBoundary(startAddress, getAddressableUnitPerLine() ); + + BigInteger startBufferAddress = fContentProvider.getBufferTopAddress(); + startBufferAddress = MemoryViewUtil.alignToBoundary(startBufferAddress, getAddressableUnitPerLine()); + + if (startAddress.compareTo(startBufferAddress) == 0) + return true; + + return false; + } + + private boolean isAtBottomLimit() + { + BigInteger endAddress = fContentInput.getEndAddress(); + endAddress = MemoryViewUtil.alignToBoundary(endAddress, getAddressableUnitPerLine()); + + BigInteger endBufferAddress = fContentProvider.getBufferEndAddress(); + endBufferAddress = MemoryViewUtil.alignToBoundary(endBufferAddress, getAddressableUnitPerLine()); + + if (endAddress.compareTo(endBufferAddress) == 0) + return true; + + return false; + } + + private boolean needMoreLines() + { + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + Table table = fTableViewer.getTable(); + TableItem firstItem = table.getItem(0); + TableItem lastItem = table.getItem(table.getItemCount()-1); + + if (firstItem == null || lastItem == null) + return true; + + TableRenderingLine first = (TableRenderingLine)firstItem.getData(); + TableRenderingLine last = (TableRenderingLine) lastItem.getData(); + + if (first == null ||last == null) + { + // For some reason, the table does not return the correct number + // of table items in table.getItemCount(), causing last to be null. + // This check is to ensure that we don't get a null pointer exception. + return true; + } + + BigInteger startAddress = new BigInteger(first.getAddress(), 16); + BigInteger lastAddress = new BigInteger(last.getAddress(), 16); + int addressableUnit = getAddressableUnitPerLine(); + lastAddress = lastAddress.add(BigInteger.valueOf(addressableUnit)); + + BigInteger topVisibleAddress = getTopVisibleAddress(); + long numVisibleLines = getNumberOfVisibleLines(); + long numOfBytes = numVisibleLines * addressableUnit; + + BigInteger lastVisibleAddrss = topVisibleAddress.add(BigInteger.valueOf(numOfBytes)); + + // if there are only 3 lines left at the top, refresh + BigInteger numTopLine = topVisibleAddress.subtract(startAddress).divide(BigInteger.valueOf(addressableUnit)); + if (numTopLine.compareTo(BigInteger.valueOf(3)) <= 0 && (startAddress.compareTo(BigInteger.valueOf(0)) != 0)) + { + if (!isAtTopLimit()) + return true; + } + + // if there are only 3 lines left at the bottom, refresh + BigInteger numBottomLine = lastAddress.subtract(lastVisibleAddrss).divide(BigInteger.valueOf(addressableUnit)); + if (numBottomLine.compareTo(BigInteger.valueOf(3)) <= 0) + { + if (!isAtBottomLimit()) + return true; + } + + return false; + } + + return false; + } + + private void handleTableMouseEvent(MouseEvent e) { + // figure out new cursor position based on here the mouse is pointing + TableItem[] tableItems = fTableViewer.getTable().getItems(); + TableItem selectedRow = null; + int colNum = -1; + int numCol = fTableViewer.getColumnProperties().length; + + for (int j=0; j 0 && col <= (getNumCol())) + activateCellEditor(null); + } + } + + /** + * Activate cell editor and pre-fill it with initial value. + * If initialValue is null, use cell content as initial value + * @param initialValue + */ + private void activateCellEditor(String initialValue) { + + int col = fTableCursor.getColumn(); + int row = findAddressIndex(fSelectedAddress); + + if (row < 0) + return; + // do not allow user to edit address column + if (col == 0 || col > getNumCol()) + { + return; + } + + ICellModifier cellModifier = null; + + if (fTableViewer == null) + { + return; + } + cellModifier = fTableViewer.getCellModifier(); + + TableItem tableItem = fTableViewer.getTable().getItem(row); + + Object element = tableItem.getData(); + Object property = fTableViewer.getColumnProperties()[col]; + Object value = cellModifier.getValue(element, (String)property); + + // The cell modifier canModify function always returns false if the edit action + // is not invoked from here. This is to prevent data to be modified when + // the table cursor loses focus from a cell. By default, data will + // be changed in a table when the cell loses focus. This is to workaround + // this default behavior and only change data when the cell editor + // is activated. + ((TableRenderingCellModifier)cellModifier).setEditActionInvoked(true); + boolean canEdit = cellModifier.canModify(element, (String)property); + ((TableRenderingCellModifier)cellModifier).setEditActionInvoked(false); + + if (!canEdit) + return; + + // activate based on current cursor position + TextCellEditor selectedEditor = (TextCellEditor)fTableViewer.getCellEditors()[col]; + + + if (fTableViewer != null && selectedEditor != null) + { + // The control that will be the editor must be a child of the Table + Text text = (Text)selectedEditor.getControl(); + + String cellValue = null; + + if (initialValue != null) + { + cellValue = initialValue; + } + else + { + cellValue = ((String)value); + } + + text.setText(cellValue); + + fCursorEditor.horizontalAlignment = SWT.LEFT; + fCursorEditor.grabHorizontal = true; + + // Open the text editor in selected column of the selected row. + fCursorEditor.setEditor (text, tableItem, col); + + // Assign focus to the text control + selectedEditor.setFocus(); + + if (initialValue != null) + { + text.clearSelection(); + } + + text.setFont(JFaceResources.getFont(IInternalDebugUIConstants.FONT_NAME)); + + // add listeners for the text control + addListeners(text); + + // move cursor below text control + fTableCursor.moveBelow(text); + } + } + + /** + * @param text + */ + private void addListeners(Text text) { + fEditorFocusListener = new FocusAdapter() { + public void focusLost(FocusEvent e) + { + handleTableEditorFocusLost(e); + } + }; + text.addFocusListener(fEditorFocusListener); + + fEditorKeyListener = new KeyAdapter() { + public void keyPressed(KeyEvent e) { + handleKeyEventInEditor(e); + } + }; + + text.addKeyListener(fEditorKeyListener); + } + + /** + * @param text + */ + private void removeListeners(Text text) { + + text.removeFocusListener(fEditorFocusListener); + text.removeKeyListener(fEditorKeyListener); + } + + private void handleTableEditorFocusLost(FocusEvent event) + { + final FocusEvent e = event; + + Display.getDefault().syncExec(new Runnable() { + + public void run() + { + try + { + int row = findAddressIndex(fSelectedAddress); + int col = fTableCursor.getColumn(); + + Text text = (Text)e.getSource(); + removeListeners(text); + + // get new value + String newValue = text.getText(); + + // modify memory at fRow and fCol + modifyValue(row, col, newValue); + + // show cursor after modification is completed + setCursorAtAddress(fSelectedAddress); + fTableCursor.moveAbove(text); + fTableCursor.setVisible(false); + fTableCursor.setVisible(true); + } + catch (NumberFormatException e1) + { + MemoryViewUtil.openError(DebugUIMessages.MemoryViewCellModifier_failure_title, + DebugUIMessages.MemoryViewCellModifier_data_is_invalid, null); + } + } + }); + } + + /** + * @param event + */ + private void handleKeyEventInEditor(KeyEvent event) { + final KeyEvent e = event; + Display.getDefault().asyncExec(new Runnable() + { + public void run() + { + Text text = (Text)e.getSource(); + int row = findAddressIndex(fSelectedAddress); + int col = fTableCursor.getColumn(); + + try + { + switch (e.keyCode) + { + case SWT.ARROW_UP : + + // move text editor box up one row + if (row-1 < 0) + return; + + // modify value for current cell + modifyValue(row, col, text.getText()); + + row--; + + // update cursor location and selection in table + fTableCursor.setSelection(row, col); + handleCursorMoved(); + + // remove listeners when focus is lost + removeListeners(text); + activateCellEditor(null); + break; + case SWT.ARROW_DOWN : + + // move text editor box down one row + + if (row+1 >= fTableViewer.getTable().getItemCount()) + return; + + // modify value for current cell + modifyValue(row, col, text.getText()); + + row++; + + // update cursor location and selection in table + fTableCursor.setSelection(row, col); + handleCursorMoved(); + + // remove traverse listener when focus is lost + removeListeners(text); + activateCellEditor(null); + break; + case 0: + + // if user has entered the max number of characters allowed in a cell, move to next cell + // Extra changes will be used as initial value for the next cell + int numCharsPerByte = getNumCharsPerByte(); + if (numCharsPerByte > 0) + { + if (text.getText().length() > getBytesPerColumn()*numCharsPerByte) + { + String newValue = text.getText(); + text.setText(newValue.substring(0, getBytesPerColumn()*numCharsPerByte)); + + modifyValue(row, col, text.getText()); + + // if cursor is at the end of a line, move to next line + if (col >= getNumCol()) + { + col = 1; + row++; + } + else + { + // move to next column + row++; + } + + // update cursor position and selected address + fTableCursor.setSelection(row, col); + handleCursorMoved(); + + removeListeners(text); + + // activate text editor at next cell + activateCellEditor(newValue.substring(getBytesPerColumn()*numCharsPerByte)); + } + } + break; + case SWT.ESC: + + // if user has pressed escape, do not commit the changes + // that's why "modifyValue" is not called + fTableCursor.setSelection(row, col); + handleCursorMoved(); + + removeListeners(text); + + // cursor needs to have focus to remove focus from cell editor + fTableCursor.setFocus(); + break; + default : + numCharsPerByte = getNumCharsPerByte(); + if (numCharsPerByte > 0) + { + if (text.getText().length()> getBytesPerColumn()* numCharsPerByte) + { + String newValue = text.getText(); + text.setText(newValue.substring(0,getBytesPerColumn()* numCharsPerByte)); + modifyValue(row, col, text.getText()); + // if cursor is at the end of a line, move to next line + if (col >= getNumCol()) + { + col = 1; + row++; + } + else + { + col++; + } + + fTableCursor.setSelection(row, col); + handleCursorMoved(); + + removeListeners(text); + + activateCellEditor(newValue.substring(getBytesPerColumn()*numCharsPerByte)); + } + } + break; + } + } + catch (NumberFormatException e1) + { + MemoryViewUtil.openError(DebugUIMessages.MemoryViewCellModifier_failure_title, + DebugUIMessages.MemoryViewCellModifier_data_is_invalid, null); + + fTableCursor.setSelection(row, col); + handleCursorMoved(); + + removeListeners(text); + } + } + }); + } + + + /** + * Modify value and send new value to debug adapter + * @param row + * @param col + * @param newValue + * @throws NumberFormatException + */ + private void modifyValue(int row, int col, String newValue) throws NumberFormatException + { + if (newValue.length() == 0) + { + // do not do anything if user has not entered anything + return; + } + + TableItem tableItem = fTableViewer.getTable().getItem(row); + + Object property = fTableViewer.getColumnProperties()[col]; + fTableViewer.getCellModifier().modify(tableItem, (String)property, newValue); + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#becomesHidden() + */ + public void becomesHidden() { + + if (isVisible() == false) + { + // super should always be called + super.becomesHidden(); + return; + } + + super.becomesHidden(); + + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + updateRenderingLabel(false); + } + + // once the view tab is disabled, all deltas information becomes invalid. + // reset changed information and recompute if data has really changed when + // user revisits the same tab. + fContentProvider.resetDeltas(); + + } + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IMemoryRendering#becomesVisible() + */ + public void becomesVisible() { + + // do not do anything if already visible + if (isVisible() == true) + { + // super should always be called + super.becomesVisible(); + return; + } + + super.becomesVisible(); + + boolean value = DebugUIPlugin.getDefault().getPreferenceStore().getBoolean(IDebugPreferenceConstants.PREF_DYNAMIC_LOAD_MEM); + if (value != isDynamicLoad()) + // this call will cause a reload + handleDyanicLoadChanged(); + else + refresh(); + + synchronize(); + updateRenderingLabel(true); + } + + /** + * Resets this memory rendering. + * The cursor will be moved to the base address of the memory block. + * The table will be positioned to have the base address + * at the top. + * + * @deprecated use resetRendering to reset this rendering. + */ + public void reset() + { + try { + resetToBaseAddress(); + } catch (DebugException e) { + MemoryViewUtil.openError(DebugUIMessages.AbstractTableRendering_12, DebugUIMessages.AbstractTableRendering_13, e); // + } + } + + /** + * Reset this rendering to the base address. + * The cursor will be moved to the base address of the memory block. + * The table will be positioned to have the base address + * at the top. + * @throws DebugException + */ + private void resetToBaseAddress() throws DebugException + { + BigInteger baseAddress; + + if (getMemoryBlock() instanceof IMemoryBlockExtension) + { + baseAddress = ((IMemoryBlockExtension)getMemoryBlock()).getBigBaseAddress(); + } + else + { + baseAddress = BigInteger.valueOf(getMemoryBlock().getStartAddress()); + } + + goToAddress(baseAddress); + topVisibleAddressChanged(baseAddress, true); + } + + /** + * Returns the currently selected address in this rendering. + * + * @return the currently selected address in this rendering + */ + public BigInteger getSelectedAddress() { + return fSelectedAddress; + } + + /** + * Returns the currently selected content in this rendering as a String. + * + * @return the currently selected content in this rendering + */ + public String getSelectedAsString() { + + if (isAddressOutOfRange(fSelectedAddress)) + return ""; //$NON-NLS-1$ + + int col = fTableCursor.getColumn(); + TableItem rowItem = fTableCursor.getRow(); + int row = fTableViewer.getTable().indexOf(rowItem); + + if (col == 0) + { + return rowItem.getText(0); + } + + // check precondition + if (col > getBytesPerLine()/getBytesPerColumn()) + { + return ""; //$NON-NLS-1$ + } + + TableItem tableItem = getTableViewer().getTable().getItem(row); + + return tableItem.getText(col); + } + + /** + * Returns the currently selected content in this rendering as MemoryByte. + * + * @return the currently selected content in array of MemoryByte. + * Returns an empty array if the selected address is out of buffered range. + */ + public MemoryByte[] getSelectedAsBytes() + { + if (isAddressOutOfRange(fSelectedAddress)) + return new MemoryByte[0]; + + int col = fTableCursor.getColumn(); + TableItem rowItem = fTableCursor.getRow(); + + // check precondition + if (col == 0 || col > getBytesPerLine()/getBytesPerColumn()) + { + return new MemoryByte[0]; + } + + Object data = rowItem.getData(); + if (data == null || !(data instanceof TableRenderingLine)) + return new MemoryByte[0]; + + TableRenderingLine line = (TableRenderingLine)data; + int offset = (col-1)*(getAddressableUnitPerColumn()*getAddressableSize()); + int end = offset + (getAddressableUnitPerColumn()*getAddressableSize()); + + // make a copy of the bytes to ensure that data cannot be changed + // by caller + MemoryByte[] bytes = line.getBytes(offset, end); + MemoryByte[] retBytes = new MemoryByte[bytes.length]; + + System.arraycopy(bytes, 0, retBytes, 0, bytes.length); + + return retBytes; + } + + /** + * Returns the number of characters a byte will convert to + * or -1 if unknown. + * + * @return the number of characters a byte will convert to + * or -1 if unknown + */ + public int getNumCharsPerByte() + { + return -1; + } + + private int getMinTableItemHeight(Table table){ + + // Hack to get around Linux GTK problem. + // On Linux GTK, table items have variable item height as + // carriage returns are actually shown in a cell. Some rows will be + // taller than others. When calculating number of visible lines, we + // need to find the smallest table item height. Otherwise, the rendering + // underestimates the number of visible lines. As a result the rendering + // will not be able to get more memory as needed. + if (MemoryViewUtil.isLinuxGTK()) + { + // check each of the items and find the minimum + TableItem[] items = table.getItems(); + int minHeight = table.getItemHeight(); + for (int i=0; inull if none. + *

+ * By default a color provider is obtained by asking this rendering's + * memory block for its {@link IColorProvider} adapter. When the color + * provider is queried for color information, it is provided with a + * {@link MemoryRenderingElement} as an argument. + *

+ * @return the color provider for this rendering's memory block, + * or null + */ + protected IColorProvider getColorProviderAdapter() + { + return (IColorProvider)getMemoryBlock().getAdapter(IColorProvider.class); + } + + /** + * Returns the label provider for this rendering's memory block or + * null if none. + *

+ * By default a label provider is obtained by asking this rendering's + * memory block for its {@link ILabelProvider} adapter. When the label + * provider is queried for label information, it is provided with a + * {@link MemoryRenderingElement} as an argument. + *

+ * @return the label provider for this rendering's memory block, + * or null + */ + protected ILabelProvider getLabelProviderAdapter() + { + return (ILabelProvider)getMemoryBlock().getAdapter(ILabelProvider.class); + } + + /** + * Returns the font provider for this rendering's memory block or + * null if none. + *

+ * By default a font provider is obtained by asking this rendering's + * memory block for its {@link IFontProvider} adapter. When the font + * provider is queried for font information, it is provided with a + * {@link MemoryRenderingElement} as an argument. + *

+ * @return the font provider for this rendering's memory block, + * or null + */ + protected IFontProvider getFontProviderAdapter() + { + return (IFontProvider)getMemoryBlock().getAdapter(IFontProvider.class); + } + + /** + * Returns the table presentation for this rendering's memory block or + * null if none. + *

+ * By default a table presentation is obtained by asking this rendering's + * memory block for its {@link IMemoryBlockTablePresentation} adapter. + *

+ * @return the table presentation for this rendering's memory block, + * or null + */ + protected IMemoryBlockTablePresentation getTablePresentationAdapter() + { + return (IMemoryBlockTablePresentation)getMemoryBlock().getAdapter(IMemoryBlockTablePresentation.class); + } + + private boolean isDynamicLoad() + { + return fContentProvider.isDynamicLoad(); + } + + private int getPageSizeInUnits() + { + return fPageSize * getAddressableUnitPerLine(); + } + + private void setSelectedAddress(BigInteger address) + { + fSelectedAddress = address; + } + + /** + * Setup the viewer so it supports hovers to show the offset of each field + */ + private void createToolTip() { + + fToolTipShell = new Shell(DebugUIPlugin.getShell(), SWT.ON_TOP | SWT.RESIZE ); + GridLayout gridLayout = new GridLayout(); + gridLayout.numColumns = 1; + gridLayout.marginWidth = 2; + gridLayout.marginHeight = 0; + fToolTipShell.setLayout(gridLayout); + fToolTipShell.setBackground(fTableViewer.getTable().getDisplay().getSystemColor(SWT.COLOR_INFO_BACKGROUND)); + + final Control toolTipControl = createToolTipControl(fToolTipShell); + + if (toolTipControl == null) + { + // if client decide not to use tooltip support + fToolTipShell.dispose(); + return; + } + + MouseTrackAdapter listener = new MouseTrackAdapter(){ + + private TableItem fTooltipItem = null; + private int fCol = -1; + + public void mouseExit(MouseEvent e){ + + if (!fToolTipShell.isDisposed()) + fToolTipShell.setVisible(false); + fTooltipItem = null; + } + + public void mouseHover(MouseEvent e){ + + Point hoverPoint = new Point(e.x, e.y); + Control control = null; + + if (e.widget instanceof Control) + control = (Control)e.widget; + + if (control == null) + return; + + hoverPoint = control.toDisplay(hoverPoint); + TableItem item = getItem(hoverPoint); + int column = getColumn(hoverPoint); + + //Only if there is a change in hover + if(this.fTooltipItem != item || fCol != column){ + + //Keep Track of the latest hover + fTooltipItem = item; + fCol = column; + + if(item != null){ + toolTipAboutToShow(toolTipControl, fTooltipItem, column); + + //Setting location of the tooltip + Rectangle shellBounds = fToolTipShell.getBounds(); + shellBounds.x = hoverPoint.x; + shellBounds.y = hoverPoint.y + item.getBounds(0).height; + + fToolTipShell.setBounds(shellBounds); + fToolTipShell.pack(); + + fToolTipShell.setVisible(true); + } + else { + fToolTipShell.setVisible(false); + } + } + } + }; + + fTableViewer.getTable().addMouseTrackListener(listener); + fTableCursor.addMouseTrackListener(listener); + } + + /** + * Bug with table widget,BUG 113015, the widget is not able to return the correct + * table item if SWT.FULL_SELECTION is not on when the table is created. + * Created the following function to work around the problem. + * We can remove this method when the bug is fixed. + * @param point + * @return the table item where the point is located, return null if the item cannot be located. + */ + private TableItem getItem(Point point) + { + TableItem[] items = fTableViewer.getTable().getItems(); + for (int i=0; i point.x) + return i; + } + return -1; + } + + /** + * Creates the control used to display tool tips for cells in this table. By default + * a label is used to display the address of the cell. Clients may override this + * method to create custom tooltip controls. + *

+ * Also see the methods getToolTipText(...) and + * toolTipAboutToShow(...). + *

+ * @param composite parent for the tooltip control + * @return the tooltip control to be displayed + * @since 3.2 + */ + protected Control createToolTipControl(Composite composite) { + Control fToolTipLabel = new Label(composite, SWT.NONE); + fToolTipLabel.setForeground(fTableViewer.getTable().getDisplay().getSystemColor(SWT.COLOR_INFO_FOREGROUND)); + fToolTipLabel.setBackground(fTableViewer.getTable().getDisplay().getSystemColor(SWT.COLOR_INFO_BACKGROUND)); + fToolTipLabel.setLayoutData(new GridData(GridData.FILL_HORIZONTAL | + GridData.VERTICAL_ALIGN_CENTER)); + return fToolTipLabel; + } + + + /* (non-Javadoc) + * @see org.eclipse.debug.ui.memory.IResettableMemoryRendering#resetRendering() + */ + public void resetRendering() throws DebugException { + resetToBaseAddress(); + } + + /** + * Called when the tool tip is about to show in this rendering. + * Clients who overrides createTooltipControl may need to + * also override this method to ensure that the tooltip shows up properly + * in their customized control. + *

+ * By default a text tooltip is displayed, and the contents for the tooltip + * are generated by the getToolTipText(...) method. + *

+ * @param toolTipControl - the control for displaying the tooltip + * @param item - the table item where the mouse is pointing. + * @param col - the column at which the mouse is pointing. + * @since 3.2 + */ + protected void toolTipAboutToShow(Control toolTipControl, TableItem item, + int col) { + if (toolTipControl instanceof Label) { + BigInteger address = getAddressFromTableItem(item, col); + if (address != null) { + Object data = item.getData(); + if (data instanceof TableRenderingLine) { + TableRenderingLine line = (TableRenderingLine) data; + + if (col > 0) { + int start = (col - 1) * getBytesPerColumn(); + int end = start + getBytesPerColumn(); + MemoryByte[] bytes = line.getBytes(start, end); + + String str = getToolTipText(address, bytes); + + if (str != null) + ((Label) toolTipControl).setText(str); + } else { + String str = getToolTipText(address, + new MemoryByte[] {}); + + if (str != null) + ((Label) toolTipControl).setText(str); + } + } + } + } + } + + /** + * Returns the text to display in a tool tip at the specified address + * for the specified bytes. By default the address of the bytes is displayed. + * Subclasses may override. + * + * @param address address of cell that tool tip is displayed for + * @param bytes the bytes in the cell + * @return the tooltip text for the memory bytes located at the specified + * address + * @since 3.2 + */ + protected String getToolTipText(BigInteger address, MemoryByte[] bytes) + { + StringBuffer buf = new StringBuffer("0x"); //$NON-NLS-1$ + buf.append(address.toString(16).toUpperCase()); + + return buf.toString(); + } + + + private String getRowPrefId(String modelId) { + String rowPrefId = IDebugPreferenceConstants.PREF_ROW_SIZE + ":" + modelId; //$NON-NLS-1$ + return rowPrefId; + } + + private String getColumnPrefId(String modelId) { + String colPrefId = IDebugPreferenceConstants.PREF_COLUMN_SIZE + ":" + modelId; //$NON-NLS-1$ + return colPrefId; + } + + /** + * @param modelId + * @return default number of addressable units per line for the model + */ + private int getDefaultRowSizeByModel(String modelId) + { + int row = DebugUITools.getPreferenceStore().getInt(getRowPrefId(modelId)); + if (row == 0) + { + DebugUITools.getPreferenceStore().setValue(getRowPrefId(modelId), IDebugPreferenceConstants.PREF_ROW_SIZE_DEFAULT); + } + + row = DebugUITools.getPreferenceStore().getInt(getRowPrefId(modelId)); + return row; + + } + + /** + * @param modelId + * @return default number of addressable units per column for the model + */ + private int getDefaultColumnSizeByModel(String modelId) + { + int col = DebugUITools.getPreferenceStore().getInt(getColumnPrefId(modelId)); + if (col == 0) + { + DebugUITools.getPreferenceStore().setValue(getColumnPrefId(modelId), IDebugPreferenceConstants.PREF_COLUMN_SIZE_DEFAULT); + } + + col = DebugUITools.getPreferenceStore().getInt(getColumnPrefId(modelId)); + return col; + } + + + /** + * Returns text for the given memory bytes at the specified address for the specified + * rendering type. This is called by the label provider for. + * Subclasses must override. + * + * @param renderingTypeId rendering type identifier + * @param address address where the bytes belong to + * @param data the bytes + * @return a string to represent the memory. Cannot not return null. + * Returns a string to pad the cell if the memory cannot be converted + * successfully. + */ + abstract public String getString(String renderingTypeId, BigInteger address, MemoryByte[] data); + + /** + * Returns bytes for the given text corresponding to bytes at the given + * address for the specified rendering type. This is called by the cell modifier + * when modifying bytes in a memory block. + * Subclasses must convert the string value to an array of bytes. The bytes will + * be passed to the debug adapter for memory block modification. + * Returns null if the bytes cannot be formatted properly. + * + * @param renderingTypeId rendering type identifier + * @param address address the bytes begin at + * @param currentValues current values of the data in bytes format + * @param newValue the string to be converted to bytes + * @return the bytes converted from a string + */ + abstract public byte[] getBytes(String renderingTypeId, BigInteger address, MemoryByte[] currentValues, String newValue); + + +} + diff --git a/samples/vectors/sample-vectors_100_0 b/samples/vectors/sample-vectors_100_0 new file mode 100644 index 0000000..c7b63bd --- /dev/null +++ b/samples/vectors/sample-vectors_100_0 @@ -0,0 +1,188 @@ +# FILE:src/AbstractAsyncTableRendering.java, LINE:173, OFFSET:3064, NODE_KIND:128, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:29472, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:1539, TEID:12881, VARs:{}740, +472 0 10 0 0 0 0 0 0 0 0 0 1 0 0 0 0 8 0 0 0 0 0 0 0 0 0 0 0 0 0 3479 0 0 0 0 1706 0 0 0 0 0 0 0 0 0 1411 0 0 0 12 0 0 0 286 0 0 131 0 0 1667 17 0 7 0 0 0 0 0 0 7 0 0 220 0 0 0 0 0 0 1280 0 341 34 0 0 0 1101 0 0 187 0 0 0 0 156 1 0 1598 0 0 0 0 996 902 0 0 0 1634 0 0 0 0 0 1553 0 0 0 0 0 0 0 58 0 228 273 0 0 1 0 0 0 0 0 0 52 0 0 0 0 0 0 0 0 21 1635 0 0 6 0 7 0 0 1411 0 0 41 0 0 2 386 0 80 8 36 65 134 221 0 2 0 0 12 0 0 28 4 15 3 21 8 0 54 0 0 806 100 0 1 0 0 37 0 0 0 5 18 3 0 34 0 0 0 0 0 44 84 28 14 0 0 0 0 10 0 2 0 12 7 1 0 95 10 0 42 0 0 0 0 0 0 151 2 21 0 0 117 11 4 0 14 0 0 1 46 2 0 12 3479 0 0 1 24 44 0 25 0 0 6 23 107 +# FILE:src/AbstractAsyncTableRendering.java, LINE:845, OFFSET:890, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:592, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:4144, TEID:4354, VARs:{}26, +16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 71 0 0 0 0 38 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 7 0 0 6 0 0 36 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 26 0 3 1 0 0 0 22 0 0 1 0 0 0 0 0 0 0 32 0 0 0 0 20 24 0 0 0 32 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 32 0 0 4 0 0 0 13 0 0 0 0 0 0 2 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 16 0 0 0 0 0 2 0 0 0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 71 0 0 0 0 0 0 0 0 0 0 0 2 +# FILE:src/AbstractAsyncTableRendering.java, LINE:991, OFFSET:1009, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:410, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:4736, TEID:4876, VARs:{}18, +5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 47 0 0 0 0 27 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 5 4 0 0 0 9 0 0 1 0 0 0 0 0 0 0 27 0 0 0 0 11 12 0 0 0 32 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 0 5 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 12 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 47 0 0 0 0 0 0 0 0 0 0 5 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:995, OFFSET:1009, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:334, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:4762, TEID:4876, VARs:{}16, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 38 0 0 0 0 22 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 4 3 0 0 0 8 0 0 1 0 0 0 0 0 0 0 22 0 0 0 0 9 10 0 0 0 26 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 0 0 0 4 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 38 0 0 0 0 0 0 0 0 0 0 4 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1040, OFFSET:1063, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:530, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:4980, TEID:5175, VARs:{}29, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 67 0 0 0 0 32 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 1 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 6 5 0 0 0 14 0 0 1 0 0 0 0 0 0 0 32 0 0 0 0 13 18 0 0 0 37 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 27 0 0 0 0 0 0 1 0 0 0 5 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 67 0 0 0 0 0 0 0 0 0 0 5 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1044, OFFSET:1063, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:436, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5014, TEID:5175, VARs:{}27, +5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 56 0 0 0 0 26 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 1 0 0 0 0 0 26 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 5 4 0 0 0 11 0 0 1 0 0 0 0 0 0 0 26 0 0 0 0 11 15 0 0 0 30 0 0 0 0 0 26 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 26 0 0 0 0 0 0 0 22 0 0 0 0 0 0 1 0 0 0 4 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 56 0 0 0 0 0 0 0 0 0 0 4 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1049, OFFSET:1063, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:342, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5048, TEID:5175, VARs:{}23, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 0 0 0 20 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 1 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 4 3 0 0 0 8 0 0 1 0 0 0 0 0 0 0 20 0 0 0 0 9 12 0 0 0 23 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 17 0 0 0 0 0 0 1 0 0 0 3 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 0 0 0 0 0 0 0 0 0 3 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1053, OFFSET:1063, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:263, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5076, TEID:5175, VARs:{}21, +3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 0 0 0 15 0 0 0 0 0 0 0 0 0 13 0 0 0 0 0 0 0 1 0 0 0 0 0 15 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 0 3 2 0 0 0 7 0 0 1 0 0 0 0 0 0 0 15 0 0 0 0 7 10 0 0 0 17 0 0 0 0 0 15 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 15 0 0 0 0 0 0 0 13 0 0 0 0 0 0 1 0 0 0 2 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 0 0 0 0 0 0 0 0 0 2 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1187, OFFSET:1214, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:364, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5638, TEID:5757, VARs:{}16, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 25 0 0 0 0 0 0 0 0 0 19 0 0 0 0 0 0 0 3 0 0 4 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 5 1 0 0 0 17 0 0 5 0 0 0 0 0 0 0 24 0 0 0 0 14 9 0 0 0 24 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 19 0 0 0 0 0 0 7 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 1 0 2 0 0 7 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 0 0 0 0 0 0 1 0 0 4 0 0 0 2 0 0 0 0 0 0 0 32 0 0 0 1 0 0 0 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1226, OFFSET:1253, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:384, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5806, TEID:5932, VARs:{}16, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 34 0 0 0 0 27 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 3 0 0 4 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 5 1 0 0 0 18 0 0 5 0 0 0 0 0 0 0 25 0 0 0 0 15 10 0 0 0 25 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 20 0 0 0 0 0 0 7 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 1 0 2 0 0 7 2 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 0 0 0 0 1 0 1 0 0 4 0 0 0 2 0 0 0 0 0 0 0 34 0 0 0 1 0 0 0 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1756, OFFSET:1781, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:380, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:7812, TEID:7951, VARs:{}17, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 0 0 0 24 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 1 0 0 6 0 0 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 0 6 1 0 0 0 23 0 0 3 0 0 0 0 0 0 0 23 0 0 0 0 14 14 0 0 0 24 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 20 0 0 3 0 0 0 5 0 0 0 1 0 0 4 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 10 2 0 0 0 0 0 0 0 0 0 0 2 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 0 0 1 0 0 0 35 0 0 0 0 0 0 1 0 0 0 1 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1922, OFFSET:1947, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:567, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8408, TEID:8579, VARs:{}26, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 54 0 0 0 0 43 0 0 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 7 0 0 1 0 0 42 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 2 0 0 0 0 24 0 0 3 0 0 0 0 0 0 0 37 0 0 0 0 10 12 0 0 0 38 0 0 0 0 0 35 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 2 38 0 0 0 0 0 0 0 33 0 0 0 0 0 0 8 0 0 0 1 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 8 1 0 0 0 0 1 0 0 0 0 3 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 3 0 0 0 0 0 0 0 0 0 3 0 1 0 0 6 0 0 0 1 0 0 0 0 0 0 1 54 0 0 0 0 5 0 0 0 0 0 0 3 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1924, OFFSET:1947, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:544, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8414, TEID:8579, VARs:{}26, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 53 0 0 0 0 41 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 7 0 0 1 0 0 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 0 2 0 0 0 0 23 0 0 2 0 0 0 0 0 0 0 35 0 0 0 0 9 12 0 0 0 36 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 2 36 0 0 0 0 0 0 0 32 0 0 0 0 0 0 8 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 8 0 0 0 0 0 1 0 0 0 0 3 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 3 0 0 0 0 0 0 0 0 0 3 0 1 0 0 6 0 0 0 1 0 0 0 0 0 0 1 53 0 0 0 0 5 0 0 0 0 0 0 3 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1986, OFFSET:2032, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:601, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8681, TEID:8877, VARs:{}30, +8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 68 0 0 0 0 41 0 0 0 0 0 0 0 0 0 35 0 0 0 0 0 0 0 6 0 0 0 0 0 39 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 5 1 0 0 0 19 0 0 3 0 0 0 0 0 0 0 39 0 0 0 0 18 17 0 0 0 40 0 0 0 0 0 36 0 0 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 0 0 0 0 0 0 0 35 0 0 0 0 0 2 6 0 0 0 1 0 0 4 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 14 2 0 0 0 0 2 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 1 0 0 0 0 0 0 0 68 0 0 0 0 0 0 0 0 0 0 0 4 +# FILE:src/AbstractAsyncTableRendering.java, LINE:1997, OFFSET:2032, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:524, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8709, TEID:8877, VARs:{}28, +5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 60 0 0 0 0 36 0 0 0 0 0 0 0 0 0 31 0 0 0 0 0 0 0 6 0 0 0 0 0 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 0 4 0 0 0 0 17 0 0 3 0 0 0 0 0 0 0 34 0 0 0 0 14 13 0 0 0 35 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 34 0 0 0 0 0 0 0 31 0 0 0 0 0 2 6 0 0 0 1 0 0 3 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 13 2 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 1 0 0 0 0 0 0 0 60 0 0 0 0 0 0 0 0 0 0 0 4 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2312, OFFSET:2370, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:691, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9854, TEID:10107, VARs:{}29, +16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 74 0 0 0 0 42 0 0 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 7 0 0 5 0 0 41 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 28 0 8 2 0 0 0 26 0 0 9 0 0 0 0 0 0 0 41 0 0 0 0 33 25 0 0 0 42 0 0 0 0 0 38 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 41 0 0 0 0 0 0 0 33 0 0 0 0 0 0 12 0 0 0 1 3 0 11 0 0 0 0 0 0 0 1 0 0 0 0 0 0 5 0 0 17 5 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 3 0 0 0 0 0 0 0 74 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2315, OFFSET:2370, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:668, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9860, TEID:10107, VARs:{}29, +16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 73 0 0 0 0 40 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 7 0 0 5 0 0 39 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 8 2 0 0 0 25 0 0 8 0 0 0 0 0 0 0 39 0 0 0 0 32 25 0 0 0 40 0 0 0 0 0 36 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 0 0 0 0 0 0 0 32 0 0 0 0 0 0 12 0 0 0 1 3 0 10 0 0 0 0 0 0 0 1 0 0 0 0 0 0 4 0 0 17 4 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 3 0 0 0 0 0 0 0 73 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2339, OFFSET:2366, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:358, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9965, TEID:10098, VARs:{}16, +7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 41 0 0 0 0 22 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 3 0 0 1 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 5 2 0 0 0 12 0 0 4 0 0 0 0 0 0 0 21 0 0 0 0 16 13 0 0 0 22 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 17 0 0 0 0 0 0 4 0 0 0 1 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 11 3 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 41 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2672, OFFSET:2682, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:398, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11084, TEID:11213, VARs:{}18, +1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 47 0 0 0 0 26 0 0 0 0 0 0 0 0 0 19 0 0 0 1 0 0 0 4 0 0 1 0 0 26 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 2 0 0 0 0 20 0 0 2 0 0 0 0 0 0 0 24 0 0 0 0 7 7 0 0 0 24 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 26 0 0 0 0 0 0 0 19 0 0 0 0 0 0 5 0 0 1 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 3 0 0 1 0 0 18 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 47 0 0 0 0 1 0 2 0 0 0 1 2 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2674, OFFSET:2682, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:352, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11100, TEID:11213, VARs:{}17, +1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 42 0 0 0 0 23 0 0 0 0 0 0 0 0 0 17 0 0 0 1 0 0 0 3 0 0 1 0 0 23 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 2 0 0 0 0 19 0 0 2 0 0 0 0 0 0 0 21 0 0 0 0 6 7 0 0 0 21 0 0 0 0 0 19 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 17 0 0 0 0 0 0 4 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 17 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 42 0 0 0 0 1 0 2 0 0 0 1 2 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2675, OFFSET:2682, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:317, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11109, TEID:11212, VARs:{}14, +1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 38 0 0 0 0 21 0 0 0 0 0 0 0 0 0 15 0 0 0 0 0 0 0 2 0 0 1 0 0 21 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 0 1 0 0 0 0 18 0 0 2 0 0 0 0 0 0 0 19 0 0 0 0 5 7 0 0 0 19 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 15 0 0 0 0 0 0 3 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 17 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 38 0 0 0 0 0 0 2 0 0 0 1 2 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2705, OFFSET:2714, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:357, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11293, TEID:11408, VARs:{}15, +1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 0 0 0 24 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 3 0 0 1 0 0 24 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 2 0 0 0 0 14 0 0 2 0 0 0 0 0 0 0 22 0 0 0 0 7 6 0 0 0 22 0 0 0 0 0 19 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 17 0 0 0 0 0 0 4 0 0 1 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 16 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 45 0 0 0 0 0 0 2 0 0 0 1 2 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2707, OFFSET:2714, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:334, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11299, TEID:11408, VARs:{}15, +1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 0 22 0 0 0 0 0 0 0 0 0 16 0 0 0 0 0 0 0 3 0 0 1 0 0 22 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 2 0 0 0 0 13 0 0 1 0 0 0 0 0 0 0 20 0 0 0 0 6 6 0 0 0 20 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 16 0 0 0 0 0 0 4 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 44 0 0 0 0 0 0 2 0 0 0 1 2 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2707, OFFSET:2714, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:332, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11300, TEID:11407, VARs:{}15, +1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 0 22 0 0 0 0 0 0 0 0 0 16 0 0 0 0 0 0 0 3 0 0 1 0 0 22 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 1 0 0 0 0 13 0 0 1 0 0 0 0 0 0 0 20 0 0 0 0 5 6 0 0 0 20 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 16 0 0 0 0 0 0 4 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 44 0 0 0 0 0 0 2 0 0 0 1 2 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2736, OFFSET:2757, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:577, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11428, TEID:11607, VARs:{}25, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 59 0 0 0 0 41 0 0 0 0 0 0 0 0 0 33 0 0 0 0 0 0 0 8 0 0 0 0 0 41 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 5 1 0 0 0 18 0 0 5 0 0 0 0 0 0 0 38 0 0 0 0 12 11 0 0 0 41 0 0 0 0 0 37 0 0 0 0 0 0 0 0 0 8 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 2 40 0 0 0 0 0 0 0 33 0 0 0 0 0 0 8 0 0 0 3 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 7 3 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 59 0 0 0 0 2 0 1 0 0 0 1 1 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2737, OFFSET:2757, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:559, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11434, TEID:11607, VARs:{}25, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 57 0 0 0 0 40 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 8 0 0 0 0 0 40 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 5 1 0 0 0 18 0 0 4 0 0 0 0 0 0 0 37 0 0 0 0 11 11 0 0 0 39 0 0 0 0 0 36 0 0 0 0 0 0 0 0 0 8 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 2 39 0 0 0 0 0 0 0 32 0 0 0 0 0 0 8 0 0 0 2 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 7 3 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 57 0 0 0 0 2 0 1 0 0 0 1 1 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2738, OFFSET:2757, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:491, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11456, TEID:11606, VARs:{}21, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 48 0 0 0 0 36 0 0 0 0 0 0 0 0 0 28 0 0 0 0 0 0 0 7 0 0 0 0 0 36 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 28 0 4 1 0 0 0 15 0 0 4 0 0 0 0 0 0 0 33 0 0 0 0 10 8 0 0 0 35 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 2 35 0 0 0 0 0 0 0 28 0 0 0 0 0 0 7 0 0 0 2 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 4 3 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 48 0 0 0 0 2 0 1 0 0 0 1 1 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2739, OFFSET:2757, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:455, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11466, TEID:11606, VARs:{}21, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 0 0 0 33 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 7 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 4 1 0 0 0 14 0 0 3 0 0 0 0 0 0 0 30 0 0 0 0 9 8 0 0 0 31 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 2 32 0 0 0 0 0 0 0 27 0 0 0 0 0 0 7 0 0 0 1 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 4 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 45 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2740, OFFSET:2757, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:430, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11476, TEID:11605, VARs:{}18, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 41 0 0 0 0 32 0 0 0 0 0 0 0 0 0 26 0 0 0 0 0 0 0 6 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 26 0 3 1 0 0 0 13 0 0 3 0 0 0 0 0 0 0 29 0 0 0 0 8 7 0 0 0 30 0 0 0 0 0 28 0 0 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 2 31 0 0 0 0 0 0 0 26 0 0 0 0 0 0 6 0 0 0 1 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 41 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2741, OFFSET:2757, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:412, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11482, TEID:11605, VARs:{}18, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 0 0 0 0 31 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 6 0 0 0 0 0 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 0 3 1 0 0 0 13 0 0 2 0 0 0 0 0 0 0 28 0 0 0 0 7 7 0 0 0 28 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 2 30 0 0 0 0 0 0 0 25 0 0 0 0 0 0 6 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 39 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2743, OFFSET:2757, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:380, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11491, TEID:11604, VARs:{}16, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 0 0 0 29 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 5 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 2 1 0 0 0 13 0 0 2 0 0 0 0 0 0 0 26 0 0 0 0 6 7 0 0 0 26 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 2 28 0 0 0 0 0 0 0 23 0 0 0 0 0 0 5 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 35 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2802, OFFSET:2824, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:360, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11751, TEID:11881, VARs:{}12, +5 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 40 0 0 0 0 26 0 0 0 0 0 0 0 0 0 15 0 0 0 4 0 0 0 2 0 0 0 0 0 26 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 5 2 0 0 0 16 0 0 0 0 0 0 0 0 0 0 18 0 0 0 0 11 10 0 0 0 18 0 0 0 0 0 16 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 15 0 0 1 0 0 0 2 0 0 1 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 11 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 2 3 0 0 0 0 0 0 0 0 0 0 2 0 0 4 0 0 0 0 0 0 0 1 0 0 0 40 0 0 0 0 4 0 3 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2806, OFFSET:2824, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:308, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11769, TEID:11881, VARs:{}12, +5 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 0 0 0 22 0 0 0 0 0 0 0 0 0 13 0 0 0 4 0 0 0 1 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 0 5 2 0 0 0 14 0 0 0 0 0 0 0 0 0 0 15 0 0 0 0 10 10 0 0 0 15 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 19 0 0 0 0 0 0 0 13 0 0 1 0 0 0 1 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 1 0 0 0 35 0 0 0 0 4 0 3 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2806, OFFSET:2824, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:306, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11770, TEID:11880, VARs:{}12, +5 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 0 0 0 22 0 0 0 0 0 0 0 0 0 13 0 0 0 4 0 0 0 1 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 0 4 2 0 0 0 14 0 0 0 0 0 0 0 0 0 0 15 0 0 0 0 9 10 0 0 0 15 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 19 0 0 0 0 0 0 0 13 0 0 1 0 0 0 1 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 1 0 0 0 35 0 0 0 0 4 0 3 0 0 0 0 0 +# FILE:src/AbstractAsyncTableRendering.java, LINE:2870, OFFSET:2890, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:290, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:12020, TEID:12130, VARs:{}14, +8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 0 0 0 0 17 0 0 0 0 0 0 0 0 0 14 0 0 0 0 0 0 0 2 0 0 3 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 5 0 0 0 0 12 0 0 3 0 0 0 0 0 1 0 16 0 0 0 0 14 10 0 0 0 17 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 0 14 0 0 1 0 0 0 5 0 0 0 1 1 0 3 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 1 0 0 0 0 0 1 0 0 0 0 31 0 0 0 0 0 0 1 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:141, OFFSET:3736, NODE_KIND:128, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:36432, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:1100, TEID:14587, VARs:{}686, +564 0 22 0 0 0 0 2 0 0 0 0 1 0 0 0 0 14 0 0 0 0 0 0 0 0 0 0 0 0 0 4070 0 0 0 0 2239 0 0 0 0 0 0 0 0 1 1815 0 0 0 26 0 0 0 340 0 0 162 0 0 2176 16 0 5 0 0 0 0 0 0 5 0 0 194 0 0 0 0 0 0 1653 0 419 55 0 0 0 1401 0 0 249 0 0 0 0 141 4 0 2076 0 0 0 0 1265 1123 0 0 0 2110 0 0 0 0 0 1998 0 0 0 0 0 0 0 49 0 291 223 0 0 1 0 0 0 0 0 0 61 0 0 0 0 0 0 0 0 22 2131 0 0 4 0 5 0 0 1815 0 0 30 0 5 3 452 0 65 14 34 90 123 304 4 2 0 0 15 0 0 28 4 10 3 35 9 0 58 0 0 867 126 0 7 1 0 63 0 0 0 0 29 3 0 55 0 0 0 0 0 61 79 14 10 0 0 0 0 6 0 12 1 12 6 1 0 127 22 0 39 0 0 0 0 0 0 165 4 34 0 2 186 18 2 0 22 0 0 4 76 1 0 14 4070 0 1 1 18 61 0 35 0 0 4 31 106 +# FILE:src/AbstractTableRendering.java, LINE:397, OFFSET:420, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:484, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:1946, TEID:2111, VARs:{}22, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 51 0 0 0 0 32 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 6 4 0 0 0 14 0 0 3 0 0 0 0 0 0 0 32 0 0 0 0 15 16 0 0 0 37 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 5 2 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 12 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 51 0 0 0 0 0 0 0 0 0 0 5 0 +# FILE:src/AbstractTableRendering.java, LINE:401, OFFSET:420, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:408, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:1972, TEID:2111, VARs:{}20, +5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 42 0 0 0 0 27 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 5 3 0 0 0 13 0 0 3 0 0 0 0 0 0 0 27 0 0 0 0 13 14 0 0 0 31 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 0 4 2 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 42 0 0 0 0 0 0 0 0 0 0 4 0 +# FILE:src/AbstractTableRendering.java, LINE:405, OFFSET:420, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:317, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:2004, TEID:2111, VARs:{}18, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 21 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 4 2 0 0 0 10 0 0 3 0 0 0 0 0 0 0 21 0 0 0 0 11 11 0 0 0 24 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 0 0 0 3 2 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 0 3 0 +# FILE:src/AbstractTableRendering.java, LINE:532, OFFSET:585, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:640, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:2448, TEID:2666, VARs:{}24, +10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 0 0 0 0 49 0 0 0 0 0 0 0 0 0 36 0 0 0 0 0 0 0 3 0 0 0 0 0 46 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 36 0 10 7 0 0 0 25 0 0 2 0 0 0 0 0 1 0 42 0 0 0 0 22 23 0 0 0 42 0 0 0 0 0 38 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 46 0 0 0 0 0 0 0 36 0 0 0 0 0 0 3 0 0 0 0 5 0 9 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 7 0 0 0 0 0 3 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 1 0 2 0 0 4 1 0 0 0 0 0 1 0 0 0 0 55 0 0 0 0 0 0 2 0 0 0 2 0 +# FILE:src/AbstractTableRendering.java, LINE:537, OFFSET:575, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:516, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:2462, TEID:2633, VARs:{}16, +7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 41 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 3 0 0 0 0 0 39 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 29 0 8 6 0 0 0 21 0 0 1 0 0 0 0 0 0 0 35 0 0 0 0 16 18 0 0 0 35 0 0 0 0 0 31 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 0 0 0 0 0 0 0 29 0 0 0 0 0 0 3 0 0 0 0 3 0 7 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 3 0 0 0 0 0 2 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 2 0 0 4 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 0 0 2 0 0 0 2 0 +# FILE:src/AbstractTableRendering.java, LINE:541, OFFSET:568, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:403, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:2488, TEID:2622, VARs:{}11, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 33 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 1 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 6 5 0 0 0 17 0 0 1 0 0 0 0 0 0 0 28 0 0 0 0 13 14 0 0 0 28 0 0 0 0 0 24 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 22 0 0 0 0 0 0 1 0 0 0 0 2 0 6 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 1 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 2 0 0 4 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 2 0 0 0 2 0 +# FILE:src/AbstractTableRendering.java, LINE:548, OFFSET:568, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:306, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:2520, TEID:2622, VARs:{}11, +5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 24 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 1 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 5 4 0 0 0 12 0 0 1 0 0 0 0 0 0 0 21 0 0 0 0 11 11 0 0 0 21 0 0 0 0 0 19 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 18 0 0 0 0 0 0 1 0 0 0 0 2 0 5 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 1 0 0 0 1 0 +# FILE:src/AbstractTableRendering.java, LINE:900, OFFSET:945, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:592, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:3928, TEID:4138, VARs:{}26, +16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 71 0 0 0 0 38 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 7 0 0 6 0 0 36 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 26 0 3 1 0 0 0 22 0 0 1 0 0 0 0 0 0 0 32 0 0 0 0 20 24 0 0 0 32 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 32 0 0 4 0 0 0 13 0 0 0 0 0 0 2 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 16 0 0 0 0 0 2 0 0 0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 71 0 0 0 0 0 0 0 0 0 0 0 2 +# FILE:src/AbstractTableRendering.java, LINE:960, OFFSET:987, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:364, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:4188, TEID:4307, VARs:{}16, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 25 0 0 0 0 0 0 0 0 0 19 0 0 0 0 0 0 0 3 0 0 4 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 5 1 0 0 0 17 0 0 5 0 0 0 0 0 0 0 24 0 0 0 0 14 9 0 0 0 24 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 19 0 0 0 0 0 0 7 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 1 0 2 0 0 7 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 0 0 0 0 0 0 1 0 0 4 0 0 0 2 0 0 0 0 0 0 0 32 0 0 0 1 0 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:999, OFFSET:1026, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:384, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:4356, TEID:4482, VARs:{}16, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 34 0 0 0 0 27 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 3 0 0 4 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 5 1 0 0 0 18 0 0 5 0 0 0 0 0 0 0 25 0 0 0 0 15 10 0 0 0 25 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 20 0 0 0 0 0 0 7 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 1 0 2 0 0 7 2 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 1 0 0 0 0 0 0 1 0 1 0 0 4 0 0 0 2 0 0 0 0 0 0 0 34 0 0 0 1 0 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1190, OFFSET:1212, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:242, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5192, TEID:5294, VARs:{}14, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 0 0 0 16 0 0 0 0 0 0 0 0 0 11 0 0 0 0 0 0 0 1 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 7 1 0 0 0 11 0 0 6 0 0 0 0 0 0 0 13 0 0 0 0 17 13 0 0 0 14 0 0 0 0 0 11 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 0 0 0 0 0 0 0 11 0 0 0 0 0 0 1 0 0 0 1 0 0 7 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 4 0 0 0 0 0 2 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1365, OFFSET:1386, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:292, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5769, TEID:5876, VARs:{}12, +2 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 29 0 0 0 0 17 0 0 0 0 0 0 0 0 0 13 0 0 0 2 0 0 0 4 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 0 3 0 0 0 0 12 0 0 3 0 0 0 0 0 0 0 17 0 0 0 0 12 6 0 0 0 17 0 0 0 0 0 15 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 17 0 0 0 0 0 0 0 13 0 0 0 0 0 0 4 0 0 2 0 2 0 3 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 8 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 2 0 0 0 0 0 0 0 0 2 0 0 0 0 2 0 0 0 0 0 0 0 2 0 0 0 29 0 0 0 0 4 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1367, OFFSET:1386, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:278, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5773, TEID:5876, VARs:{}11, +2 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 28 0 0 0 0 16 0 0 0 0 0 0 0 0 0 12 0 0 0 2 0 0 0 4 0 0 0 0 0 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 12 0 3 0 0 0 0 12 0 0 2 0 0 0 0 0 0 0 16 0 0 0 0 11 6 0 0 0 16 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 16 0 0 0 0 0 0 0 12 0 0 0 0 0 0 4 0 0 2 0 2 0 2 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 8 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 2 0 0 0 0 0 0 0 0 2 0 0 0 0 2 0 0 0 0 0 0 0 2 0 0 0 28 0 0 0 0 4 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1407, OFFSET:1428, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:374, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5975, TEID:6115, VARs:{}18, +3 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 0 0 0 28 0 0 0 0 0 0 0 0 0 16 0 0 0 1 0 0 0 3 0 0 0 0 0 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 5 2 0 0 0 16 0 0 0 0 0 0 0 0 0 0 19 0 0 0 0 9 12 0 0 0 19 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 16 0 0 1 0 0 0 3 0 0 1 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 2 1 0 0 0 0 0 0 0 0 0 0 1 0 0 4 0 0 0 0 0 0 0 1 0 0 1 45 0 0 0 0 1 0 4 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:1409, OFFSET:1428, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:328, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:5991, TEID:6115, VARs:{}18, +3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 40 0 0 0 0 25 0 0 0 0 0 0 0 0 0 14 0 0 0 1 0 0 0 2 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 0 5 2 0 0 0 15 0 0 0 0 0 0 0 0 0 0 16 0 0 0 0 8 12 0 0 0 16 0 0 0 0 0 15 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 14 0 0 1 0 0 0 2 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 13 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 3 0 0 0 0 0 0 0 1 0 0 1 40 0 0 0 0 1 0 4 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:1413, OFFSET:1428, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:259, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:6013, TEID:6114, VARs:{}13, +3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 32 0 0 0 0 20 0 0 0 0 0 0 0 0 0 10 0 0 0 1 0 0 0 1 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 4 2 0 0 0 12 0 0 0 0 0 0 0 0 0 0 12 0 0 0 0 7 11 0 0 0 12 0 0 0 0 0 11 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 0 0 0 0 0 0 10 0 0 1 0 0 0 1 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 1 0 0 1 32 0 0 0 0 1 0 3 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1635, OFFSET:1669, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:405, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:6799, TEID:6941, VARs:{}24, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 41 0 0 0 0 27 0 0 0 0 0 0 0 0 0 19 0 0 0 0 0 0 0 5 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 19 0 6 0 0 0 0 15 0 0 8 0 0 0 0 0 0 0 25 0 0 0 0 20 14 0 0 0 25 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 19 0 0 0 0 0 0 5 0 0 0 0 0 0 8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 7 6 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6 0 0 0 41 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1637, OFFSET:1669, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:387, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:6806, TEID:6941, VARs:{}23, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 40 0 0 0 0 25 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 5 0 0 0 0 0 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 6 0 0 0 0 14 0 0 7 0 0 0 0 0 0 0 24 0 0 0 0 19 13 0 0 0 24 0 0 0 0 0 24 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 18 0 0 0 0 0 0 5 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 7 6 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6 0 0 0 40 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1938, OFFSET:1963, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:380, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:7830, TEID:7969, VARs:{}17, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 0 0 0 24 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 1 0 0 6 0 0 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 0 6 1 0 0 0 23 0 0 3 0 0 0 0 0 0 0 23 0 0 0 0 14 14 0 0 0 24 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 20 0 0 3 0 0 0 5 0 0 0 1 0 0 4 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 10 2 0 0 0 0 0 0 0 0 0 0 2 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 0 0 1 0 0 0 35 0 0 0 0 0 0 1 0 0 0 1 0 +# FILE:src/AbstractTableRendering.java, LINE:1993, OFFSET:2014, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:341, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8097, TEID:8220, VARs:{}11, +5 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 38 0 0 0 0 25 0 0 0 0 0 0 0 0 0 14 0 0 0 4 0 0 0 1 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 0 5 2 0 0 0 15 0 0 0 0 0 0 0 0 0 0 17 0 0 0 0 11 9 0 0 0 17 0 0 0 0 0 15 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 14 0 0 1 0 0 0 1 0 0 1 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 11 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 3 0 0 0 0 0 0 0 0 0 0 2 0 0 4 0 0 0 0 0 0 0 1 0 0 0 38 0 0 0 0 4 0 3 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1997, OFFSET:2014, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:289, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8115, TEID:8220, VARs:{}11, +5 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 0 0 0 21 0 0 0 0 0 0 0 0 0 12 0 0 0 4 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 12 0 5 2 0 0 0 13 0 0 0 0 0 0 0 0 0 0 14 0 0 0 0 10 9 0 0 0 14 0 0 0 0 0 13 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 12 0 0 1 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 1 0 0 0 33 0 0 0 0 4 0 3 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:1997, OFFSET:2014, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:287, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8116, TEID:8219, VARs:{}11, +5 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 0 0 0 21 0 0 0 0 0 0 0 0 0 12 0 0 0 4 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 12 0 4 2 0 0 0 13 0 0 0 0 0 0 0 0 0 0 14 0 0 0 0 9 9 0 0 0 14 0 0 0 0 0 13 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 12 0 0 1 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 0 0 0 0 0 0 0 0 0 0 3 0 0 2 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 1 0 0 0 33 0 0 0 0 4 0 3 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2030, OFFSET:2078, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:542, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8238, TEID:8440, VARs:{}32, +17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 0 0 0 0 33 0 0 0 0 0 0 0 0 0 30 0 0 0 0 0 0 0 1 0 0 5 0 0 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 0 8 3 0 0 0 22 0 0 2 0 0 0 0 0 0 0 32 0 0 0 0 28 24 0 0 0 33 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 30 0 0 0 0 0 0 6 0 0 0 1 4 0 5 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 10 2 0 0 0 0 1 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 2 1 0 0 0 0 0 0 1 0 0 0 55 0 0 0 0 0 0 0 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:2032, OFFSET:2072, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:456, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8245, TEID:8416, VARs:{}29, +15 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 46 0 0 0 0 28 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 1 0 0 4 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 6 2 0 0 0 19 0 0 2 0 0 0 0 0 0 0 27 0 0 0 0 24 21 0 0 0 27 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 25 0 0 0 0 0 0 5 0 0 0 0 3 0 4 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 8 2 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 2 1 0 0 0 0 0 0 1 0 0 0 46 0 0 0 0 0 0 0 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:2042, OFFSET:2068, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:266, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8298, TEID:8401, VARs:{}18, +11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 16 0 0 0 0 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 0 3 0 0 15 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 3 2 0 0 0 12 0 0 1 0 0 0 0 0 0 0 15 0 0 0 0 15 16 0 0 0 15 0 0 0 0 0 15 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 0 0 0 0 0 0 14 0 0 0 0 0 0 3 0 0 0 0 2 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 5 0 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2087, OFFSET:2129, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:597, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8462, TEID:8659, VARs:{}27, +9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 57 0 0 0 0 40 0 0 0 0 0 0 0 0 0 38 0 0 0 0 0 0 0 3 0 0 3 0 0 39 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 0 8 3 0 0 0 23 0 0 3 0 0 0 0 0 1 0 39 0 0 0 0 19 20 0 0 0 40 0 0 0 0 0 39 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 39 0 0 0 0 0 0 0 38 0 0 0 0 0 0 5 0 0 0 1 2 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 10 2 0 0 0 0 1 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 1 0 0 1 1 0 0 0 0 0 1 0 0 0 0 57 0 0 0 0 0 0 0 0 0 0 0 2 +# FILE:src/AbstractTableRendering.java, LINE:2168, OFFSET:2181, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:340, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:8822, TEID:8927, VARs:{}16, +0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 34 0 0 0 0 23 0 0 0 0 0 0 0 0 0 18 0 0 0 2 0 0 0 5 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 3 0 0 0 0 15 0 0 2 0 0 0 0 0 0 0 23 0 0 0 0 7 7 0 0 0 23 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 18 0 0 0 0 0 0 5 0 0 1 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 1 0 1 0 0 7 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 1 0 0 0 0 0 0 0 0 1 0 0 0 0 4 0 0 0 1 0 0 0 0 0 0 0 34 0 0 0 0 2 0 0 0 0 0 1 1 +# FILE:src/AbstractTableRendering.java, LINE:2222, OFFSET:2274, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:575, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9030, TEID:9261, VARs:{}42, +17 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 65 0 0 0 0 33 0 0 0 0 0 0 0 0 0 25 0 0 0 1 0 0 0 3 0 0 4 0 0 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 5 0 0 0 0 26 0 0 6 0 0 0 0 0 0 0 31 0 0 0 0 29 25 0 0 0 31 0 0 0 0 0 30 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 31 0 0 0 0 0 0 0 25 0 0 0 0 0 0 7 0 0 1 0 1 0 6 0 0 0 0 1 0 0 0 0 1 0 1 0 0 4 0 0 23 7 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 3 0 0 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 65 0 0 0 0 1 0 0 0 0 0 1 0 +# FILE:src/AbstractTableRendering.java, LINE:2318, OFFSET:2390, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:802, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9377, TEID:9656, VARs:{}44, +13 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 90 0 0 0 0 54 0 0 0 0 0 0 0 0 0 46 0 0 0 0 0 0 0 7 0 0 1 0 0 49 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 9 1 0 0 0 26 0 0 5 0 0 0 0 0 1 0 49 0 0 0 0 29 27 0 0 0 50 0 0 0 0 0 47 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 49 0 0 0 0 0 0 0 46 0 0 0 0 0 3 8 0 0 0 1 2 0 6 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 19 2 0 0 0 0 5 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 2 1 0 0 1 0 0 1 0 0 0 0 90 0 0 0 0 0 0 0 0 0 0 0 4 +# FILE:src/AbstractTableRendering.java, LINE:2324, OFFSET:2380, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:705, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9391, TEID:9628, VARs:{}38, +11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 79 0 0 0 0 48 0 0 0 0 0 0 0 0 0 41 0 0 0 0 0 0 0 7 0 0 1 0 0 44 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 40 0 6 1 0 0 0 24 0 0 4 0 0 0 0 0 0 0 44 0 0 0 0 23 24 0 0 0 45 0 0 0 0 0 42 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 0 0 0 0 41 0 0 0 0 0 2 8 0 0 0 1 1 0 5 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 16 2 0 0 0 0 4 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 1 0 0 0 0 0 0 0 79 0 0 0 0 0 0 0 0 0 0 0 4 +# FILE:src/AbstractTableRendering.java, LINE:2341, OFFSET:2380, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:531, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9454, TEID:9628, VARs:{}29, +7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 60 0 0 0 0 36 0 0 0 0 0 0 0 0 0 31 0 0 0 0 0 0 0 6 0 0 1 0 0 34 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 30 0 4 0 0 0 0 18 0 0 3 0 0 0 0 0 0 0 34 0 0 0 0 16 14 0 0 0 35 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 34 0 0 0 0 0 0 0 31 0 0 0 0 0 2 7 0 0 0 1 1 0 3 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 12 2 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 1 0 0 0 0 0 0 0 60 0 0 0 0 0 0 0 0 0 0 0 4 +# FILE:src/AbstractTableRendering.java, LINE:2457, OFFSET:2515, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:591, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9903, TEID:10112, VARs:{}32, +9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 52 0 0 0 0 42 0 0 0 0 0 0 0 0 0 34 0 0 0 0 0 0 0 4 0 0 1 0 0 38 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 11 4 0 0 0 22 0 0 7 0 0 0 0 0 1 0 36 0 0 0 0 28 21 0 0 0 37 0 0 0 0 0 34 0 0 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 38 0 0 0 0 0 0 0 34 0 0 0 0 0 0 5 0 0 0 1 5 0 11 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 7 0 0 0 0 0 4 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 2 0 1 0 0 2 1 0 0 0 0 0 1 0 0 0 0 52 0 0 0 0 0 0 1 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2473, OFFSET:2510, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:441, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9950, TEID:10101, VARs:{}22, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 36 0 0 0 0 32 0 0 0 0 0 0 0 0 0 26 0 0 0 0 0 0 0 3 0 0 0 0 0 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 26 0 8 4 0 0 0 19 0 0 4 0 0 0 0 0 0 0 28 0 0 0 0 18 16 0 0 0 29 0 0 0 0 0 26 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 30 0 0 0 0 0 0 0 26 0 0 0 0 0 0 3 0 0 0 1 5 0 8 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 4 0 0 0 0 0 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 36 0 0 0 0 0 0 1 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2476, OFFSET:2510, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:421, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9958, TEID:10101, VARs:{}20, +6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 34 0 0 0 0 31 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 3 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 0 8 4 0 0 0 18 0 0 3 0 0 0 0 0 0 0 27 0 0 0 0 17 15 0 0 0 27 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 25 0 0 0 0 0 0 3 0 0 0 0 5 0 7 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 4 0 0 0 0 0 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 34 0 0 0 0 0 0 1 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2476, OFFSET:2503, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:354, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9959, TEID:10078, VARs:{}16, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 28 0 0 0 0 27 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 3 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 7 3 0 0 0 15 0 0 3 0 0 0 0 0 0 0 23 0 0 0 0 14 12 0 0 0 23 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 21 0 0 0 0 0 0 3 0 0 0 0 3 0 6 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 28 0 0 0 0 0 0 1 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2478, OFFSET:2503, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:338, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9965, TEID:10078, VARs:{}15, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 26 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 3 0 0 0 0 0 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20 0 7 3 0 0 0 14 0 0 2 0 0 0 0 0 0 0 22 0 0 0 0 13 11 0 0 0 22 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 20 0 0 0 0 0 0 3 0 0 0 0 3 0 5 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 1 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2478, OFFSET:2503, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:336, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:9966, TEID:10077, VARs:{}15, +4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 26 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 3 0 0 0 0 0 24 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20 0 6 3 0 0 0 14 0 0 2 0 0 0 0 0 0 0 22 0 0 0 0 12 11 0 0 0 22 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 20 0 0 0 0 0 0 3 0 0 0 0 3 0 5 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 2 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 1 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2550, OFFSET:2597, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:955, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:10256, TEID:10554, VARs:{}36, +1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 95 0 0 0 0 66 0 0 0 0 0 0 0 0 0 52 0 0 0 0 0 0 0 14 0 0 1 0 0 64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 51 0 4 0 0 0 0 52 0 0 7 0 0 0 0 0 0 0 62 0 0 0 0 17 30 0 0 0 63 0 0 0 0 0 60 0 0 0 0 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 63 0 0 0 0 0 0 0 52 0 0 0 0 0 0 15 0 0 0 1 5 0 7 0 0 0 0 0 0 0 0 0 0 0 0 2 0 1 0 0 24 4 0 0 0 0 2 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 1 0 0 0 0 0 0 0 0 0 5 2 1 0 0 10 0 0 0 0 0 0 0 4 0 0 0 95 0 0 0 0 0 0 0 0 0 0 1 2 +# FILE:src/AbstractTableRendering.java, LINE:2552, OFFSET:2597, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:935, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:10264, TEID:10554, VARs:{}34, +1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 93 0 0 0 0 65 0 0 0 0 0 0 0 0 0 51 0 0 0 0 0 0 0 14 0 0 1 0 0 63 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 50 0 4 0 0 0 0 51 0 0 6 0 0 0 0 0 0 0 61 0 0 0 0 16 29 0 0 0 61 0 0 0 0 0 59 0 0 0 0 0 0 0 0 0 14 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 62 0 0 0 0 0 0 0 51 0 0 0 0 0 0 15 0 0 0 0 5 0 6 0 0 0 0 0 0 0 0 0 0 0 0 2 0 1 0 0 24 4 0 0 0 0 2 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 1 0 0 0 0 0 0 0 0 0 5 2 1 0 0 10 0 0 0 0 0 0 0 4 0 0 0 93 0 0 0 0 0 0 0 0 0 0 1 2 +# FILE:src/AbstractTableRendering.java, LINE:2742, OFFSET:2780, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:392, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11106, TEID:11236, VARs:{}28, +11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 0 0 0 25 0 0 0 0 0 0 0 0 0 20 0 0 0 0 0 0 0 2 0 0 4 0 0 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 0 4 1 0 0 0 9 0 0 2 0 0 0 0 0 0 0 25 0 0 0 0 17 9 0 0 0 25 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 20 0 0 0 0 0 0 6 0 0 0 0 1 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0 0 12 5 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 0 0 0 0 0 0 0 0 0 0 1 0 +# FILE:src/AbstractTableRendering.java, LINE:2745, OFFSET:2780, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:351, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11116, TEID:11236, VARs:{}27, +11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 21 0 0 0 0 0 0 0 0 0 19 0 0 0 0 0 0 0 2 0 0 4 0 0 21 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 4 1 0 0 0 7 0 0 1 0 0 0 0 0 0 0 21 0 0 0 0 16 9 0 0 0 21 0 0 0 0 0 21 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 19 0 0 0 0 0 0 6 0 0 0 0 1 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 12 3 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:src/AbstractTableRendering.java, LINE:2818, OFFSET:2845, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:327, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11344, TEID:11469, VARs:{}30, +8 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 18 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 4 0 0 0 0 0 18 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 18 0 3 0 0 0 0 9 0 0 0 0 0 0 0 1 0 0 18 0 0 0 0 9 13 0 0 0 18 0 0 0 0 0 18 0 0 0 0 0 0 0 0 0 4 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 18 0 0 0 0 0 0 0 18 0 0 0 0 0 0 4 0 1 0 0 2 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 11 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 0 0 0 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:2856, OFFSET:2995, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:1376, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:11485, TEID:12006, VARs:{}46, +40 0 5 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 161 0 0 0 0 86 0 0 0 0 0 0 0 0 1 70 0 0 0 0 0 0 0 6 0 0 3 0 0 86 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 67 0 11 2 0 0 0 44 0 0 6 0 0 0 0 1 0 0 78 0 0 0 0 63 61 0 0 0 78 0 0 0 0 0 70 0 0 0 0 0 0 0 0 0 6 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 80 0 0 0 0 0 0 0 70 0 0 0 0 5 0 9 0 1 0 0 0 0 8 4 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 36 3 0 5 1 0 0 0 0 0 0 6 0 0 2 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 3 1 0 0 0 0 3 5 0 0 0 0 0 0 0 0 2 0 1 0 1 10 1 0 0 4 0 0 0 0 0 0 0 161 0 0 0 0 0 0 1 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:3272, OFFSET:3332, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:618, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:12864, TEID:13133, VARs:{}48, +19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 72 0 0 0 0 33 0 0 0 0 0 0 0 0 0 28 0 0 0 0 0 0 0 3 0 0 2 0 0 32 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 26 0 12 2 0 0 0 22 0 0 6 0 0 0 0 2 0 0 32 0 0 0 0 35 30 0 0 0 34 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 32 0 0 0 0 0 0 0 28 0 0 0 0 0 0 5 0 2 0 2 2 0 8 0 0 0 0 1 0 0 0 0 0 0 0 0 0 2 0 0 17 1 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 1 0 0 2 1 0 0 0 0 0 0 2 0 0 0 72 0 0 0 1 0 0 0 0 0 0 0 2 +# FILE:src/AbstractTableRendering.java, LINE:3273, OFFSET:3332, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:592, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:12872, TEID:13133, VARs:{}47, +19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 70 0 0 0 0 31 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 3 0 0 2 0 0 30 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 25 0 12 2 0 0 0 21 0 0 5 0 0 0 0 2 0 0 30 0 0 0 0 34 30 0 0 0 32 0 0 0 0 0 30 0 0 0 0 0 0 0 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 30 0 0 0 0 0 0 0 27 0 0 0 0 0 0 5 0 2 0 2 2 0 7 0 0 0 0 1 0 0 0 0 0 0 0 0 0 2 0 0 16 1 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 1 0 0 2 1 0 0 0 0 0 0 1 0 0 0 70 0 0 0 0 0 0 0 0 0 0 0 2 +# FILE:src/AbstractTableRendering.java, LINE:3273, OFFSET:3327, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:577, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:12873, TEID:13129, VARs:{}47, +19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 69 0 0 0 0 30 0 0 0 0 0 0 0 0 0 26 0 0 0 0 0 0 0 3 0 0 2 0 0 29 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 24 0 11 2 0 0 0 21 0 0 5 0 0 0 0 2 0 0 29 0 0 0 0 32 30 0 0 0 31 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 26 0 0 0 0 0 0 5 0 2 0 2 2 0 7 0 0 0 0 1 0 0 0 0 0 0 0 0 0 2 0 0 16 1 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 1 0 0 0 0 0 0 1 0 0 0 69 0 0 0 0 0 0 0 0 0 0 0 2 +# FILE:src/AbstractTableRendering.java, LINE:3274, OFFSET:3327, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:554, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:12879, TEID:13129, VARs:{}47, +19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 68 0 0 0 0 28 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 3 0 0 2 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 23 0 11 2 0 0 0 20 0 0 4 0 0 0 0 2 0 0 27 0 0 0 0 31 30 0 0 0 29 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 25 0 0 0 0 0 0 5 0 2 0 2 2 0 6 0 0 0 0 1 0 0 0 0 0 0 0 0 0 2 0 0 16 0 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 1 0 0 0 0 0 0 0 0 0 0 68 0 0 0 0 0 0 0 0 0 0 0 2 +# FILE:src/AbstractTableRendering.java, LINE:3274, OFFSET:3327, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:552, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:12880, TEID:13128, VARs:{}47, +19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 68 0 0 0 0 28 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 3 0 0 2 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 23 0 10 2 0 0 0 20 0 0 4 0 0 0 0 2 0 0 27 0 0 0 0 30 30 0 0 0 29 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 25 0 0 0 0 0 0 5 0 2 0 2 2 0 6 0 0 0 0 1 0 0 0 0 0 0 0 0 0 2 0 0 16 0 0 0 0 0 1 0 0 0 0 0 0 0 2 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 1 0 0 0 0 0 0 0 0 0 0 68 0 0 0 0 0 0 0 0 0 0 0 2 +# FILE:src/AbstractTableRendering.java, LINE:3538, OFFSET:3545, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:410, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:13764, TEID:13888, VARs:{}15, +1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 46 0 0 0 0 27 0 0 0 0 0 0 0 0 0 21 0 0 0 5 0 0 0 3 0 0 1 0 0 27 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20 0 1 0 0 0 0 22 0 0 1 0 0 0 0 0 0 0 25 0 0 0 0 5 6 0 0 0 25 0 0 0 0 0 22 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 27 0 0 0 0 0 0 0 21 0 0 0 0 0 0 4 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 46 0 0 0 0 5 0 2 0 0 0 1 2 +# FILE:src/AbstractTableRendering.java, LINE:3540, OFFSET:3545, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:364, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:13780, TEID:13888, VARs:{}14, +1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 41 0 0 0 0 24 0 0 0 0 0 0 0 0 0 19 0 0 0 5 0 0 0 2 0 0 1 0 0 24 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 0 1 0 0 0 0 21 0 0 1 0 0 0 0 0 0 0 22 0 0 0 0 4 6 0 0 0 22 0 0 0 0 0 20 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 19 0 0 0 0 0 0 3 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 4 0 0 0 0 0 0 0 0 0 0 0 41 0 0 0 0 5 0 2 0 0 0 1 2 +# FILE:src/AbstractTableRendering.java, LINE:3559, OFFSET:3566, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:332, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:13928, TEID:14035, VARs:{}15, +1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 0 22 0 0 0 0 0 0 0 0 0 16 0 0 0 0 0 0 0 3 0 0 1 0 0 22 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 15 0 1 0 0 0 0 13 0 0 1 0 0 0 0 0 0 0 20 0 0 0 0 5 6 0 0 0 20 0 0 0 0 0 17 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 22 0 0 0 0 0 0 0 16 0 0 0 0 0 0 4 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 44 0 0 0 0 0 0 2 0 0 0 1 2 +# FILE:src/AbstractTableRendering.java, LINE:3616, OFFSET:3637, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:516, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:14154, TEID:14312, VARs:{}22, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 51 0 0 0 0 37 0 0 0 0 0 0 0 0 0 30 0 0 0 0 0 0 0 8 0 0 0 0 0 37 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 30 0 5 1 0 0 0 16 0 0 5 0 0 0 0 0 0 0 34 0 0 0 0 12 9 0 0 0 36 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 8 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 2 36 0 0 0 0 0 0 0 30 0 0 0 0 0 0 8 0 0 0 2 0 0 6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 4 3 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 51 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:3617, OFFSET:3637, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:498, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:14160, TEID:14312, VARs:{}22, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 49 0 0 0 0 36 0 0 0 0 0 0 0 0 0 29 0 0 0 0 0 0 0 8 0 0 0 0 0 36 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 29 0 5 1 0 0 0 16 0 0 4 0 0 0 0 0 0 0 33 0 0 0 0 11 9 0 0 0 34 0 0 0 0 0 32 0 0 0 0 0 0 0 0 0 8 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 2 35 0 0 0 0 0 0 0 29 0 0 0 0 0 0 8 0 0 0 1 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 4 3 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 49 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:3618, OFFSET:3637, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:454, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:14171, TEID:14311, VARs:{}20, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 0 33 0 0 0 0 0 0 0 0 0 26 0 0 0 0 0 0 0 7 0 0 0 0 0 33 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 26 0 4 1 0 0 0 15 0 0 4 0 0 0 0 0 0 0 30 0 0 0 0 10 8 0 0 0 31 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 2 32 0 0 0 0 0 0 0 26 0 0 0 0 0 0 7 0 0 0 1 0 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 0 0 4 3 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 44 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:3619, OFFSET:3637, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:431, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:14177, TEID:14311, VARs:{}20, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43 0 0 0 0 31 0 0 0 0 0 0 0 0 0 25 0 0 0 0 0 0 0 7 0 0 0 0 0 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 25 0 4 1 0 0 0 14 0 0 3 0 0 0 0 0 0 0 28 0 0 0 0 9 8 0 0 0 29 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 7 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 2 30 0 0 0 0 0 0 0 25 0 0 0 0 0 0 7 0 0 0 1 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 4 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 43 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:3620, OFFSET:3637, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:406, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:14187, TEID:14310, VARs:{}17, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 0 0 0 0 30 0 0 0 0 0 0 0 0 0 24 0 0 0 0 0 0 0 6 0 0 0 0 0 30 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 0 3 1 0 0 0 13 0 0 3 0 0 0 0 0 0 0 27 0 0 0 0 8 7 0 0 0 28 0 0 0 0 0 26 0 0 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 2 29 0 0 0 0 0 0 0 24 0 0 0 0 0 0 6 0 0 0 1 0 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 39 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:3621, OFFSET:3637, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:388, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:14193, TEID:14310, VARs:{}17, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 37 0 0 0 0 29 0 0 0 0 0 0 0 0 0 23 0 0 0 0 0 0 0 6 0 0 0 0 0 29 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 0 3 1 0 0 0 13 0 0 2 0 0 0 0 0 0 0 26 0 0 0 0 7 7 0 0 0 26 0 0 0 0 0 25 0 0 0 0 0 0 0 0 0 6 0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 2 28 0 0 0 0 0 0 0 23 0 0 0 0 0 0 6 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 37 0 0 0 0 2 0 1 0 0 0 0 1 +# FILE:src/AbstractTableRendering.java, LINE:3623, OFFSET:3637, NODE_KIND:103, CONTEXT_KIND:0, NEIGHBOR_KIND:0, NUM_NODE:356, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, TBID:14202, TEID:14309, VARs:{}15, +2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 33 0 0 0 0 27 0 0 0 0 0 0 0 0 0 21 0 0 0 0 0 0 0 5 0 0 0 0 0 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 21 0 2 1 0 0 0 13 0 0 2 0 0 0 0 0 0 0 24 0 0 0 0 6 7 0 0 0 24 0 0 0 0 0 23 0 0 0 0 0 0 0 0 0 5 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 2 26 0 0 0 0 0 0 0 21 0 0 0 0 0 0 5 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 3 2 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 33 0 0 0 0 2 0 1 0 0 0 0 1 diff --git a/scripts/bugdetect/bugcounting b/scripts/bugdetect/bugcounting new file mode 100644 index 0000000..5417364 --- /dev/null +++ b/scripts/bugdetect/bugcounting @@ -0,0 +1,66 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 65 +fi + +if [[ ! (-f $1) ]]; then + echo "Usage: no file: $1" + exit 65 +fi + +RANKS='1 2 3 4 5 6 7' +echo -n "Total: " +grep "Rank score: " $1 | wc -l +for i in ${RANKS}; do + echo -n "Rank $i: " + grep "Rank score: $i" $1 | wc -l +done + +BUGS='1 2 3 4 5 6 7 8 9 10 11 12 13 14 15' +echo -n "Bugs: " +grep "^B" $1 | wc -l +for i in ${BUGS}; do + echo -n "B${i}=" + grep "^B" $1 | sed -r "s/^B([[:digit:]]+).*/B\1/" | grep "^B${i}$" | wc -l +done +echo -n "Styles: " +grep "^S" $1 | wc -l +for i in ${BUGS}; do + echo -n "S${i}=" + grep "^S" $1 | sed -r "s/^S([[:digit:]]+).*/S\1/" | grep "^S${i}$" | wc -l +done diff --git a/scripts/bugdetect/bugfiltering b/scripts/bugdetect/bugfiltering new file mode 100644 index 0000000..4a64dd5 --- /dev/null +++ b/scripts/bugdetect/bugfiltering @@ -0,0 +1,92 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: $0 +# output to stdout + +import re +import sys +import os +#import shutil + +if len(sys.argv) != 3: + print "Usage: ", sys.argv[0], " " + sys.exit(1) +elif sys.argv[2] != 'c' and sys.argv[2] != 'java' and sys.argv[2] != 'php': + print "We only support c, java, and php for now" + sys.exit(1) + +def readclusters( file ): + f= open(file,'r') + clusters= [] + cluster= [] + for line in f.readlines(): + if line == '\n' or line == '': + if len(cluster) > 0: + clusters.append(cluster) + cluster=[] + continue + cluster.append(line) + f.close() + if len(cluster) > 0: + clusters.append(cluster) + return clusters + +clusters= readclusters( sys.argv[1] ) +#os.system('echo -n "" > ' + sys.argv[1] + '.bug') +for cluster in clusters: + f=open( sys.argv[1] + '.clustertmp', 'w' ) + for line in cluster: + f.write(line) + f.close() +# meaning of IDs to cbugfilters: +# 0: apply all filters and filter it iff all filters agree. +# 1: apply context comparison +# 2: apply context comparison and condition comparison +# 3: apply variable filtering +# none: apply all filters and filter it iff any filter says yes + if sys.argv[2] == 'c': + os.system('cbugfilters 2 < ' + sys.argv[1] + '.clustertmp') + elif sys.argv[2] == 'java': + os.system('jbugfilters 2 < ' + sys.argv[1] + '.clustertmp') + elif sys.argv[2] == 'php': + os.system('phpbugfilters 2 < ' + sys.argv[1] + '.clustertmp') + else: + print "parameter error for bugfiltering." + sys.exit(1) +# os.system('cbugfilters < ' + sys.argv[1] + '.clustertmp >> ' + sys.argv[1] + '.bug') +# os.system('echo "" >> ' + sys.argv[1] + '.bug') +os.remove(sys.argv[1] + '.clustertmp') + diff --git a/scripts/bugdetect/bugmerging b/scripts/bugdetect/bugmerging new file mode 100644 index 0000000..05a55cf --- /dev/null +++ b/scripts/bugdetect/bugmerging @@ -0,0 +1,165 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: $0 +# output to stdout +# only merge "category marks" + +import re +import sys +import os +#import shutil + +if len(sys.argv) != 3: + print "Usage: ", sys.argv[0], " " + sys.exit(1) + +rs= re.compile('.*Rank score:\s*(-?\d+)\s*\*\s*(\d+)\s*=\s*(-?\d+)\s*buggy score:\s*(-?\d+).*') +#rs= re.compile('.*Rank score:\s*(-?\d+)\s*\*\s*(\d+)\s*=\s*(-?\d+)\s*buggy score:\s*(-?\d+)\s(-?\d+)\s(-?\d+)\s(-?\d+).*') +range_r= re.compile('.*FILE (.*) LINE:(\d+):(\d+) NODE_KIND.*') +bugcategory_r= re.compile('(^B|^S|^G|^F)(\d+)\s*.*') +def readclusters( file ): + f= open(file,'r') + clusters= [] + cluster= [] + for line in f.readlines(): + if line == '\n' or line == '': + if len(cluster) > 0: + clusters.append(cluster) + cluster=[] + continue + cluster.append(line) + f.close() + if len(cluster) > 0: + clusters.append(cluster) + return clusters + +def print_cluster(cluster): + for line in cluster: + print line, + +def print_clusters(clusters): + for cluster in clusters: + print_cluster(cluster) + print + +def cluster_lineranges( cluster ): + size= len(cluster) +# rsl[0]: rank score and other comments +# rsl[1]: category marks, its length should be <=1 +# rsl[2]: the clones +# rsl[3]: the ranges of clones +# rsl[4]: the size of cluster + rsl=([], [], [], [], size) + for i in range(size): + rs_m= rs.match(cluster[i]) + if rs_m != None: + rsl[0].append(cluster[i]) + continue + bug_m= bugcategory_r.match(cluster[i]) + if bug_m != None: + rsl[1].append(cluster[i]) + continue + m= range_r.match(cluster[i]) + if m != None: + rsl[2].append(cluster[i]) + rsl[3].append( ( m.group(1), int(m.group(2)), int(m.group(2))+int(m.group(3))-1 ) ) + continue + rsl[0].append(cluster[i]) + return rsl + +def buggy_clusters( clusters ): + newlist= [] + for i in range(len(clusters)): + keep= False + for j in range(len(clusters[i])): + m= bugcategory_r.match(clusters[i][j]) + if m == None: + continue + else: + keep= True + if keep: + newlist.append(clusters[i]) + return newlist + +def merge_clusters(c1, c2): + c1ranges= map(cluster_lineranges, c1) + c2ranges= map(cluster_lineranges, c2) + falseneg= 0 + for i in range(len(c1ranges)): + findmatch= -1 + for j in range(len(c2ranges)): + if len(c1ranges[i][3]) == len(c2ranges[j][3]): + matched= True + for k in range(len(c2ranges[j][3])): + linematched= False + for l in range(len(c2ranges[j][3])): + if c1ranges[i][3][k][2]<=c2ranges[j][3][l][2] and c1ranges[i][3][k][1]>=c2ranges[j][3][l][1] \ + and c1ranges[i][3][k][0]==c2ranges[j][3][l][0]: + linematched= True + break + else: + continue + if not linematched: + matched= False + break + if matched: + findmatch= j + break + if findmatch>=0: +# update c2: +# only need B, S marks, and C2's own marks take priority: + if len(c2ranges[findmatch][1][:])<1: + c2ranges[findmatch][1][:] = c1ranges[i][1] + else: + falseneg = falseneg+1 + for line in c1ranges[i][0]+c1ranges[i][1]+c1ranges[i][2]: + print >> sys.stderr, line, + print >> sys.stderr + newlist=[] + for i in range(len(c2ranges)): + newlist.append(c2ranges[i][0]+c2ranges[i][1]+c2ranges[i][2]) + print >> sys.stderr, "Miss bugs: ", falseneg + return newlist + +clusters1= readclusters( sys.argv[1] ) +clusters2= readclusters( sys.argv[2] ) +clusters1= buggy_clusters(clusters1) +merged_clusters=merge_clusters(clusters1, clusters2) +# mark the output as auto generated: +print "Auto generated: merging ", len(clusters1), len(clusters2) +print_clusters(merged_clusters) + + diff --git a/scripts/bugdetect/bugordering b/scripts/bugdetect/bugordering new file mode 100644 index 0000000..0a16929 --- /dev/null +++ b/scripts/bugdetect/bugordering @@ -0,0 +1,101 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: $0 +# output to stdout + +# To change the way the clusters are sorted, change +# the cmp_rank_score function + +import re +import sys +import os + +if len(sys.argv) != 2: + print "Usage: ", sys.argv[0], " " + sys.exit(1) + +rs= re.compile('.*Rank score:\s*(-?\d+)\s*\*\s*(\d+)\s*=\s*(-?\d+)\s*buggy score:\s*(-?\d+)\s(-?\d+)\s(-?\d+)\s(-?\d+).*') +def cluster_rank_score( line ): + m= rs.match(line) + s= (0,0,0) + if m == None: + s= (0,0,0) + else: + s= ( int(m.group(1)), int(m.group(2)), int(m.group(4)) ) + return s + +def cmp_rank_score( c1, c2 ): + s11= c1[0][0]*c1[0][1] + s12= c1[0][2] + s21= c2[0][0]*c2[0][1] + s22= c2[0][2] + if s11 > s21: return -1 + elif s11 == s21 and s12 > s22: return -1 + elif s11 < s21: return 1 + elif s11 == s21 and s12 < s22: return 1 + else: return 0 + +def readclusters( file ): + f= open(file,'r') + clusters= [] + cluster= [] + for line in f.readlines(): + if line == '\n' or line == '': + if len(cluster) > 0: + clusters.append((cluster_rank_score(cluster[0]), cluster)) + cluster=[] + continue + cluster.append(line) + f.close() + if len(cluster) > 0: + clusters.append((cluster_rank_score(cluster[0]), cluster)) + cluster=[] + return clusters + +clusters= readclusters( sys.argv[1] ) +#clusters= map( remove_overlaping, clusters ) +#clusters= filter( only_one_diff, clusters ) +clusters.sort( cmp_rank_score ) + + +#print len(clusters), 'clusters' +#print +for (score, cluster) in clusters: + for line in cluster: + print line, + print + + diff --git a/scripts/bugdetect/bugtypecounting b/scripts/bugdetect/bugtypecounting new file mode 100644 index 0000000..c34c11e --- /dev/null +++ b/scripts/bugdetect/bugtypecounting @@ -0,0 +1,153 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# the regular expressions are copied from bugmerging +# Usage: $0 +# output to stdout + +import re +import sys +import os +#import shutil + +if len(sys.argv) != 2: + print "Usage: ", sys.argv[0], " " + sys.exit(1) + +rs= re.compile('.*Rank score:\s*(-?\d+)\s*\*\s*(\d+)\s*=\s*(-?\d+)\s*buggy score:\s*(-?\d+).*') +#rs= re.compile('.*Rank score:\s*(-?\d+)\s*\*\s*(\d+)\s*=\s*(-?\d+)\s*buggy score:\s*(-?\d+)\s(-?\d+)\s(-?\d+)\s(-?\d+).*') +range_r= re.compile('.*FILE (.*) LINE:(\d+):(\d+) NODE_KIND.*') +bugcategory_r= re.compile('(^B|^S|^G|^F)(\d+)\s*.*') +onlybug_r= re.compile('^B.*') +onlystyle_r= re.compile('^S.*') +def readclusters( file ): + f= open(file,'r') + clusters= [] + cluster= [] + for line in f.readlines(): + if line == '\n' or line == '': + if len(cluster) > 0: + clusters.append(cluster) + cluster=[] + continue + cluster.append(line) + f.close() + if len(cluster) > 0: + clusters.append(cluster) + return clusters + +def print_cluster(cluster): + for line in cluster: + print line, + +def print_clusters(clusters): + for cluster in clusters: + print_cluster(cluster) + print + +def buggy_clusters( clusters ): + newlist= [] + for i in range(len(clusters)): + keep= False + for j in range(len(clusters[i])): + m= bugcategory_r.match(clusters[i][j]) + if m == None: + continue + else: + keep= True + if keep: + newlist.append(clusters[i]) + return newlist + +# This function currently can't handle the cases when one cluster contains several lines of Rank score. +def rank_buggy_clusters( rankid, clusters ): + newlist= [] + for i in range(len(clusters)): + keep= False + rank= 0 + marks= [] + for j in range(len(clusters[i])): + rs_m= rs.match(clusters[i][j]) + if rs_m != None: + rank= int(rs_m.group(1)) + m= onlybug_r.match(clusters[i][j]) + if m != None: + marks.append(clusters[i][j]) + if len(marks)>0 and rank==rankid: + keep= True + if keep: + newlist.append(clusters[i]) + return newlist + +def type_buggy_clusters( typeid, clusters ): + newlist= [] + typeids= {} + if typeid==1: + typeids= set([1, 3, 5, 7]) + elif typeid==2: + typeids= set([2, 3, 6, 7]) + elif typeid==3: + typeids= set([4, 5, 6, 7]) + for i in range(len(clusters)): + keep= False + rank= 0 + marks= [] + for j in range(len(clusters[i])): + rs_m= rs.match(clusters[i][j]) + if rs_m != None: + rank= int(rs_m.group(1)) + m= onlybug_r.match(clusters[i][j]) + if m != None: + marks.append(clusters[i][j]) + if len(marks)>0 and rank in typeids: + keep= True + if keep: + newlist.append(clusters[i]) + return newlist + +clustersall= readclusters( sys.argv[1] ) +rankclusters= [] +for i in range(1,8): + rankclusters.append( rank_buggy_clusters(i, clustersall) ) +sum= 0 +for i in range(7): + sum += len(rankclusters[i]) + print len(rankclusters[i]), +print +print "Total bugs: ", sum +print "Type 1 bugs: ", len(rankclusters[0])+len(rankclusters[2])+len(rankclusters[4])+len(rankclusters[6]) +print "Type 2 bugs: ", len(rankclusters[1])+len(rankclusters[2])+len(rankclusters[5])+len(rankclusters[6]) +print "Type 3 bugs: ", len(rankclusters[3])+len(rankclusters[4])+len(rankclusters[5])+len(rankclusters[6]) + diff --git a/scripts/bugdetect/mergecomments b/scripts/bugdetect/mergecomments new file mode 100644 index 0000000..eb6d6a8 --- /dev/null +++ b/scripts/bugdetect/mergecomments @@ -0,0 +1,178 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# copied from bugmerging +# Usage: $0 +# output to stdout + +import re +import sys +import os +#import shutil + +if len(sys.argv) != 3: + print "Usage: ", sys.argv[0], " " + sys.exit(1) + +rs= re.compile('.*Rank score:\s*(-?\d+)\s*\*\s*(\d+)\s*=\s*(-?\d+)\s*buggy score:\s*(-?\d+).*') +#rs= re.compile('.*Rank score:\s*(-?\d+)\s*\*\s*(\d+)\s*=\s*(-?\d+)\s*buggy score:\s*(-?\d+)\s(-?\d+)\s(-?\d+)\s(-?\d+).*') +range_r= re.compile('.*FILE (.*) LINE:(\d+):(\d+) NODE_KIND.*') +bugcategory_r= re.compile('(^B|^S|^G|^F)(\d+)\s*.*') +bugflag_r= re.compile('(^\?|^Y|^N|^C).*') +def readclusters( file ): + f= open(file,'r') + clusters= [] + cluster= [] + for line in f.readlines(): + if line == '\n' or line == '': + if len(cluster) > 0: + clusters.append(cluster) + cluster=[] + continue + cluster.append(line) + f.close() + if len(cluster) > 0: + clusters.append(cluster) + return clusters + +def print_cluster(cluster): + for line in cluster: + print line, + +def print_clusters(clusters): + for cluster in clusters: + print_cluster(cluster) + print + +def cluster_lineranges( cluster ): + size= len(cluster) +# rsl[0]: rank score, its length should be <=1 +# rsl[1]: category marks, its length should be <=1 +# rsl[2]: the clones +# rsl[3]: the ranges of clones +# rsl[4]: bug marks, its length should be <=1 +# rsl[5]: other comments +# rsl[6]: the size of cluster + rsl=([], [], [], [], [], [], size) + for i in range(size): + rs_m= rs.match(cluster[i]) + if rs_m != None: + rsl[0].append(cluster[i]) + continue + bug_m= bugcategory_r.match(cluster[i]) + if bug_m != None: + rsl[1].append(cluster[i]) + continue + m= range_r.match(cluster[i]) + if m != None: + rsl[2].append(cluster[i]) + rsl[3].append( ( m.group(1), int(m.group(2)), int(m.group(2))+int(m.group(3))-1 ) ) + continue + flag_m= bugflag_r.match(cluster[i]) + if flag_m != None: + rsl[4].append(cluster[i]) + continue + rsl[5].append(cluster[i]) + return rsl + +def clear_clusters( clusters ): +# only keep clones: + newlist= [] + for i in range(len(clusters)): + newlines= [] + for j in range(len(clusters[i])): + m= range_r.match(clusters[i][j]) + if m == None: + continue + else: + newlines.append(clusters[i][j]) + if len(newlines)>0: + newlist.append(newlines) + return newlist + +def merge_clusters(c1, c2): + c1ranges= map(cluster_lineranges, c1) + c2ranges= map(cluster_lineranges, c2) + falseneg= 0 + for i in range(len(c1ranges)): + findmatch= -1 + for j in range(len(c2ranges)): + if len(c1ranges[i][3]) == len(c2ranges[j][3]): + matched= True + for k in range(len(c1ranges[i][3])): + linematched= False + for l in range(len(c2ranges[j][3])): + if c1ranges[i][3][k][2]<=c2ranges[j][3][l][2] and c1ranges[i][3][k][1]>=c2ranges[j][3][l][1] \ + and c1ranges[i][3][k][0]==c2ranges[j][3][l][0]: + linematched= True + break + else: + continue + if not linematched: + matched= False + break + if matched: + findmatch= j + break + if findmatch>=0: +# update c2: +# concatenate textual comments: + c2ranges[findmatch][5][:] = c2ranges[findmatch][5] + c1ranges[i][5] +# replace category marks if there is no in the c2: + if len(c2ranges[findmatch][1])<1: + c2ranges[findmatch][1][:] = c1ranges[i][1] +# replace bug marks if there is no in the c2: + if len(c2ranges[findmatch][4])<1: + c2ranges[findmatch][4][:] = c1ranges[i][4] + else: + falseneg = falseneg+1 + for line in c1ranges[i][0]+c1ranges[i][4]+c1ranges[i][1]+c1ranges[i][5]+c1ranges[i][2]: + print >> sys.stderr, line, + print >> sys.stderr + newlist=[] + for i in range(len(c2ranges)): + newlist.append(c2ranges[i][0]+c2ranges[i][4]+c2ranges[i][1]+c2ranges[i][5]+c2ranges[i][2]) + print >> sys.stderr, "Miss clusters: ", falseneg + return newlist + +clusters1= readclusters( sys.argv[1] ) +clusters2= readclusters( sys.argv[2] ) +#clusters2= clear_clusters( clusters2 ) +merged_clusters=merge_clusters(clusters1, clusters2) +# mark the output as auto generated: +print "Auto generated: merging ", len(clusters1), len(clusters2) +print_clusters(merged_clusters) + + diff --git a/scripts/clonedetect/cd_coverage b/scripts/clonedetect/cd_coverage new file mode 100644 index 0000000..077de24 --- /dev/null +++ b/scripts/clonedetect/cd_coverage @@ -0,0 +1,68 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +#import psyco +#psyco.full() + +import sys +f= open(sys.argv[1]) + +inputs=[] +import re +reg= re.compile(".*FILE (.*) LINE:(\d+):(\d+) NODE_KIND.*") +for line in f.readlines(): + m= reg.match(line) + if m != None: + l1= int(m.group(2)) + l2= int(m.group(3))+l1 + inputs.append( (m.group(1), (l1, l2)) ) + +fileCoverSet={} + +for file,linerange in inputs: + if linerange[0] > linerange[1]: + print "bad linerange:",linerange + continue + if file not in fileCoverSet: + fileCoverSet[file] = set() + fileCoverSet[file].update(range(linerange[0],linerange[1])) + +sum = 0 +for file,coverset in fileCoverSet.items(): + #print file,',',coverset + sum += len(coverset) + +print sum + diff --git a/scripts/clonedetect/config-sample b/scripts/clonedetect/config-sample new file mode 100644 index 0000000..7556256 --- /dev/null +++ b/scripts/clonedetect/config-sample @@ -0,0 +1,102 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +############################################################# +# Configuration file for clone detection. +# + +############################################################ +# Often, need to change these common parameters: +# - FILE_PATTERN : for source files in different languages +# - SRC_DIR : the root directory containing the source files +# - DECKARD_DIR : Where is the home directory of DECKARD +# - clone detection parameters: c.f. DECKARD's paper +# -- MIN_TOKENS +# -- STRIDE +# -- SIMILARITY +# +# java, c, or php? +FILE_PATTERN='*.java' # used for the 'find' command +# where are the source files? +SRC_DIR='src' +# where is Deckard? +DECKARD_DIR='/home/deckard' +# clone parameters; refer to paper. +MIN_TOKENS='50 100' # can be a sequence of integers +STRIDE='2 0' # can be a sequence of integers +SIMILARITY='1.0 0.95' # can be a sequence of values <= 1 +#DISTANCE='0 0.70711 1.58114 2.236' + +########################################################### +# Where to store result files? +# +# where to output generated vectors? +VECTOR_DIR='vectors' +# where to output detected clone clusters? +CLUSTER_DIR='clusters' +# where to output timing/debugging info? +TIME_DIR='times' + +########################################################## +# where are several programs we need? +# +# where is the vector generator? +VGEN_EXEC="$DECKARD_DIR/src/main" +case $FILE_PATTERN in + *.java ) + VGEN_EXEC="$VGEN_EXEC/jvecgen" ;; + *.php ) + VGEN_EXEC="$VGEN_EXEC/phpvecgen" ;; + *.c | *.h ) + VGEN_EXEC="$VGEN_EXEC/cvecgen" ;; + * ) + echo "Error: invalid FILE_PATTERN: $FILE_PATTERN" + VGEN_EXEC="$VGEN_EXEC/invalidvecgen" ;; +esac +# how to divide the vectors into groups? +GROUPING_EXEC="$DECKARD_DIR/src/vgen/vgrouping/runvectorsort" +# where is the lsh for vector clustering? +CLUSTER_EXEC="$DECKARD_DIR/src/lsh/bin/enumBuckets" +# how to post process clone groups? +POSTPRO_EXEC="$DECKARD_DIR/scripts/clonedetect/post_process_groupfile" +# how to transform source code html? +SRC2HTM_EXEC=source-highlight +SRC2HTM_OPTS=--line-number-ref + +################################################################## +# Some additional, internal parameters; can be ignored +# +# the maximal vector size for the first group; not really useful +GROUPING_S='50' # should be a single value +#GROUPING_D +#GROUPING_C + diff --git a/scripts/clonedetect/configure b/scripts/clonedetect/configure new file mode 100644 index 0000000..04cdceb --- /dev/null +++ b/scripts/clonedetect/configure @@ -0,0 +1,132 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +check_or_create_dir() +{ + if [[ ! -d "${!1}" ]]; then + if [[ -e "${!1}" ]]; then + echo "Error: $1 should be a directory. Check your config" + exit 1 + else + mkdir "${!1}" + if [[ $? -ne 0 ]]; then + echo "Error: $1 doesn't exist and cannot be created. Check your config" + exit 1 + fi + fi +fi +} + +check_file() +{ + if [[ ! -f $1 ]]; then + echo "Error: missing file $1. Check your config" + exit 1 + fi +} + +check_var() +{ + if [[ -z ${!1} ]]; + then + echo "Error: $1 not set in config. Check your config" + exit 1 + fi + + export $1 +} + +check_exe() +{ + if (which "$1" >& /dev/null); then + return 0 + else + echo "Error: missing required program: $1" + exit 1 + fi +} + +if [[ -e config ]]; +then + . config +else + echo "Error: no config file in current directory" + exit 0 +fi + +check_var "DECKARD_DIR" +check_var "FILE_PATTERN" +check_var "SRC_DIR" + +check_var "VECTOR_DIR" +check_or_create_dir "VECTOR_DIR" +check_var "CLUSTER_DIR" +check_or_create_dir "CLUSTER_DIR" +check_var "TIME_DIR" +check_or_create_dir "TIME_DIR" + +check_var "VGEN_EXEC" +check_file "$VGEN_EXEC" +check_var "GROUPING_EXEC" +check_file "$GROUPING_EXEC" +check_var "CLUSTER_EXEC" +check_file "$CLUSTER_EXEC" +check_var "POSTPRO_EXEC" +check_file "$POSTPRO_EXEC" + +check_var "MIN_TOKENS" +check_var "STRIDE" +#check_var "DISTANCE" +check_var "SIMILARITY" +check_var "GROUPING_S" +#check_var "GROUPING_D" +#check_var "GROUPING_C" + +check_exe "which" +check_exe "env" +check_exe "python" +check_exe "awk" +check_exe "sed" +check_exe "grep" +check_exe "find" +check_exe "bash" +check_exe "cat" +check_exe "touch" +check_exe "head" +check_exe "tail" +check_exe "time" + + + diff --git a/scripts/clonedetect/deckard.sh b/scripts/clonedetect/deckard.sh new file mode 100644 index 0000000..a2166b1 --- /dev/null +++ b/scripts/clonedetect/deckard.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +echo "DECKARD--A Tree-Based Code Clone Detection Toolkit. Version 1.0" +echo "Copyright (c) 2007-2010. University of California" +echo "Distributed under the three-clause BSD license." +echo + +echo -n "==== Configuration checking..." +. `dirname $0`/configure +errcode=$? +if [[ $errcode -eq 0 ]]; then + echo "Done." + echo +else + exit $errcode +fi + +TOOVERWRITE= +if [[ $# -ge 1 ]]; then + case "$1" in + clean ) + "`dirname $0`/vdbgen" clean + "`dirname $0`/vertical-param-batch" clean + exit $? + ;; + clean_all ) + "`dirname $0`/vdbgen" clean_all + "`dirname $0`/vertical-param-batch" clean_all + exit $? + ;; + overwrite ) + TOOVERWRITE="overwrite" + ;; + * ) + echo "Usage: $0 [overwrite | clean | clean_all]" + exit 1 + ;; + esac +fi + + + +echo "==== Start clone detection ====" +echo + +SRCLANG=${FILE_PATTERN##*.} +case $SRCLANG in + c | h | java | php ) + ;; + *) + echo "Error: language '$SRCLANG' not supported." + exit 1 ;; +esac + +echo -n "Vector generation..." +"`dirname $0`/vdbgen" $TOOVERWRITE +errcode=$? +if [[ $errcode -ne 0 ]]; then + echo "Error: problem in vec generator step. Stop and check logs in $TIME_DIR/" + exit $errcode +else + echo "Vector generation done. Logs in $TIME_DIR/vgen_*" + echo "Vector files in $VECTOR_DIR/vdb_*" + echo +fi + +echo "Vector clustering and filtering..." +"`dirname $0`/vertical-param-batch" $TOOVERWRITE +errcode=$? +if [[ $errcode -ne 0 ]]; then + echo "Error: problem in vec clustering step. Stop and check logs in $TIME_DIR/" + exit $errcode +fi + +echo "Clone detection done. Logs in $TIME_DIR/*" +echo "Clone reports in $VECTOR_DIR/post_cluster_*" +echo + +# Bug Finding: +echo "In addition, potential clone-related bugs may be produced by running the scripts:" +echo "(Be careful about the file overwriting and the choice for programming language)" + +echo "(1) search clone reports and find out suspicious ones: " +echo " \"$DECKARD_DIR/scripts/bugdetect/bugfiltering\" \"${CLUSTER_DIR}/post_\" $SRCLANG > \"${CLUSTER_DIR}/bug_\" 2> \"${TIME_DIR}/bugfiltering_\"" +echo "(2) transform the bug reports to html for easier investigation:" +echo " \"$DECKARD_DIR/src/main/out2html\" \"${CLUSTER_DIR}/bug_\" > \"${CLUSTER_DIR}/bug_.html\"" +echo +## # the actual commands for bug finding: +## find "$CLUSTER_DIR" -type f -name "post_cluster_vdb_*_*_allg_*_*" | while read cdb; +## do +## basecdb=`basename "$cdb"` +## basecdb=${basecdb#post_} +## ( time "`dirname $0`/../bugdetect/bugfiltering" "$cdb" $SRCLANG > "${CLUSTER_DIR}/bug_$basecdb" ) 2> "${TIME_DIR}/bugfiltering_$basecdb" +## "$DECKARD_DIR/src/main/out2html" "${CLUSTER_DIR}/bug_$basecdb" > "${CLUSTER_DIR}/bug_${basecdb}.html" +## done + +echo +echo "==== All Done for the current 'config' file ====" +echo + diff --git a/scripts/clonedetect/generateparam b/scripts/clonedetect/generateparam new file mode 100644 index 0000000..57314bc --- /dev/null +++ b/scripts/clonedetect/generateparam @@ -0,0 +1,48 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 " + exit 65 +fi + +head -n 2 "$1" +echo $3 +head -n 8 "$1" | tail -n 5 +echo $3 | awk '{printf("%.7g\n", $1*$1)}' +head -n 20 "$1" | tail -n 11 +echo $2 +tail -n 2 "$1" + diff --git a/scripts/clonedetect/paramsetting b/scripts/clonedetect/paramsetting new file mode 100644 index 0000000..7ea72f7 --- /dev/null +++ b/scripts/clonedetect/paramsetting @@ -0,0 +1,96 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: $1=, $2=, $3= + +# make sure ${GROUPING_S} and ${VECTOR_DIR} are set correctly. + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 " + exit 65 +fi + +# TODO: may check whether parameters have been set or not... + +t=$1 +s=$2 +sim=$3 +# convert SIMILARITY to DISTANCE: +i=`echo "$sim ${GROUPING_S}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` +find $VECTOR_DIR -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}" -not -name '*.param' -print0 | xargs -0 ls -S > tempfile +# not easy to pipe "head -n 1" with "ls", coz. often cause signal 13 (no more reader, while writer's still writing) +groupfortuning=`head -n 1 tempfile` +rm tempfile +grptuningid=`echo $vdb | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}/\1/"` +grpal=0 +grpdist=0 +# if $grptuningid is empty, error occurs... +if [[ $grptuningid -eq 0 ]]; then + grpal=`head -n $(expr $grptuningid + 1) $(dirname $vdb)/vdb_${t}_${s}_ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` +else + grpal=`head -n $(expr $grptuningid + 1) $(dirname $vdb)/vdb_${t}_${s}_ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'` +fi +echo "Looking for local optimal parameters by clustering $groupfortuning $grpdist ..." +cluster $groupfortuning $grpdist + +if [[ ! -s ${groupfortuning}.param ]]; then + echo "Parameter tuning failed ... exit ..." + exit 3 +fi + +echo "Setting Parameters for all other groups..." +find $VECTOR_DIR -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}" -not -name '*.param' | while read vdb; +#NOTE: trouble with filenames with spaces: +#for vdb in `find $VECTOR_DIR -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}" -not -name '*.param'`; + do + grpfileid=`echo "$vdb" | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}/\1/"` + grpal=0 + grpdist=0 + if [[ $grptuningid -eq $grpfileid ]]; then + continue + elif [[ $grpfileid -eq 0 ]]; then + grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/vdb_${t}_${s}_ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` + else + grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/vdb_${t}_${s}_ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'` + fi + lineno=`wc -l "$vdb" | awk '{print $1}'` + lineno=$(($lineno / 2)) + `dirname $0`/generateparam ${groupfortuning}.param $lineno $grpdist > "${vdb}.param" +done + diff --git a/scripts/clonedetect/post_process_groupfile b/scripts/clonedetect/post_process_groupfile new file mode 100644 index 0000000..9c5a31a --- /dev/null +++ b/scripts/clonedetect/post_process_groupfile @@ -0,0 +1,248 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +import re +import sys + +if len(sys.argv) != 2: + print "Usage: ", sys.argv[0], " " + sys.exit(1) + +#required_lines= int(sys.argv[2]) +el= re.compile('.*:(\d+) NODE_KIND.*') +def vector_linecount( vector ): + m= el.match(vector) + if m == None: + return -1 + return int( m.group(1) ) + +#NOTE: filenames can contain spaces... +range_r= re.compile('.*FILE (.*) LINE:(\d+):(\d+) NODE_KIND.*') +def vector_linerange( vector ): + m= range_r.match(vector) + if m == None: + return (-1,-1,-1) + return ( int(m.group(2)), int(m.group(2))+int(m.group(3))-1, m.group(1) ) + + +vc= re.compile('.*nVARs:(\d+) NUM_NODE.*') +def vector_varcount( vector ): + m= vc.match(vector) + if m == None: + return -1 + return int( m.group(1) ) + + +def has_enough_lines( cluster ): + for vector in cluster: + lines= vector_linecount( vector ) + if lines >= required_lines: + return True + return False + +def line_diff_big( cluster ): + maxl= reduce( lambda x,y: max(x,y), map( vector_linecount, cluster )) + minl= reduce( lambda x,y: min(x,y), map( vector_linecount, cluster )) + if maxl>minl+5: + return False + return True + +def count_duplicates( l, i ): + n= 0 + for item in l: + if i==item: + n+=1 + return n + +def only_one_diff( cluster ): + counts= map( vector_varcount, cluster ) + counts.sort() + if counts[-1] - counts[0] > 2: + return False + front= count_duplicates(counts,counts[0]) + back= count_duplicates(counts,counts[-1]) + if front < len(counts) and front >= len(counts)-2: + return True + if back < len(counts) and back >= len(counts)-2: + return True + return False + +def remove_staggered( cluster ): + rs= map( vector_linerange, cluster ) + keep= map( lambda x: True, cluster ) + for i in range(len(rs)-1): + for j in range(i+1,len(rs)): + if rs[j][2] == rs[i][2] and rs[j][0] < rs[i][1] \ + and rs[j][0] > rs[i][0] and rs[j][1] > rs[i][1] \ + and (rs[i][1]-rs[j][0])*2 >= rs[i][1]-rs[i][0] \ + and (rs[j][1]-rs[i][1])*2 <= rs[i][1]-rs[i][0]: + keep[j]= False + newlist= [] + for i in range(len(cluster)): + if keep[i]: + newlist.append(cluster[i]) + return newlist + +def remove_overlaping( cluster ): + rs= map( vector_linerange, cluster ) + keep= map( lambda x: True, cluster ) + for i in range(len(rs)-1): + for j in range(i+1,len(rs)): + if rs[j][2] == rs[i][2] and rs[j][0] < rs[i][1] \ + and rs[j][0] > rs[i][0] and rs[j][1] > rs[i][1]: \ + keep[j]= False + newlist= [] + for i in range(len(cluster)): + if keep[i]: + newlist.append(cluster[i]) + return newlist + +def cluster_lineranges( cluster ): + size= len(cluster) + rsl=([], [], size) + for i in range(size): + m= range_r.match(cluster[i]) + if m != None: + rsl[0].append( (int(m.group(2)), int(m.group(2))+int(m.group(3))-1) ) + rsl[1].append( m.group(1) ) + return rsl + +def remove_contained_across_clusters ( clusters ): +#should have no effect on cd_coverage: compare each range and filename (luckily sorted by filenames) +#take quadratic time (from seconds to minutes; range trees will perform better), but it can help to reduce re-parsing time (about 1/3, several minutes) + rs= map( cluster_lineranges, clusters ) + keep= map( lambda x: True, clusters ) + for i in range(len(rs)): + for j in range(len(rs)): + if not keep[i] or not keep[j] or i == j: + continue + if rs[j][2] == rs[i][2] and len(rs[j][1]) == len(rs[i][1]): + filematch= True + for fn in range(len(rs[i][1])): + if rs[i][0][fn][0] >= rs[j][0][fn][0] and rs[i][0][fn][1] <= rs[j][0][fn][1] \ + and rs[j][1][fn] == rs[i][1][fn]: + continue + else: + filematch= False + break + if filematch: + keep[i]= False + newlist= [] + for i in range(len(clusters)): + if keep[i]: + newlist.append(clusters[i]) + return newlist + +def remove_contained( cluster ): + rs= map( vector_linerange, cluster ) + keep= map( lambda x: True, cluster ) + for i in range(len(rs)): + for j in range(len(rs)): + if not keep[i] or i == j or not keep[j]: + continue + if rs[j][2] == rs[i][2] \ + and rs[i][0] >= rs[j][0] and rs[i][1] <= rs[j][1]: + keep[i]= False + newlist= [] + for i in range(len(cluster)): + if keep[i]: + newlist.append(cluster[i]) + return newlist + +def not_empty( cluster ): + if len(cluster) <= 1: + return False + return True + +def cmp_linecount( c1, c2 ): + c1l= map( vector_linecount, c1 ) + c2l= map( vector_linecount, c2 ) + m1= reduce( max, c1l ) + m2= reduce( max, c2l ) + if m1 > m2: return -1 + if m1 == m2: return 0 + return 1 + +def readclusters( file ): +# print file + f= open(file,'r') + clusters= [] + cluster= [] + for line in f.readlines(): + if line == '\n' or line == '': + if len(cluster) > 0: + clusters.append(cluster) + cluster=[] + continue + cluster.append(line) + f.close() + if len(cluster) > 0: + clusters.append(cluster) +# clusters= clusters[1:] + return clusters + + +# read in all clone clusters from a file +clusters= readclusters( sys.argv[1] ) + +# remove clones completely contained in another clone in the same cluster +clusters= map( remove_contained, clusters ) + +# remove clones overlapped with another clone more than 50% +clusters= map( remove_staggered, clusters ) + +# remove clones overlapped with another clone more than 0% +# disabled: clusters= map( remove_overlaping, clusters ) + +# remove clones contained in another clone in another cluster +clusters= remove_contained_across_clusters(clusters) + +# some of the filtering may only be useful for bug detection purpose, which should and has been separated into another module +#clusters= filter( has_enough_lines , clusters ) +#clusters= filter( line_diff_big, clusters ) +#clusters= filter( only_one_diff, clusters ) + +# remove clusters containing less than 2 clones +clusters= filter( not_empty, clusters ) +# sort clusters by the cloned lines of code +clusters.sort( cmp_linecount ) + +# print the result clusters +for cluster in clusters: + for line in cluster: + print line, + print + + diff --git a/scripts/clonedetect/vdbgen b/scripts/clonedetect/vdbgen new file mode 100644 index 0000000..85ebbd7 --- /dev/null +++ b/scripts/clonedetect/vdbgen @@ -0,0 +1,159 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +TOOVERWRITE= + +clean() +{ + echo -n "...deleting intermediate vector files..." + find "$SRC_DIR" -iname '*.vec' -print0 | xargs -0 -r --max-args 500 rm -f + echo "Done" +} + +clean_log() +{ + echo -n "...deleting vgen logs..." + find "$TIME_DIR" -iname 'vgen_*' -print0 | xargs -0 -r --max-args 500 rm -f + echo "Done" + echo -n "...deleting vector db files..." + find "$VECTOR_DIR" -iname 'vdb_*' -print0 | xargs -0 -r --max-args 500 rm -f + echo "Done" +} + +#vgen $1 $2: $1=min_token; $2=stride +vgen() +{ + if (($# != 2)); + then + return 1 + fi + + echo -n "$FUNCNAME: $1 $2 ..." # FUNCNAME: the current function name; maintained by Bash + if [[ "$TOOVERWRITE" != "true" && -s "$VECTOR_DIR/vdb_$1_$2" ]]; + then + echo + echo "Warning: Vec file $VECTOR_DIR/vdb_$1_$2 exists. Delete it manually or use '$0 overwrite'." + echo "$FUNCNAME: $1 $2 ...Skip." + else + echo -n > "$TIME_DIR/vgen_$1_$2" + +#NOTE: "for .. in .." has trouble when a filename contains spaces; use "while read .." instead: + ( time \ + find "$SRC_DIR" -iname "$FILE_PATTERN" | while read file; +# for file in `find $SRC_DIR/ -iname "$FILE_PATTERN"`; + do + echo "Parsing $file" >> "$TIME_DIR/vgen_$1_$2" + if [[ ! -s "${file}.vec" || "$TOOVERWRITE" = "true" ]]; + then + "$VGEN_EXEC" -i "$file" -m $1 -t $2 + fi + done \ + ) 1>>"$TIME_DIR/vgen_$1_$2" 2>&1 + + find "$SRC_DIR" -iname '*.vec' -print0 | xargs -0 --max-args=100 cat > "$VECTOR_DIR/vdb_$1_$2" + echo "Done. Log: $TIME_DIR/vgen_$1_$2" + fi + + if [[ -s "$VECTOR_DIR/vdb_$1_$2" ]]; then + return 0 + else + return 1 + fi +} + +# transform source code to html: +tohtml() +{ + if ( which "$SRC2HTM_EXEC" >& /dev/null ); + then + echo -n "Transforming source code to html..." + ( time \ + find $SRC_DIR/ -iname "$FILE_PATTERN" | while read file; + do + if [[ ! -e "${file}.html" || "$TOOVERWRITE" = "true" ]]; + then + echo "$SRC2HTM_EXEC $SRC2HTM_OPTS $file" + "$SRC2HTM_EXEC" "$SRC2HTM_OPTS" "$file" + fi + done \ + ) 1>>"$TIME_DIR/$SRC2HTM_EXEC.log" 2>&1 + echo "Done. Log: $TIME_DIR/$SRC2HTM_EXEC.log" + fi +} + +. `dirname $0`/configure + +if [[ $# -ge 1 ]]; then + case "$1" in + clean ) + clean + exit $? + ;; + clean_all ) + clean + clean_log + exit $? + ;; + overwrite ) + TOOVERWRITE="true" + ;; + * ) + echo "Usage: $0 [overwrite | clean | clean_all]" + exit 1 + ;; + esac +fi + +errcode=0 +echo "$VGEN_EXEC $FILE_PATTERN" +echo + +for t in ${MIN_TOKENS}; +do + for s in ${STRIDE}; + do + vgen $t $s + retcode=$? + if [[ $errcode -eq 0 ]]; then + errcode=$retcode + fi + clean # remove .vec so not to intervene with the following vgens + echo + done +done + +tohtml +exit $errcode + diff --git a/scripts/clonedetect/vertical-param-batch b/scripts/clonedetect/vertical-param-batch new file mode 100644 index 0000000..7f3a181 --- /dev/null +++ b/scripts/clonedetect/vertical-param-batch @@ -0,0 +1,434 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# To save disk space, this script does everything (clustering and +# post-processing) for each MIN_TOKENS, each STRIDE, and each SIMILARITY at +# once, and delete intermediate files (cluster_*, post_*, ) before starting +# processing the next parameter combination (thus the name of the scripts). +# Feel free to adjust the workflows in the scripts to fit your needs. + +TOOVERWRITE= +coverage=`dirname $0`/cd_coverage +GROUPING_H=100000 + +# Delete intermediate files, but leave times/*, cluster/coverage*, cluster/post_* files there: +clean() +{ + echo "Cleaning up all intermediate files..." + echo -n "...deleting vdb_*_*_g*_*_*, including parameter files ..." + find "$VECTOR_DIR" -type f -name "vdb_*_*_g[0-9]*_*_*" -print0 | xargs -0 -r --max-args=500 rm -f +# find "$VECTOR_DIR" -type f -name "vdb_*_*_g[0-9]*_*_${GROUPING_S}" -print0 | xargs -0 -r --max-args=500 rm -f +# also delete parameter files, coz. the number of files is too big to be handled by certain OS: +# find "$VECTOR_DIR" -type f -name "vdb_*_*_g[0-9]*_*_${GROUPING_S}.param" -print0 | xargs -0 -r --max-args=500 rm -f + echo "Done" + echo -n "...deleting cluster_vdb_*_*_g*_*_* ..." + find "$CLUSTER_DIR" -type f -name "cluster_vdb_*_*_g[0-9]*_*_*" -print0 | xargs -0 -r --max-args=500 rm -f + echo "Done" +} + +# Delete log files and result files: if no argv, delete all files +clean_log() +{ + echo "Cleaning up all results and log files..." + echo -n "...deleting cluster_vdb_*_*_allg_*_* ..." + find "$CLUSTER_DIR" -type f -name "cluster_vdb_*_*_allg_*_*" -print0 | xargs -0 -r rm -f + echo "Done" + echo -n "...deleting post_cluster_vdb_*_*_allg_*_* ..." + find "$CLUSTER_DIR" -type f -name "post_cluster_vdb_*_*_allg_*_*" -print0 | xargs -0 -r rm -f + echo "Done" + echo -n "...deleting ranges_*_* ..." + find "$VECTOR_DIR" -type f -name "ranges_*_*" -print0 | xargs -0 -r rm -f + echo "Done" + echo -n "...deleting logs..." + find "$TIME_DIR" -type f -name "grouping_*" -print0 | xargs -0 -r --max-args=500 rm -f + find "$TIME_DIR" -type f -name "paramsetting_*" -print0 | xargs -0 -r --max-args=500 rm -f + find "$TIME_DIR" -type f -name "merging_*" -print0 | xargs -0 -r --max-args=500 rm -f + find "$TIME_DIR" -type f -name "cluster_*" -print0 | xargs -0 -r --max-args=500 rm -f + find "$TIME_DIR" -type f -name "post_*" -print0 | xargs -0 -r --max-args=500 rm -f + find "$TIME_DIR" -type f -name "coverage_*" -print0 | xargs -0 -r --max-args=500 rm -f + echo "Done" +} + +# vector grouping: $1=, $2=, $3= +# group file names generated by rundispatchonefile: $VECTOR_DIR/vdb_$1_$2_g_$3_${GROUPING_S}_${GROUPING_H} +grouping() +{ + echo -n "$FUNCNAME: $VECTOR_DIR/vdb_$1_$2 with distance=$3..." | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" + # note for bash: must have the space before and after '!=' to make it work correctly; otherwise, silent errors. + if [[ "$TOOVERWRITE" != "true" ]]; then + togroup= + grpnumber=`find "$VECTOR_DIR" -type f -regex ".*vdb_$1_$2_g[0-9]+_$3_${GROUPING_S}.*" | wc -l` + if [[ $grpnumber -le 0 ]]; then + togroup="true" + else + while read grpf; do + if [[ ! -s "$grpf" || "$grpf" -ot "$VECTOR_DIR/vdb_$1_$2" ]]; then + togroup="true" + break + fi + done < <( find "$VECTOR_DIR" -regex ".*vdb_$1_$2_g[0-9]+_$3_${GROUPING_S}.*" ) + fi + if [[ "$togroup" != "true" ]]; then + echo "Warning: $VECTOR_DIR/vdb_$1_$2 has already been grouped for distance=$3 and starting range ${GROUPING_S}. Delete $VECTOR_DIR/vdb_$1_$2_g[0-9]* manually or use '$0 overwrite'." | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" + echo "$FUNCNAME: $1 $2 $3 ...Skip." | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" + return 0 + fi + fi + + # delete old vector groups, in case some files may not be overwritten: + find "$VECTOR_DIR/" -regex ".*vdb_$1_$2_g[0-9]+_$3_${GROUPING_S}.*" -print0 | xargs -0 -r --max-args=500 rm -f + ( time \ + "`dirname ${GROUPING_EXEC}`/rundispatchonefile" "$VECTOR_DIR/vdb_$1_$2" $3 -l $GROUPING_S -h $GROUPING_H -o "$VECTOR_DIR" -p "vdb_$1_$2" || (echo "Error: $FUNCNAME failure: $1 $2 $3"; exit 1) \ + ) 1>>"$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" 2>&1 + errcode=$? + if [[ $errcode -ne 0 ]]; then + echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" + else + echo "Done $FUNCNAME $1 $2 $3. See groups in $VECTOR_DIR/vdb_$1_$2_g[0-9]*_$3_${GROUPING_S}*" | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" + fi + return $errcode +} + +# set parameter files: $1=, $2=, $3= +paramsetting() +{ + t=$1 + s=$2 + sim=$3 + echo -n "paramsetting: $t $s $sim ..." | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" +# convert SIMILARITY to DISTANCE: + i=`echo "$sim ${GROUPING_S}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` +# create the range file if not exist: + if [[ "$TOOVERWRITE" = "true" || ! -s "${VECTOR_DIR}/ranges_${i}_${GROUPING_S}" ]]; then + "`dirname $GROUPING_EXEC`/computeranges" $i $GROUPING_S $GROUPING_H > "${VECTOR_DIR}/ranges_${i}_${GROUPING_S}" + fi + if [[ ! -s "${VECTOR_DIR}/ranges_${i}_${GROUPING_S}" ]]; then + echo "Error: $FUNCNAME failure: Can't generate the range file: ranges_${i}_${GROUPING_S}" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + exit 87 + fi +# use the largest group file for paramter tuning: +# "head -n 1" may break the pipe coming from "ls", causing signal 13 (no more reader, while writer's still writing), but should not matter here: + groupfortuning=`find $VECTOR_DIR -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" -not -name '*.param' -print0 | xargs -0 ls -S | head -n 1` + grptuningid=`echo ${groupfortuning} | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}.*/\1/"` + grpal=0 + grpdist=0 +# if $grptuningid is empty, error occurs... + if [[ $grptuningid -le 0 ]]; then + echo "Error: $FUNCNAME: invalid group id: $grptuningid: $t $s $sim $i" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + exit 127 + elif [[ $grptuningid -eq 1 ]]; then + grpal=`head -n $(expr $grptuningid + 1) ${VECTOR_DIR}/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` + else + grpal=`head -n $(expr $grptuningid + 1) ${VECTOR_DIR}/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'` + fi + echo -n "Looking for optimal parameters by " | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + cluster $groupfortuning $grpdist -c | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + errcode=$? + + if [[ $errcode -ne 0 || ! -s "${groupfortuning}.param" ]]; then + echo "Error: $FUNCNAME failure...exit." | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + exit 3 + fi + + echo -n "Setting Parameters for all other groups (may take hours on cygwin but only minutes on Linux...why?)..." | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + # TODO: performance improvements to reduce file I/O; replace head/tail/awk/generateparam + ( time \ + find "$VECTOR_DIR" -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" -not -name '*.param' | while read vdb; + do + echo "$FUNCNAME: ${vdb}.param" + grpfileid=`echo "$vdb" | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}.*/\1/"` + grpal=0 + grpdist=0 + if [[ ${grptuningid} -eq $grpfileid ]]; then + continue + elif [[ $grpfileid -eq 1 ]]; then + grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` + else + grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'` + fi + lineno=`wc -l "$vdb" | awk '{print $1}'` + lineno=$(($lineno / 2)) + "`dirname $0`/generateparam" "${groupfortuning}.param" $lineno $grpdist > "${vdb}.param" + done \ + ) 1>>"${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" 2>&1 + errcode=$? + + if [[ $errcode -ne 0 ]]; then + echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + else + echo "Done $FUNCNAME $t $s $sim. Log: $TIME_DIR/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" + fi + return $errcode +} + +# clustering: $1=, $2=, $3= +cluster() +{ + vdb="$1" + dist=$2 + vfile=`basename "$vdb"` + flag=$3 + + if [[ $3 != "-c" ]]; then + flag= + fi + + echo -n "Clustering '${vdb}' ${grpdist} ..." | tee -a "$TIME_DIR/cluster_${vfile}" + + if [[ "$TOOVERWRITE" != "true" && + -s "$CLUSTER_DIR/cluster_${vfile}_$2" && + "$CLUSTER_DIR/cluster_${vfile}_$2" -nt "$vdb" ]]; + then + echo "Warning: $CLUSTER_DIR/cluster_${vfile}_$2 exists...Skip." | tee -a "$TIME_DIR/cluster_${vfile}" + return 0 + fi + + # dumb (not flexible) memory limit setting + mem=`ls -l "$vdb" | awk '{printf("%.0f", $5/1024/1024+0.5)}'` + if [ $mem -lt 2 ]; then + mem=10000000 + elif [ $mem -lt 5 ]; then + mem=20000000 + elif [ $mem -lt 10 ]; then + mem=30000000 + elif [ $mem -lt 20 ]; then + mem=60000000 + elif [ $mem -lt 50 ]; then + mem=150000000 + elif [ $mem -lt 100 ]; then + mem=300000000 + elif [ $mem -lt 200 ]; then + mem=600000000 + elif [ $mem -lt 500 ]; then + mem=900000000 + elif [ $mem -lt 1024 ]; then + mem=1900000000 + elif [ $mem -lt 2048 ]; then + mem=3800000000 + else + echo "Error: Size of $vdb > 2G. I don't want to do it before you think of any optimization." | tee -a "$TIME_DIR/cluster_${vfile}" + exit 1; + fi + + # group vector files should have already in their names; parameter files should also exist: + if [[ $3 != "-c" && -s "${vdb}" && + ! -s "${vdb}.param" ]]; then + echo "Error: Parameter File missing for '$vdb' $dist...Exit" | tee -a "$TIME_DIR/cluster_${vfile}" + exit 65 + fi + ( time \ + "$CLUSTER_EXEC" -R $dist -M $mem -b 2 -A -f "$vdb" ${flag} -p "${vdb}.param" \ + > "$CLUSTER_DIR/cluster_${vfile}" \ + ) 1>>"$TIME_DIR/cluster_${vfile}" 2>&1 + errcode=$? + + if [[ $errcode -ne 0 ]]; then + echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/cluster_${vfile}" | tee -a "$TIME_DIR/cluster_${vfile}" + else + echo "Done clustering '${vdb}' ${grpdist}. Log: $TIME_DIR/cluster_${vfile}" | tee -a "$TIME_DIR/cluster_${vfile}" + fi + return $errcode +} + +# Post-processing and count clone line #: +pcluster() +{ + cdb="$1" + cfile=`basename "$cdb"` +# Count clone line # before post-processing: +# echo "Count clone lines for '$cdb' ..." +# ( time \ +# "$coverage" "$cdb" > "${CLUSTER_DIR}/coverage_${cfile}" \ +# ) >& "${TIME_DIR}/coverage_${cfile}" +# Post-processing: + echo -n "Post processing $cdb ..." | tee -a "${TIME_DIR}/post_${cfile}" + if [[ $(wc -l "$cdb" | awk '{print $1}') -lt 2 ]]; + then + echo "Warning: clone cluster file is too small to be right: $cdb...Skip." | tee -a "${TIME_DIR}/post_${cfile}" + return 1 + fi + + if [[ "$TOOVERWRITE" != "true" && + -s "$CLUSTER_DIR/post_${cfile}" && + "$CLUSTER_DIR/post_${cfile}" -nt "$cdb" ]]; + then + echo "Warning: $CLUSTER_DIR/post_${cfile} exists...Skip. May manually delete it or set 'overwrite'" | tee -a "${TIME_DIR}/post_${cfile}" + return 0 + else + ( time \ + "$POSTPRO_EXEC" "$cdb" > "${CLUSTER_DIR}/post_${cfile}" \ + ) 1>>"${TIME_DIR}/post_${cfile}" 2>&1 + errcode=$? + if [[ $errcode -ne 0 ]]; then + echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/post_${cfile}" | tee -a "${TIME_DIR}/post_${cfile}" + else + echo "Done post-processing $cdb. Log: $TIME_DIR/post_${cfile}" | tee -a "${TIME_DIR}/post_${cfile}" + fi + return $errcode + fi +# Count clone line # after post-processing: +# echo "Count clone lines for $CLUSTER_DIR/post_${cfile} ..." +# ( time \ +# "$coverage" "${CLUSTER_DIR}/post_${cfile}" > "${CLUSTER_DIR}/coverage_post_${cfile}" \ +# ) 2& "${TIME_DIR}/coverage_post_${cfile}" + + return 0 +} + + +. `dirname $0`/configure + +if [[ $# -ge 1 ]]; then + case "$1" in + clean ) + clean + exit $? + ;; + clean_all ) + clean + clean_log + exit $? + ;; + overwrite ) + TOOVERWRITE="true" + ;; + * ) + echo "Usage: $0 [overwrite | clean | clean_all]" + exit 1 + ;; + esac +fi + +errcode=0 +for t in ${MIN_TOKENS}; +do + for s in ${STRIDE}; + do + for sim in $SIMILARITY; + do + echo "Vector clustering w/ MIN_TOKENS=$t, STRIDE=$s, SIMILARITY=$sim ..." + echo + +# convert SIMILARITY to DISTANCE used by LSH: +# (1) always round up; (2) different for different code sizes; (3) require 'awk' for sqrt + i=`echo "$sim ${GROUPING_S}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` +# Grouping: + grouping $t $s $i + errcode=$? + # NOTE: + # range file name: ranges_${i}_${GROUPING_S} + # group file names: vdb_${t}_${s}_g[0-9]+_${i}_${GROUPING_S} +# Set parameter files for LSH: + paramsetting $t $s $sim + errcode=$? +# Clustering: + echo "Cluster every vector groups..." | tee "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}" + find "$VECTOR_DIR" -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" -not -name '*.param' | while read vdb; + do + grpfileid=`echo "$vdb" | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}.*/\1/"` + grpal=0 + grpdist=0 + # TODO: what are better distance parameters for the groups? + if [[ $grpfileid -le 0 ]]; then + echo "Warning: invalid group id: $grpfileid in group $vdb...Skip." | tee -a "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}" + continue + elif [[ $grpfileid -eq 1 ]]; then + grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'` + else + grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'` + grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'` + fi + cluster "$vdb" $grpdist 1>>"$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}" 2>&1 + errcode=$? + if [[ $errcode -ne 0 ]]; then + echo "Clustering: Possible errors for vector group: $vdb $grpdist. Check log: $TIME_DIR/cluster_$(basename "${vdb}")" | tee -a "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}" + fi + done + echo "Done clustering. Check log: $TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S} and $TIME_DIR/cluster_vdb_${t}_${s}_g[0-9]+_${i}_${GROUPING_S}" | tee -a "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}" +# Merging: + echo -n "Merging all clone reports into $CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S} ..." | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" + tomerge= + if [[ "$TOOVERWRITE" = "true" ]]; then + tomerge="true" + elif [[ ! -s "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" ]]; then + tomerge="true" + else + while read grpf; do + if [[ "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" -ot "$grpf" ]]; then + tomerge="true" + break + fi + done < <( find "$CLUSTER_DIR" -type f -name "cluster_vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" ) + fi + if [[ "$tomerge" != "true" ]]; then + echo "Warning: '$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}' exists...Skip." | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" + else + # Continue to merge: + echo -n > "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" + ( time \ + find "$CLUSTER_DIR" -type f -name "cluster_vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" | while read grpf; + do + # remove some output lines in each file outputted by LSH/enumBuckets: + lineno=`wc -l $grpf | awk '{print $1;}'` + head -n `expr $lineno - 2` $grpf | tail -n `expr $lineno - 11` >> "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" + done \ + ) 1>>"${TIME_DIR}/merging_${t}_${s}_${sim}_${GROUPING_S}" 2>&1 + errcode=$? + if [[ $errcode -ne 0 ]]; then + echo "Merging: Possible errors occurred. Check log: $TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" + else + echo "Done merging $CLUSTER_DIR/cluster_vdb_${t}_${s}_all_${sim}_${GROUPING_S}. Log: $TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" + fi + fi +# Post-processing: + pcluster "${CLUSTER_DIR}/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" + errcode=$? +# Clean up intermediate files: + clean +# Bug Finding: done in main.sh instead. + + echo + echo "Done clustering: ${t} ${s} $sim. See clone reports in $CLUSTER_DIR/post_cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" + echo + done + done +done + + diff --git a/src/examples/GridLayout.java b/src/examples/GridLayout.java new file mode 100644 index 0000000..ffdf27e --- /dev/null +++ b/src/examples/GridLayout.java @@ -0,0 +1,742 @@ +/******************************************************************************* + * Copyright (c) 2000, 2006 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package org.eclipse.swt.layout; + +import org.eclipse.swt.*; +import org.eclipse.swt.graphics.*; +import org.eclipse.swt.widgets.*; + +/** + * Instances of this class lay out the control children of a + * Composite in a grid. + *

+ * GridLayout has a number of configuration fields, and the + * controls it lays out can have an associated layout data object, called + * GridData. The power of GridLayout lies in the + * ability to configure GridData for each control in the layout. + *

+ *

+ * The following code creates a shell managed by a GridLayout + * with 3 columns: + * 

+ * 		Display display = new Display();
+ * 		Shell shell = new Shell(display);
+ * 		GridLayout gridLayout = new GridLayout();
+ * 		gridLayout.numColumns = 3;
+ * 		shell.setLayout(gridLayout);
+ *
+ * The numColumns field is the most important field in a + * GridLayout. Widgets are laid out in columns from left + * to right, and a new row is created when numColumns + 1 + * controls are added to the Composite. + *

+ * + * @see GridData + */ +public final class GridLayout extends Layout { + + /** + * numColumns specifies the number of cell columns in the layout. + * If numColumns has a value less than 1, the layout will not + * set the size and postion of any controls. + * + * The default value is 1. + */ + public int numColumns = 1; + + /** + * makeColumnsEqualWidth specifies whether all columns in the layout + * will be forced to have the same width. + * + * The default value is false. + */ + public boolean makeColumnsEqualWidth = false; + + /** + * marginWidth specifies the number of pixels of horizontal margin + * that will be placed along the left and right edges of the layout. + * + * The default value is 5. + */ + public int marginWidth = 5; + + /** + * marginHeight specifies the number of pixels of vertical margin + * that will be placed along the top and bottom edges of the layout. + * + * The default value is 5. + */ + public int marginHeight = 5; + + /** + * marginLeft specifies the number of pixels of horizontal margin + * that will be placed along the left edge of the layout. + * + * The default value is 0. + * + * @since 3.1 + */ + public int marginLeft = 0; + + /** + * marginTop specifies the number of pixels of vertical margin + * that will be placed along the top edge of the layout. + * + * The default value is 0. + * + * @since 3.1 + */ + public int marginTop = 0; + + /** + * marginRight specifies the number of pixels of horizontal margin + * that will be placed along the right edge of the layout. + * + * The default value is 0. + * + * @since 3.1 + */ + public int marginRight = 0; + + /** + * marginBottom specifies the number of pixels of vertical margin + * that will be placed along the bottom edge of the layout. + * + * The default value is 0. + * + * @since 3.1 + */ + public int marginBottom = 0; + + /** + * horizontalSpacing specifies the number of pixels between the right + * edge of one cell and the left edge of its neighbouring cell to + * the right. + * + * The default value is 5. + */ + public int horizontalSpacing = 5; + + /** + * verticalSpacing specifies the number of pixels between the bottom + * edge of one cell and the top edge of its neighbouring cell underneath. + * + * The default value is 5. + */ + public int verticalSpacing = 5; + +/** + * Constructs a new instance of this class. + */ +public GridLayout () {} + +/** + * Constructs a new instance of this class given the + * number of columns, and whether or not the columns + * should be forced to have the same width. + * If numColumns has a value less than 1, the layout will not + * set the size and postion of any controls. + * + * @param numColumns the number of columns in the grid + * @param makeColumnsEqualWidth whether or not the columns will have equal width + * + * @since 2.0 + */ +public GridLayout (int numColumns, boolean makeColumnsEqualWidth) { + this.numColumns = numColumns; + this.makeColumnsEqualWidth = makeColumnsEqualWidth; +} + +protected Point computeSize (Composite composite, int wHint, int hHint, boolean flushCache) { + Point size = layout (composite, false, 0, 0, wHint, hHint, flushCache); + if (wHint != SWT.DEFAULT) size.x = wHint; + if (hHint != SWT.DEFAULT) size.y = hHint; + return size; +} + +protected boolean flushCache (Control control) { + Object data = control.getLayoutData (); + if (data != null) ((GridData) data).flushCache (); + return true; +} + +GridData getData (Control [][] grid, int row, int column, int rowCount, int columnCount, boolean first) { + Control control = grid [row] [column]; + if (control != null) { + GridData data = (GridData) control.getLayoutData (); + int hSpan = Math.max (1, Math.min (data.horizontalSpan, columnCount)); + int vSpan = Math.max (1, data.verticalSpan); + int i = first ? row + vSpan - 1 : row - vSpan + 1; + int j = first ? column + hSpan - 1 : column - hSpan + 1; + if (0 <= i && i < rowCount) { + if (0 <= j && j < columnCount) { + if (control == grid [i][j]) return data; + } + } + } + return null; +} + +protected void layout (Composite composite, boolean flushCache) { + Rectangle rect = composite.getClientArea (); + layout (composite, true, rect.x, rect.y, rect.width, rect.height, flushCache); +} + +Point layout (Composite composite, boolean move, int x, int y, int width, int height, boolean flushCache) { + if (numColumns < 1) { + return new Point (marginLeft + marginWidth * 2 + marginRight, marginTop + marginHeight * 2 + marginBottom); + } + Control [] children = composite.getChildren (); + int count = 0; + for (int i=0; i 0) { + if (data.cacheWidth < data.minimumWidth) { + int trim = 0; + //TEMPORARY CODE + if (child instanceof Scrollable) { + Rectangle rect = ((Scrollable) child).computeTrim (0, 0, 0, 0); + trim = rect.width; + } else { + trim = child.getBorderWidth () * 2; + } + data.cacheWidth = data.cacheHeight = SWT.DEFAULT; + data.computeSize (child, Math.max (0, data.minimumWidth - trim), data.heightHint, false); + } + } + if (data.grabExcessVerticalSpace && data.minimumHeight > 0) { + data.cacheHeight = Math.max (data.cacheHeight, data.minimumHeight); + } + } + + /* Build the grid */ + int row = 0, column = 0, rowCount = 0, columnCount = numColumns; + Control [][] grid = new Control [4] [columnCount]; + for (int i=0; i= grid.length) { + Control [][] newGrid = new Control [lastRow + 4] [columnCount]; + System.arraycopy (grid, 0, newGrid, 0, grid.length); + grid = newGrid; + } + if (grid [row] == null) { + grid [row] = new Control [columnCount]; + } + while (column < columnCount && grid [row] [column] != null) { + column++; + } + int endCount = column + hSpan; + if (endCount <= columnCount) { + int index = column; + while (index < endCount && grid [row] [index] == null) { + index++; + } + if (index == endCount) break; + column = index; + } + if (column + hSpan >= columnCount) { + column = 0; + row++; + } + } + for (int j=0; j 1) { + int spanWidth = 0, spanMinWidth = 0, spanExpandCount = 0; + for (int k=0; k 0) { + if (makeColumnsEqualWidth) { + int equalWidth = (w + spanWidth) / hSpan; + int remainder = (w + spanWidth) % hSpan, last = -1; + for (int k = 0; k < hSpan; k++) { + widths [last=j-k] = Math.max (equalWidth, widths [j-k]); + } + if (last > -1) widths [last] += remainder; + } else { + if (spanExpandCount == 0) { + widths [j] += w; + } else { + int delta = w / spanExpandCount; + int remainder = w % spanExpandCount, last = -1; + for (int k = 0; k < hSpan; k++) { + if (expandColumn [j-k]) { + widths [last=j-k] += delta; + } + } + if (last > -1) widths [last] += remainder; + } + } + } + if (!data.grabExcessHorizontalSpace || data.minimumWidth != 0) { + w = !data.grabExcessHorizontalSpace || data.minimumWidth == SWT.DEFAULT ? data.cacheWidth : data.minimumWidth; + w += data.horizontalIndent - spanMinWidth - (hSpan - 1) * horizontalSpacing; + if (w > 0) { + if (spanExpandCount == 0) { + minWidths [j] += w; + } else { + int delta = w / spanExpandCount; + int remainder = w % spanExpandCount, last = -1; + for (int k = 0; k < hSpan; k++) { + if (expandColumn [j-k]) { + minWidths [last=j-k] += delta; + } + } + if (last > -1) minWidths [last] += remainder; + } + } + } + } + } + } + } + if (makeColumnsEqualWidth) { + int minColumnWidth = 0; + int columnWidth = 0; + for (int i=0; i 0; + widths [i] = columnWidth; + } + } else { + if (width != SWT.DEFAULT && expandCount > 0) { + int totalWidth = 0; + for (int i=0; i minWidths [j]) { + widths [last = j] = widths [j] + delta; + } else { + widths [j] = minWidths [j]; + expandColumn [j] = false; + c--; + } + } + } + if (last > -1) widths [last] += remainder; + + for (int j=0; j 1) { + if (!data.grabExcessHorizontalSpace || data.minimumWidth != 0) { + int spanWidth = 0, spanExpandCount = 0; + for (int k=0; k 0) { + if (spanExpandCount == 0) { + widths [j] += w; + } else { + int delta2 = w / spanExpandCount; + int remainder2 = w % spanExpandCount, last2 = -1; + for (int k = 0; k < hSpan; k++) { + if (expandColumn [j-k]) { + widths [last2=j-k] += delta2; + } + } + if (last2 > -1) widths [last2] += remainder2; + } + } + } + } + } + } + } + if (c == 0) break; + totalWidth = 0; + for (int i=0; i currentWidth)) { + int trim = 0; + if (child instanceof Scrollable) { + Rectangle rect = ((Scrollable) child).computeTrim (0, 0, 0, 0); + trim = rect.width; + } else { + trim = child.getBorderWidth () * 2; + } + data.cacheWidth = data.cacheHeight = SWT.DEFAULT; + data.computeSize (child, Math.max (0, currentWidth - trim), data.heightHint, false); + if (data.grabExcessVerticalSpace && data.minimumHeight > 0) { + data.cacheHeight = Math.max (data.cacheHeight, data.minimumHeight); + } + if (flush == null) flush = new GridData [count]; + flush [flushLength++] = data; + } + } + } + } + } + } + + /* Row heights */ + int availableHeight = height - verticalSpacing * (rowCount - 1) - (marginTop + marginHeight * 2 + marginBottom); + expandCount = 0; + int [] heights = new int [rowCount]; + int [] minHeights = new int [rowCount]; + boolean [] expandRow = new boolean [rowCount]; + for (int i=0; i 1) { + int spanHeight = 0, spanMinHeight = 0, spanExpandCount = 0; + for (int k=0; k 0) { + if (spanExpandCount == 0) { + heights [i] += h; + } else { + int delta = h / spanExpandCount; + int remainder = h % spanExpandCount, last = -1; + for (int k = 0; k < vSpan; k++) { + if (expandRow [i-k]) { + heights [last=i-k] += delta; + } + } + if (last > -1) heights [last] += remainder; + } + } + if (!data.grabExcessVerticalSpace || data.minimumHeight != 0) { + h = !data.grabExcessVerticalSpace || data.minimumHeight == SWT.DEFAULT ? data.cacheHeight : data.minimumHeight; + h += data.verticalIndent - spanMinHeight - (vSpan - 1) * verticalSpacing; + if (h > 0) { + if (spanExpandCount == 0) { + minHeights [i] += h; + } else { + int delta = h / spanExpandCount; + int remainder = h % spanExpandCount, last = -1; + for (int k = 0; k < vSpan; k++) { + if (expandRow [i-k]) { + minHeights [last=i-k] += delta; + } + } + if (last > -1) minHeights [last] += remainder; + } + } + } + } + } + } + } + if (height != SWT.DEFAULT && expandCount > 0) { + int totalHeight = 0; + for (int i=0; i minHeights [i]) { + heights [last = i] = heights [i] + delta; + } else { + heights [i] = minHeights [i]; + expandRow [i] = false; + c--; + } + } + } + if (last > -1) heights [last] += remainder; + + for (int i=0; i 1) { + if (!data.grabExcessVerticalSpace || data.minimumHeight != 0) { + int spanHeight = 0, spanExpandCount = 0; + for (int k=0; k 0) { + if (spanExpandCount == 0) { + heights [i] += h; + } else { + int delta2 = h / spanExpandCount; + int remainder2 = h % spanExpandCount, last2 = -1; + for (int k = 0; k < vSpan; k++) { + if (expandRow [i-k]) { + heights [last2=i-k] += delta2; + } + } + if (last2 > -1) heights [last2] += remainder2; + } + } + } + } + } + } + } + if (c == 0) break; + totalHeight = 0; + for (int i=0; iswim3; + + out_8(&sw->select, RELAX); + if (sel & 8) + out_8(&sw->control_bis, SELECT); + else + out_8(&sw->control_bic, SELECT); + out_8(&sw->select, sel & CA_MASK); +} + +static void swim3_action(struct floppy_state *fs, int action) +{ + struct swim3 __iomem *sw = fs->swim3; + + swim3_select(fs, action); + udelay(1); + out_8(&sw->select, sw->select | LSTRB); + udelay(2); + out_8(&sw->select, sw->select & ~LSTRB); + udelay(1); +} + +static int swim3_readbit(struct floppy_state *fs, int bit) +{ + struct swim3 __iomem *sw = fs->swim3; + int stat; + + swim3_select(fs, bit); + udelay(1); + stat = in_8(&sw->status); + return (stat & DATA) == 0; +} + +static void do_fd_request(request_queue_t * q) +{ + int i; + for(i=0;istate == idle && fs->wanted) { + fs->state = available; + wake_up(&fs->wait); + return; + } + while (fs->state == idle && (req = elv_next_request(swim3_queue))) { +#if 0 + printk("do_fd_req: dev=%s cmd=%d sec=%ld nr_sec=%ld buf=%p\n", + req->rq_disk->disk_name, req->cmd, + (long)req->sector, req->nr_sectors, req->buffer); + printk(" errors=%d current_nr_sectors=%ld\n", + req->errors, req->current_nr_sectors); +#endif + + if (req->sector < 0 || req->sector >= fs->total_secs) { + end_request(req, 0); + continue; + } + if (req->current_nr_sectors == 0) { + end_request(req, 1); + continue; + } + if (fs->ejected) { + end_request(req, 0); + continue; + } + + if (rq_data_dir(req) == WRITE) { + if (fs->write_prot < 0) + fs->write_prot = swim3_readbit(fs, WRITE_PROT); + if (fs->write_prot) { + end_request(req, 0); + continue; + } + } + + /* Do not remove the cast. req->sector is now a sector_t and + * can be 64 bits, but it will never go past 32 bits for this + * driver anyway, so we can safely cast it down and not have + * to do a 64/32 division + */ + fs->req_cyl = ((long)req->sector) / fs->secpercyl; + x = ((long)req->sector) % fs->secpercyl; + fs->head = x / fs->secpertrack; + fs->req_sector = x % fs->secpertrack + 1; + fd_req = req; + fs->state = do_transfer; + fs->retries = 0; + + act(fs); + } +} + +static void set_timeout(struct floppy_state *fs, int nticks, + void (*proc)(unsigned long)) +{ + unsigned long flags; + + spin_lock_irqsave(&fs->lock, flags); + if (fs->timeout_pending) + del_timer(&fs->timeout); + fs->timeout.expires = jiffies + nticks; + fs->timeout.function = proc; + fs->timeout.data = (unsigned long) fs; + add_timer(&fs->timeout); + fs->timeout_pending = 1; + spin_unlock_irqrestore(&fs->lock, flags); +} + +static inline void scan_track(struct floppy_state *fs) +{ + struct swim3 __iomem *sw = fs->swim3; + + swim3_select(fs, READ_DATA_0); + in_8(&sw->intr); /* clear SEEN_SECTOR bit */ + in_8(&sw->error); + out_8(&sw->intr_enable, SEEN_SECTOR); + out_8(&sw->control_bis, DO_ACTION); + /* enable intr when track found */ + set_timeout(fs, HZ, scan_timeout); /* enable timeout */ +} + +static inline void seek_track(struct floppy_state *fs, int n) +{ + struct swim3 __iomem *sw = fs->swim3; + + if (n >= 0) { + swim3_action(fs, SEEK_POSITIVE); + sw->nseek = n; + } else { + swim3_action(fs, SEEK_NEGATIVE); + sw->nseek = -n; + } + fs->expect_cyl = (fs->cur_cyl >= 0)? fs->cur_cyl + n: -1; + swim3_select(fs, STEP); + in_8(&sw->error); + /* enable intr when seek finished */ + out_8(&sw->intr_enable, SEEK_DONE); + out_8(&sw->control_bis, DO_SEEK); + set_timeout(fs, 3*HZ, seek_timeout); /* enable timeout */ + fs->settle_time = 0; +} + +static inline void init_dma(struct dbdma_cmd *cp, int cmd, + void *buf, int count) +{ + st_le16(&cp->req_count, count); + st_le16(&cp->command, cmd); + st_le32(&cp->phy_addr, virt_to_bus(buf)); + cp->xfer_status = 0; +} + +static inline void setup_transfer(struct floppy_state *fs) +{ + int n; + struct swim3 __iomem *sw = fs->swim3; + struct dbdma_cmd *cp = fs->dma_cmd; + struct dbdma_regs __iomem *dr = fs->dma; + + if (fd_req->current_nr_sectors <= 0) { + printk(KERN_ERR "swim3: transfer 0 sectors?\n"); + return; + } + if (rq_data_dir(fd_req) == WRITE) + n = 1; + else { + n = fs->secpertrack - fs->req_sector + 1; + if (n > fd_req->current_nr_sectors) + n = fd_req->current_nr_sectors; + } + fs->scount = n; + swim3_select(fs, fs->head? READ_DATA_1: READ_DATA_0); + out_8(&sw->sector, fs->req_sector); + out_8(&sw->nsect, n); + out_8(&sw->gap3, 0); + out_le32(&dr->cmdptr, virt_to_bus(cp)); + if (rq_data_dir(fd_req) == WRITE) { + /* Set up 3 dma commands: write preamble, data, postamble */ + init_dma(cp, OUTPUT_MORE, write_preamble, sizeof(write_preamble)); + ++cp; + init_dma(cp, OUTPUT_MORE, fd_req->buffer, 512); + ++cp; + init_dma(cp, OUTPUT_LAST, write_postamble, sizeof(write_postamble)); + } else { + init_dma(cp, INPUT_LAST, fd_req->buffer, n * 512); + } + ++cp; + out_le16(&cp->command, DBDMA_STOP); + out_8(&sw->control_bic, DO_ACTION | WRITE_SECTORS); + in_8(&sw->error); + out_8(&sw->control_bic, DO_ACTION | WRITE_SECTORS); + if (rq_data_dir(fd_req) == WRITE) + out_8(&sw->control_bis, WRITE_SECTORS); + in_8(&sw->intr); + out_le32(&dr->control, (RUN << 16) | RUN); + /* enable intr when transfer complete */ + out_8(&sw->intr_enable, TRANSFER_DONE); + out_8(&sw->control_bis, DO_ACTION); + set_timeout(fs, 2*HZ, xfer_timeout); /* enable timeout */ +} + +static void act(struct floppy_state *fs) +{ + for (;;) { + switch (fs->state) { + case idle: + return; /* XXX shouldn't get here */ + + case locating: + if (swim3_readbit(fs, TRACK_ZERO)) { + fs->cur_cyl = 0; + if (fs->req_cyl == 0) + fs->state = do_transfer; + else + fs->state = seeking; + break; + } + scan_track(fs); + return; + + case seeking: + if (fs->cur_cyl < 0) { + fs->expect_cyl = -1; + fs->state = locating; + break; + } + if (fs->req_cyl == fs->cur_cyl) { + printk("whoops, seeking 0\n"); + fs->state = do_transfer; + break; + } + seek_track(fs, fs->req_cyl - fs->cur_cyl); + return; + + case settling: + /* check for SEEK_COMPLETE after 30ms */ + fs->settle_time = (HZ + 32) / 33; + set_timeout(fs, fs->settle_time, settle_timeout); + return; + + case do_transfer: + if (fs->cur_cyl != fs->req_cyl) { + if (fs->retries > 5) { + end_request(fd_req, 0); + fs->state = idle; + return; + } + fs->state = seeking; + break; + } + setup_transfer(fs); + return; + + case jogging: + seek_track(fs, -5); + return; + + default: + printk(KERN_ERR"swim3: unknown state %d\n", fs->state); + return; + } + } +} + +static void scan_timeout(unsigned long data) +{ + struct floppy_state *fs = (struct floppy_state *) data; + struct swim3 __iomem *sw = fs->swim3; + + fs->timeout_pending = 0; + out_8(&sw->control_bic, DO_ACTION | WRITE_SECTORS); + out_8(&sw->select, RELAX); + out_8(&sw->intr_enable, 0); + fs->cur_cyl = -1; + if (fs->retries > 5) { + end_request(fd_req, 0); + fs->state = idle; + start_request(fs); + } else { + fs->state = jogging; + act(fs); + } +} + +static void seek_timeout(unsigned long data) +{ + struct floppy_state *fs = (struct floppy_state *) data; + struct swim3 __iomem *sw = fs->swim3; + + fs->timeout_pending = 0; + out_8(&sw->control_bic, DO_SEEK); + out_8(&sw->select, RELAX); + out_8(&sw->intr_enable, 0); + printk(KERN_ERR "swim3: seek timeout\n"); + end_request(fd_req, 0); + fs->state = idle; + start_request(fs); +} + +static void settle_timeout(unsigned long data) +{ + struct floppy_state *fs = (struct floppy_state *) data; + struct swim3 __iomem *sw = fs->swim3; + + fs->timeout_pending = 0; + if (swim3_readbit(fs, SEEK_COMPLETE)) { + out_8(&sw->select, RELAX); + fs->state = locating; + act(fs); + return; + } + out_8(&sw->select, RELAX); + if (fs->settle_time < 2*HZ) { + ++fs->settle_time; + set_timeout(fs, 1, settle_timeout); + return; + } + printk(KERN_ERR "swim3: seek settle timeout\n"); + end_request(fd_req, 0); + fs->state = idle; + start_request(fs); +} + +static void xfer_timeout(unsigned long data) +{ + struct floppy_state *fs = (struct floppy_state *) data; + struct swim3 __iomem *sw = fs->swim3; + struct dbdma_regs __iomem *dr = fs->dma; + struct dbdma_cmd *cp = fs->dma_cmd; + unsigned long s; + int n; + + fs->timeout_pending = 0; + out_le32(&dr->control, RUN << 16); + /* We must wait a bit for dbdma to stop */ + for (n = 0; (in_le32(&dr->status) & ACTIVE) && n < 1000; n++) + udelay(1); + out_8(&sw->intr_enable, 0); + out_8(&sw->control_bic, WRITE_SECTORS | DO_ACTION); + out_8(&sw->select, RELAX); + if (rq_data_dir(fd_req) == WRITE) + ++cp; + if (ld_le16(&cp->xfer_status) != 0) + s = fs->scount - ((ld_le16(&cp->res_count) + 511) >> 9); + else + s = 0; + fd_req->sector += s; + fd_req->current_nr_sectors -= s; + printk(KERN_ERR "swim3: timeout %sing sector %ld\n", + (rq_data_dir(fd_req)==WRITE? "writ": "read"), (long)fd_req->sector); + end_request(fd_req, 0); + fs->state = idle; + start_request(fs); +} + +static irqreturn_t swim3_interrupt(int irq, void *dev_id) +{ + struct floppy_state *fs = (struct floppy_state *) dev_id; + struct swim3 __iomem *sw = fs->swim3; + int intr, err, n; + int stat, resid; + struct dbdma_regs __iomem *dr; + struct dbdma_cmd *cp; + + intr = in_8(&sw->intr); + err = (intr & ERROR_INTR)? in_8(&sw->error): 0; + if ((intr & ERROR_INTR) && fs->state != do_transfer) + printk(KERN_ERR "swim3_interrupt, state=%d, dir=%x, intr=%x, err=%x\n", + fs->state, rq_data_dir(fd_req), intr, err); + switch (fs->state) { + case locating: + if (intr & SEEN_SECTOR) { + out_8(&sw->control_bic, DO_ACTION | WRITE_SECTORS); + out_8(&sw->select, RELAX); + out_8(&sw->intr_enable, 0); + del_timer(&fs->timeout); + fs->timeout_pending = 0; + if (sw->ctrack == 0xff) { + printk(KERN_ERR "swim3: seen sector but cyl=ff?\n"); + fs->cur_cyl = -1; + if (fs->retries > 5) { + end_request(fd_req, 0); + fs->state = idle; + start_request(fs); + } else { + fs->state = jogging; + act(fs); + } + break; + } + fs->cur_cyl = sw->ctrack; + fs->cur_sector = sw->csect; + if (fs->expect_cyl != -1 && fs->expect_cyl != fs->cur_cyl) + printk(KERN_ERR "swim3: expected cyl %d, got %d\n", + fs->expect_cyl, fs->cur_cyl); + fs->state = do_transfer; + act(fs); + } + break; + case seeking: + case jogging: + if (sw->nseek == 0) { + out_8(&sw->control_bic, DO_SEEK); + out_8(&sw->select, RELAX); + out_8(&sw->intr_enable, 0); + del_timer(&fs->timeout); + fs->timeout_pending = 0; + if (fs->state == seeking) + ++fs->retries; + fs->state = settling; + act(fs); + } + break; + case settling: + out_8(&sw->intr_enable, 0); + del_timer(&fs->timeout); + fs->timeout_pending = 0; + act(fs); + break; + case do_transfer: + if ((intr & (ERROR_INTR | TRANSFER_DONE)) == 0) + break; + out_8(&sw->intr_enable, 0); + out_8(&sw->control_bic, WRITE_SECTORS | DO_ACTION); + out_8(&sw->select, RELAX); + del_timer(&fs->timeout); + fs->timeout_pending = 0; + dr = fs->dma; + cp = fs->dma_cmd; + if (rq_data_dir(fd_req) == WRITE) + ++cp; + /* + * Check that the main data transfer has finished. + * On writing, the swim3 sometimes doesn't use + * up all the bytes of the postamble, so we can still + * see DMA active here. That doesn't matter as long + * as all the sector data has been transferred. + */ + if ((intr & ERROR_INTR) == 0 && cp->xfer_status == 0) { + /* wait a little while for DMA to complete */ + for (n = 0; n < 100; ++n) { + if (cp->xfer_status != 0) + break; + udelay(1); + barrier(); + } + } + /* turn off DMA */ + out_le32(&dr->control, (RUN | PAUSE) << 16); + stat = ld_le16(&cp->xfer_status); + resid = ld_le16(&cp->res_count); + if (intr & ERROR_INTR) { + n = fs->scount - 1 - resid / 512; + if (n > 0) { + fd_req->sector += n; + fd_req->current_nr_sectors -= n; + fd_req->buffer += n * 512; + fs->req_sector += n; + } + if (fs->retries < 5) { + ++fs->retries; + act(fs); + } else { + printk("swim3: error %sing block %ld (err=%x)\n", + rq_data_dir(fd_req) == WRITE? "writ": "read", + (long)fd_req->sector, err); + end_request(fd_req, 0); + fs->state = idle; + } + } else { + if ((stat & ACTIVE) == 0 || resid != 0) { + /* musta been an error */ + printk(KERN_ERR "swim3: fd dma: stat=%x resid=%d\n", stat, resid); + printk(KERN_ERR " state=%d, dir=%x, intr=%x, err=%x\n", + fs->state, rq_data_dir(fd_req), intr, err); + end_request(fd_req, 0); + fs->state = idle; + start_request(fs); + break; + } + fd_req->sector += fs->scount; + fd_req->current_nr_sectors -= fs->scount; + fd_req->buffer += fs->scount * 512; + if (fd_req->current_nr_sectors <= 0) { + end_request(fd_req, 1); + fs->state = idle; + } else { + fs->req_sector += fs->scount; + if (fs->req_sector > fs->secpertrack) { + fs->req_sector -= fs->secpertrack; + if (++fs->head > 1) { + fs->head = 0; + ++fs->req_cyl; + } + } + act(fs); + } + } + if (fs->state == idle) + start_request(fs); + break; + default: + printk(KERN_ERR "swim3: don't know what to do in state %d\n", fs->state); + } + return IRQ_HANDLED; +} + +/* +static void fd_dma_interrupt(int irq, void *dev_id) +{ +} +*/ + +static int grab_drive(struct floppy_state *fs, enum swim_state state, + int interruptible) +{ + unsigned long flags; + + spin_lock_irqsave(&fs->lock, flags); + if (fs->state != idle) { + ++fs->wanted; + while (fs->state != available) { + if (interruptible && signal_pending(current)) { + --fs->wanted; + spin_unlock_irqrestore(&fs->lock, flags); + return -EINTR; + } + interruptible_sleep_on(&fs->wait); + } + --fs->wanted; + } + fs->state = state; + spin_unlock_irqrestore(&fs->lock, flags); + return 0; +} + +static void release_drive(struct floppy_state *fs) +{ + unsigned long flags; + + spin_lock_irqsave(&fs->lock, flags); + fs->state = idle; + start_request(fs); + spin_unlock_irqrestore(&fs->lock, flags); +} + +static int fd_eject(struct floppy_state *fs) +{ + int err, n; + + err = grab_drive(fs, ejecting, 1); + if (err) + return err; + swim3_action(fs, EJECT); + for (n = 20; n > 0; --n) { + if (signal_pending(current)) { + err = -EINTR; + break; + } + swim3_select(fs, RELAX); + schedule_timeout_interruptible(1); + if (swim3_readbit(fs, DISK_IN) == 0) + break; + } + swim3_select(fs, RELAX); + udelay(150); + fs->ejected = 1; + release_drive(fs); + return err; +} + +static struct floppy_struct floppy_type = + { 2880,18,2,80,0,0x1B,0x00,0xCF,0x6C,NULL }; /* 7 1.44MB 3.5" */ + +static int floppy_ioctl(struct inode *inode, struct file *filp, + unsigned int cmd, unsigned long param) +{ + struct floppy_state *fs = inode->i_bdev->bd_disk->private_data; + int err; + + if ((cmd & 0x80) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + +#ifdef CONFIG_PMAC_MEDIABAY + if (fs->media_bay && check_media_bay(fs->media_bay, MB_FD)) + return -ENXIO; +#endif + + switch (cmd) { + case FDEJECT: + if (fs->ref_count != 1) + return -EBUSY; + err = fd_eject(fs); + return err; + case FDGETPRM: + if (copy_to_user((void __user *) param, &floppy_type, + sizeof(struct floppy_struct))) + return -EFAULT; + return 0; + } + return -ENOTTY; +} + +static int floppy_open(struct inode *inode, struct file *filp) +{ + struct floppy_state *fs = inode->i_bdev->bd_disk->private_data; + struct swim3 __iomem *sw = fs->swim3; + int n, err = 0; + + if (fs->ref_count == 0) { +#ifdef CONFIG_PMAC_MEDIABAY + if (fs->media_bay && check_media_bay(fs->media_bay, MB_FD)) + return -ENXIO; +#endif + out_8(&sw->setup, S_IBM_DRIVE | S_FCLK_DIV2); + out_8(&sw->control_bic, 0xff); + out_8(&sw->mode, 0x95); + udelay(10); + out_8(&sw->intr_enable, 0); + out_8(&sw->control_bis, DRIVE_ENABLE | INTR_ENABLE); + swim3_action(fs, MOTOR_ON); + fs->write_prot = -1; + fs->cur_cyl = -1; + for (n = 0; n < 2 * HZ; ++n) { + if (n >= HZ/30 && swim3_readbit(fs, SEEK_COMPLETE)) + break; + if (signal_pending(current)) { + err = -EINTR; + break; + } + swim3_select(fs, RELAX); + schedule_timeout_interruptible(1); + } + if (err == 0 && (swim3_readbit(fs, SEEK_COMPLETE) == 0 + || swim3_readbit(fs, DISK_IN) == 0)) + err = -ENXIO; + swim3_action(fs, SETMFM); + swim3_select(fs, RELAX); + + } else if (fs->ref_count == -1 || filp->f_flags & O_EXCL) + return -EBUSY; + + if (err == 0 && (filp->f_flags & O_NDELAY) == 0 + && (filp->f_mode & 3)) { + check_disk_change(inode->i_bdev); + if (fs->ejected) + err = -ENXIO; + } + + if (err == 0 && (filp->f_mode & 2)) { + if (fs->write_prot < 0) + fs->write_prot = swim3_readbit(fs, WRITE_PROT); + if (fs->write_prot) + err = -EROFS; + } + + if (err) { + if (fs->ref_count == 0) { + swim3_action(fs, MOTOR_OFF); + out_8(&sw->control_bic, DRIVE_ENABLE | INTR_ENABLE); + swim3_select(fs, RELAX); + } + return err; + } + + if (filp->f_flags & O_EXCL) + fs->ref_count = -1; + else + ++fs->ref_count; + + return 0; +} + +static int floppy_release(struct inode *inode, struct file *filp) +{ + struct floppy_state *fs = inode->i_bdev->bd_disk->private_data; + struct swim3 __iomem *sw = fs->swim3; + if (fs->ref_count > 0 && --fs->ref_count == 0) { + swim3_action(fs, MOTOR_OFF); + out_8(&sw->control_bic, 0xff); + swim3_select(fs, RELAX); + } + return 0; +} + +static int floppy_check_change(struct gendisk *disk) +{ + struct floppy_state *fs = disk->private_data; + return fs->ejected; +} + +static int floppy_revalidate(struct gendisk *disk) +{ + struct floppy_state *fs = disk->private_data; + struct swim3 __iomem *sw; + int ret, n; + +#ifdef CONFIG_PMAC_MEDIABAY + if (fs->media_bay && check_media_bay(fs->media_bay, MB_FD)) + return -ENXIO; +#endif + + sw = fs->swim3; + grab_drive(fs, revalidating, 0); + out_8(&sw->intr_enable, 0); + out_8(&sw->control_bis, DRIVE_ENABLE); + swim3_action(fs, MOTOR_ON); /* necessary? */ + fs->write_prot = -1; + fs->cur_cyl = -1; + mdelay(1); + for (n = HZ; n > 0; --n) { + if (swim3_readbit(fs, SEEK_COMPLETE)) + break; + if (signal_pending(current)) + break; + swim3_select(fs, RELAX); + schedule_timeout_interruptible(1); + } + ret = swim3_readbit(fs, SEEK_COMPLETE) == 0 + || swim3_readbit(fs, DISK_IN) == 0; + if (ret) + swim3_action(fs, MOTOR_OFF); + else { + fs->ejected = 0; + swim3_action(fs, SETMFM); + } + swim3_select(fs, RELAX); + + release_drive(fs); + return ret; +} + +static struct block_device_operations floppy_fops = { + .open = floppy_open, + .release = floppy_release, + .ioctl = floppy_ioctl, + .media_changed = floppy_check_change, + .revalidate_disk= floppy_revalidate, +}; + +static int swim3_add_device(struct macio_dev *mdev, int index) +{ + struct device_node *swim = mdev->ofdev.node; + struct device_node *mediabay; + struct floppy_state *fs = &floppy_states[index]; + int rc = -EBUSY; + + /* Check & Request resources */ + if (macio_resource_count(mdev) < 2) { + printk(KERN_WARNING "ifd%d: no address for %s\n", + index, swim->full_name); + return -ENXIO; + } + if (macio_irq_count(mdev) < 2) { + printk(KERN_WARNING "fd%d: no intrs for device %s\n", + index, swim->full_name); + } + if (macio_request_resource(mdev, 0, "swim3 (mmio)")) { + printk(KERN_ERR "fd%d: can't request mmio resource for %s\n", + index, swim->full_name); + return -EBUSY; + } + if (macio_request_resource(mdev, 1, "swim3 (dma)")) { + printk(KERN_ERR "fd%d: can't request dma resource for %s\n", + index, swim->full_name); + macio_release_resource(mdev, 0); + return -EBUSY; + } + dev_set_drvdata(&mdev->ofdev.dev, fs); + + mediabay = (strcasecmp(swim->parent->type, "media-bay") == 0) ? + swim->parent : NULL; + if (mediabay == NULL) + pmac_call_feature(PMAC_FTR_SWIM3_ENABLE, swim, 0, 1); + + memset(fs, 0, sizeof(*fs)); + spin_lock_init(&fs->lock); + fs->state = idle; + fs->swim3 = (struct swim3 __iomem *) + ioremap(macio_resource_start(mdev, 0), 0x200); + if (fs->swim3 == NULL) { + printk("fd%d: couldn't map registers for %s\n", + index, swim->full_name); + rc = -ENOMEM; + goto out_release; + } + fs->dma = (struct dbdma_regs __iomem *) + ioremap(macio_resource_start(mdev, 1), 0x200); + if (fs->dma == NULL) { + printk("fd%d: couldn't map DMA for %s\n", + index, swim->full_name); + iounmap(fs->swim3); + rc = -ENOMEM; + goto out_release; + } + fs->swim3_intr = macio_irq(mdev, 0); + fs->dma_intr = macio_irq(mdev, 1);; + fs->cur_cyl = -1; + fs->cur_sector = -1; + fs->secpercyl = 36; + fs->secpertrack = 18; + fs->total_secs = 2880; + fs->media_bay = mediabay; + init_waitqueue_head(&fs->wait); + + fs->dma_cmd = (struct dbdma_cmd *) DBDMA_ALIGN(fs->dbdma_cmd_space); + memset(fs->dma_cmd, 0, 2 * sizeof(struct dbdma_cmd)); + st_le16(&fs->dma_cmd[1].command, DBDMA_STOP); + + if (request_irq(fs->swim3_intr, swim3_interrupt, 0, "SWIM3", fs)) { + printk(KERN_ERR "fd%d: couldn't request irq %d for %s\n", + index, fs->swim3_intr, swim->full_name); + pmac_call_feature(PMAC_FTR_SWIM3_ENABLE, swim, 0, 0); + goto out_unmap; + return -EBUSY; + } +/* + if (request_irq(fs->dma_intr, fd_dma_interrupt, 0, "SWIM3-dma", fs)) { + printk(KERN_ERR "Couldn't get irq %d for SWIM3 DMA", + fs->dma_intr); + return -EBUSY; + } +*/ + + init_timer(&fs->timeout); + + printk(KERN_INFO "fd%d: SWIM3 floppy controller %s\n", floppy_count, + mediabay ? "in media bay" : ""); + + return 0; + + out_unmap: + iounmap(fs->dma); + iounmap(fs->swim3); + + out_release: + macio_release_resource(mdev, 0); + macio_release_resource(mdev, 1); + + return rc; +} + +static int __devinit swim3_attach(struct macio_dev *mdev, const struct of_device_id *match) +{ + int i, rc; + struct gendisk *disk; + + /* Add the drive */ + rc = swim3_add_device(mdev, floppy_count); + if (rc) + return rc; + + /* Now create the queue if not there yet */ + if (swim3_queue == NULL) { + /* If we failed, there isn't much we can do as the driver is still + * too dumb to remove the device, just bail out + */ + if (register_blkdev(FLOPPY_MAJOR, "fd")) + return 0; + swim3_queue = blk_init_queue(do_fd_request, &swim3_lock); + if (swim3_queue == NULL) { + unregister_blkdev(FLOPPY_MAJOR, "fd"); + return 0; + } + } + + /* Now register that disk. Same comment about failure handling */ + i = floppy_count++; + disk = disks[i] = alloc_disk(1); + if (disk == NULL) + return 0; + + disk->major = FLOPPY_MAJOR; + disk->first_minor = i; + disk->fops = &floppy_fops; + disk->private_data = &floppy_states[i]; + disk->queue = swim3_queue; + disk->flags |= GENHD_FL_REMOVABLE; + sprintf(disk->disk_name, "fd%d", i); + set_capacity(disk, 2880); + add_disk(disk); + + return 0; +} + +static struct of_device_id swim3_match[] = +{ + { + .name = "swim3", + }, + { + .compatible = "ohare-swim3" + }, + { + .compatible = "swim3" + }, +}; + +static struct macio_driver swim3_driver = +{ + .name = "swim3", + .match_table = swim3_match, + .probe = swim3_attach, +#if 0 + .suspend = swim3_suspend, + .resume = swim3_resume, +#endif +}; + + +int swim3_init(void) +{ + macio_register_driver(&swim3_driver); + return 0; +} + +module_init(swim3_init) + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Paul Mackerras"); +MODULE_ALIAS_BLOCKDEV_MAJOR(FLOPPY_MAJOR); diff --git a/src/include/ptree.h b/src/include/ptree.h new file mode 100755 index 0000000..3fdf70f --- /dev/null +++ b/src/include/ptree.h @@ -0,0 +1,295 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _PARSE_TREE_H_ +#define _PARSE_TREE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* better NOT use this in header files for the sake of large programs with lots of components. */ +using namespace std; + +/** A Tree Node or a tree */ +class Tree; + +/** A whole parse tree */ +class ParseTree { + public: + ParseTree(Tree *root, int nTypes, map *typeNames, map *typeIds); + ~ParseTree(); + + Tree *getRoot(); + + int typeCount(); + + /** Valid type values range from 0 to typeCount-1 */ + const string & getTypeName(int); + + int getTypeID( const string& ); /* "IDENTIFER" for identifiers. */ + + string filename; + + /** relevantNodes, are those that shouold be counted within the vector */ + vector relevantNodes; + + /** leafNodes are the smallest nodes which are used to advance the + * sliding window */ + vector leafNodes; + + /** validParents are the nodes from which we will generate vectors if they + * have the required counts */ + vector validParents; + + /** this's something similar to relevantNodes??? just because of a different vector merging strategy. */ + vector mergeableNodes; + + /** dump the whole tree in a graph-like format; output filename is the 'filename'+'.grp' */ + bool dumpParseTree(bool toOveride); + + /** return the smallest tree containing all elements from the line number */ + Tree* line2Tree(int ln); + /** return the smallest tree containing all elements from the line range */ + Tree* line2Tree(int startln, int endln); + + /** return the smallest common ancestor in the parse tree that contains all the tokens in the range: */ + Tree* tokenRange2Tree(long startTokenId, long endTokenId); + /** return the "contextual" node above the given node: */ + Tree* getContextualNode(Tree* node); + Tree* getContextualNode(long startTokenId, long endTokenId); + + /** return the path from the root to the token: */ + std::list* root2Token(long tid); + bool root2TokenAux(long tid, Tree* node, std::list& path); + + /** get the order number of a tree node in the parse tree. + * The order number is based on depth-first traversal and starts with 1. + * It's currently for use dumpParseTree. */ + long tree2sn(Tree* n); + + private: + ParseTree(); + ParseTree(const ParseTree &); + const ParseTree &operator=( const ParseTree &rhs); + + int nTypes; /* dimension of a vector */ + Tree *root; /* the root tree node */ + + /** map node type ids to type names */ + map *typeNames; + + /** map node type names to type ids */ + map *typeIDs; +}; + +/* Valid type ids range from 0 to typeCount-1 */ +int typeCount(std::map& id2name); +int typeCount(std::map& name2id); +const string & getTypeName(std::map& id2name, int id); +int getTypeID(std::map& name2id, const string& name); /* "IDENTIFER" for identifiers. */ +bool isContextualNode(Tree* node); // language-dependent operation + +/** create a parse tree from a file: */ +ParseTree* parseFile(const char * fn); + +class Terminal; +class NonTerminal; + +/* enum type used as "subscripts" of tree attributes. */ +typedef enum { + NODE_VECTOR, /* the vector */ + NODE_ID, /* range of node IDs in the serialized tree */ + NODE_TOKEN_ID, /* range of token IDs */ + NODE_SERIALIZED_NEIGHBOR, +} NodeAttributeName_t; + +class Tree { + public: + /** the type id of this node */ + int type; /* need to rely on getTypeName to get its type name */ + + /** the child nodes of this node */ + vector children; + + virtual bool isTerminal() { return false;} + virtual bool isNonTerminal() { return false;} + virtual Terminal *toTerminal() {return NULL;} + virtual NonTerminal *toNonTerminal() {return NULL;} + + /** get the left most leaf node */ + virtual Tree* getLeftMostChild() { + if ( children.size()==0 ) + return this; + else + return children.front()->getLeftMostChild(); + } + + /** append a child node */ + virtual void addChild( Tree *t ) { + children.push_back(t); + } + + virtual void print() { + cout << "[ " << type << " "; + for (int i= 0; i < children.size(); i++) { + children[i]->print(); + } + cout << "]"; + } + + virtual void printTok() { + for (int i= 0; i < children.size(); i++) { + children[i]->printTok(); + } + } + + std::map attributes; + + ~Tree() + { + /* tree nodes can not be shared: */ + for (int i= 0; i < children.size(); i++) { + if ( children[i]!=NULL ) { + delete children[i]; + children[i] = NULL; + } + } + nextSibbling = NULL; + parent = NULL; + + /* TODO: possible mem leak from the elements in attributes. */ + attributes.clear(); + } + + Tree() { + nextSibbling= NULL; + parent= NULL; + type= -1; + } + + /** calculate the range of line numbers for this tree node, store results in [min, max]. + * For performance concerns, this function should only be called once from the root node. */ + int max, min; + virtual void lineRange() { + max=-1; + min=INT_MAX; + for (int i= 0; i < children.size(); i++ ) { + children[i]->lineRange(); // recursion + if (max < children[i]->max) { + max= children[i]->max; + } + if (min > children[i]->min) { + min= children[i]->min; + } + } + } + + Tree *nextSibbling; + Tree *parent; + int terminal_number; /* The number of terminals under *this* node */ + + /** output the nodes and the edges under this tree: */ + long dumpTree(ofstream & out, long n); + + /** get the order number of a tree node under this tree. + * The order number is based on depth-first traversal and starts with 'n'. + * It's for use with dumpParseTree. */ + long tree2sn(Tree* nd, long& n); + + /** Recursively compare node kinds. return false iff any difference. */ + friend bool compareTree(Tree* t1, Tree* t2); +}; + +class Terminal : public Tree { +public: + Terminal( int type, char *s, int line ) { + this->type= type; + value= new string(s); + this->line= line; + } + int line; + + ~Terminal() { + delete value; + } + + virtual void lineRange() { + min= max= line; + } + + virtual bool isTerminal() {return true;} + virtual Terminal *toTerminal() {return this;} + + virtual void print() + { + cout << "<" << *value << ">"; + } + + virtual void printTok() + { + cout << *value << ", " << line << endl; + } + + string *value; + +}; + +class NonTerminal : public Tree { +public: + NonTerminal( int type ) { + this->type= type; + } + + virtual bool isNonTerminal() {return true;} + virtual NonTerminal *toNonTerminal() {return this;} +}; + + +/** stuff from Ylex & Bison */ +extern FILE *yyin; +void yyrestart( FILE *new_file ); +int yyparse(); +extern Tree* root; + +/** stuff from ptgen */ +void id_init(); +extern std::map name2id; +extern std::map id2name; /* */ + +#endif /* _PARSE_TREE_H_ */ + diff --git a/src/lsh/Makefile b/src/lsh/Makefile new file mode 100755 index 0000000..7803961 --- /dev/null +++ b/src/lsh/Makefile @@ -0,0 +1,79 @@ +SOURCES_DIR:=sources +OBJ_DIR:=bin +OUT_DIR:=bin +TEST_DIR:=$(SOURCES_DIR) +DIR:=$(shell basename $(shell pwd)) + +#H_SOURCES:=`find $(SOURCES_DIR) -name "*.h"` +#CPP_SOURCES:=`find $(SOURCES_DIR) -name "*.cpp"` +#TEST_SOURCES:=`find $(TEST_DIR) -name "*.cpp"` +OBJ_SOURCES:=$(SOURCES_DIR)/BucketHashing.cpp \ + $(SOURCES_DIR)/Geometry.cpp \ + $(SOURCES_DIR)/LocalitySensitiveHashing.cpp \ + $(SOURCES_DIR)/Random.cpp \ + $(SOURCES_DIR)/Util.cpp \ + $(SOURCES_DIR)/GlobalVars.cpp \ + $(SOURCES_DIR)/SelfTuning.cpp \ + $(SOURCES_DIR)/NearNeighbors.cpp + +LSH_BUILD:=LSHMain + +TEST_BUILDS:=exactNNs \ + genDS \ + compareOutputs \ + genPlantedDS + +GCC:=g++ +OPTIONS:=-DREAL_FLOAT -O3 #-DDEBUG +CFLAGS:=$(OPTIONS) +# -march=athlon -msse -mfpmath=sse +LIBRARIES:=-lm +#-ldmalloc + +all: + $(MAKE) -C sources all + +clean: # remove all intermediate file from compilation + $(MAKE) -C sources clean + +clean_all: # remove all files from compilation + $(MAKE) -C sources clean_all + +arch: clean_all + tar --exclude=experiments -C .. -cvzf ../enumBuckets-$(shell date -u +'%y%m%d-%H%M').tar.gz $(DIR) + +# bin/compile + +c: compile + +compile: + @mkdir -p $(OUT_DIR) + $(GCC) -o $(OUT_DIR)/$(LSH_BUILD) $(OPTIONS) $(OBJ_SOURCES) $(SOURCES_DIR)/$(LSH_BUILD).cpp $(LIBRARIES) + chmod g+rwx $(OUT_DIR)/$(LSH_BUILD) + +ct: + @mkdir -p $(OUT_DIR) + (for i in $(TEST_BUILDS); do \ + $(GCC) -o $(OUT_DIR)/$$i $(OPTIONS) $(OBJ_SOURCES) $(TEST_DIR)/$${i}.cpp $(LIBRARIES); chmod g+rwx $(OUT_DIR)/$$i; done) + +zip: + zip -r LSHarchive.zip Makefile sources bin Documentation + chmod g+rw LSHarchive.zip + + +ship: + \rm -rf E2LSH-0.1 + mkdir E2LSH-0.1 + cp -r Documentation/manual.ps sources Makefile E2LSH-0.1 + \rm -rf E2LSH-0.1/sources/CVS + cp mnist1k.dts05 E2LSH-0.1/mnist1k.dts + cp mnist10k_q05 E2LSH-0.1/mnist1k.q + mkdir E2LSH-0.1/bin + cp bin/compile bin/exact bin/lsh* E2LSH-0.1/bin + \rm -f E2LSH-0.1.tar E2LSH-0.1.tar.gz + tar -cvf E2LSH-0.1.tar E2LSH-0.1/ + gzip -c E2LSH-0.1.tar > E2LSH-0.1.tar.gz + \rm -f E2LSH-0.1.tar + chmod -R g+rw E2LSH-0.1 + chmod g+rw E2LSH-0.1.tar.gz + diff --git a/src/lsh/README b/src/lsh/README new file mode 100755 index 0000000..307672c --- /dev/null +++ b/src/lsh/README @@ -0,0 +1,85 @@ +============================================================================== +Based on E2LSH 0.1: + Alexandr Andoni (andoni@mit.edu) + Piotr Indyk (indyk@mit.edu) +Modified by: + Stephane Glondu (stephane.glondu@dptinfo.ens-cachan.fr) + Lingxiao Jiang (lxjiang@ucdavis.edu) + Andreas Saebjornsen (andsebjo@ucdavis.edu) + Jeremiah Willcock (jewillco@osl.iu.edu) +LICENSE: + Following E2LSH, the MIT License applies to all files in this directory. + +============================================================================== +Installation on Linux or Cygwin/Windows + +- Run "make", which should generate several binaries in bin/, Deckard uses only + "enumBuckets". There may (or may not) be precompiled binaries in bin/ + depending on the package. + + There are also several bash shell scripts in bin/ for easy invocations of LSH. + +- For convenience, users may add "bin/" into $PATH. + +- If trouble, try changing DEFINE_FLOAT in sources/Makefile. + +- To generate a time-stamped archive, run "make arch". + +- The source may or may not compile under different versions of Cygwin/Windows + due to some library incompatibility. But Cygwin 1.7.7 and later with gcc + 4.3.4 should be fine. + +- The main changes in this modified version of E2LSH are in + sources/enumBuckets.cpp (which is a modified copy of sources/LSHMain.cpp) + + some changes in headers + Makefiles + some bug fixes. + + Also, the base version does not work when distance is zero, while the +modified version works (in an ad-hoc way, not 100% accurate). To improve upon +this, we can directly employ exact hashing when distance is zero (c.f. +Andreas's binary clone detection paper on ISSTA '09). + +============================================================================== +Usage of the binary: enumBuckets + +-A output all clones, ignoring all filtering parameters except for lowerBound +-B output all clones, ignoring the filtering parameters for bug reports; + used for clone detections. +-l minimum number of lines for a vector to be reported +-b minimum number of vectors for a bucket to be reported +-F inter-file clone detection. +==Options not implemented== +-V maximum number of different nVARs for a bucket to be reported +-e maximum diff among the number of different nVARs for a bucket to be reported +-E maximum diff among nVARs for a bucket to be reported. +==Many of these options are for filtering purposes, which can be +==better organized in a separate module, such as post-processing== + +-v minimum nVARS for an item to be reported +-m minimum NUM_NODES for a vector to be reported +-t maximum number of vectors for a bucket to be reported +-N number of points (automatically detected) +-d dimension (automatically detected) +-p parameter file name (will be created if needed) +-P probability of success (LSH parameter) +-M available memory (default: 800000000) +-a read lines at once (prefetch parameter) +-c forces parameters to be recomputed +-R radius parameter of the LSH algorithm +-f input file + + is a positive integer, a floating-point value, a filename + +Results are printed to stdout, error and warnings are printed to stderr. + +============================================================================== +A typical use: + time bin/enumBuckets -p gcc.params -f gcc.vector -R 0.9 > gcc.result + +Used in Deckard's scripts/: + time bin/enumBuckets -R $distance -M $memory -b 2 -A \ + -f -c -p > cluster_report 2> /dev/null + +============================================================================== +LSH may be used for many other purposes besides its use in Deckard. Please +refer to the detailed manual.ps. + diff --git a/src/lsh/bin/compile b/src/lsh/bin/compile new file mode 100755 index 0000000..206caae --- /dev/null +++ b/src/lsh/bin/compile @@ -0,0 +1,32 @@ +#!/bin/bash + +OUT_DIR=bin +SOURCES_DIR=sources +OBJ_SOURCES="$SOURCES_DIR/BucketHashing.cpp \ + $SOURCES_DIR/Geometry.cpp \ + $SOURCES_DIR/LocalitySensitiveHashing.cpp \ + $SOURCES_DIR/Random.cpp \ + $SOURCES_DIR/Util.cpp \ + $SOURCES_DIR/GlobalVars.cpp \ + $SOURCES_DIR/SelfTuning.cpp \ + $SOURCES_DIR/NearNeighbors.cpp" + +TEST_BUILDS="exactNNs \ + genDS \ + compareOutputs \ + genPlantedDS enumBuckets" + +defineFloat=REAL_FLOAT + +g++ -o $OUT_DIR/testFloat -DREAL_FLOAT $OBJ_SOURCES $SOURCES_DIR/testFloat.cpp -lm >/dev/null 2>&1 || defineFloat=REAL_DOUBLE + +OPTIONS="-O3 -D$defineFloat" + +g++ -o $OUT_DIR/LSHMain $OPTIONS $OBJ_SOURCES $SOURCES_DIR/LSHMain.cpp -lm + +chmod g+rwx $OUT_DIR/LSHMain + +for i in $TEST_BUILDS; do + g++ -o ${OUT_DIR}/$i $OPTIONS ${SOURCES_DIR}/${i}.cpp $OBJ_SOURCES -lm; chmod g+rwx $OUT_DIR/${i}; + +done \ No newline at end of file diff --git a/src/lsh/bin/exact b/src/lsh/bin/exact new file mode 100755 index 0000000..47c9134 --- /dev/null +++ b/src/lsh/bin/exact @@ -0,0 +1,18 @@ +#!/bin/bash + +successProbability=0.9 + +if [ $# -le 2 ]; then + echo Usage: $0 radius data_set_file query_set_file + exit +fi + +nDataSet=` wc -l $2` +for x in $nDataSet; do nDataSet=$x; break; done +nQuerySet=` wc -l $3` +for x in $nQuerySet; do nQuerySet=$x; break; done +dimension=`head -1 $2 | wc -w` + +#echo $nDataSet $nQuerySet $dimension + +bin/exactNNs $nDataSet $nQuerySet $dimension $successProbability $1 $2 $3 diff --git a/src/lsh/bin/lsh b/src/lsh/bin/lsh new file mode 100755 index 0000000..3ac728e --- /dev/null +++ b/src/lsh/bin/lsh @@ -0,0 +1,42 @@ +#!/bin/bash + +dir=bin + +if [ $# -le 2 ]; then + echo Usage: $0 radius data_set_file query_set_file "[successProbability]" + exit +fi + +paramsFile=$2.params + +if [ $# -ge 4 ]; then + # success probability supplied + $dir/lsh_computeParams $1 $2 $3 $4 > $paramsFile || exit 1 +else + # success probability not supplied + $dir/lsh_computeParams $1 $2 $3 > $paramsFile || exit 1 +fi + +chmod g+rw $paramsFile + +echo "R*******" >/dev/stderr +echo "R*********************" >/dev/stderr +echo "R-NN DS params computed." >/dev/stderr +echo "R*********************" >/dev/stderr +echo "R*******" >/dev/stderr + +$dir/lsh_fromParams $2 $3 $paramsFile + +# successProbability=0.9 + +# nDataSet=` wc -l $2` # | sed -e 's/^\ //g' | sed -e 's/\.//'` +# for x in $nDataSet; do nDataSet=$x; break; done +# nQuerySet=` wc -l $3` # $2 | sed -e 's/^\ //g' | sed -e 's/\.*//'` +# for x in $nQuerySet; do nQuerySet=$x; break; done +# dimension=`head -1 $3 | wc -w` + +# echo $nDataSet $nQuerySet $dimension + +# m=`cat bin/mem` + +# bin/LSHMain $nDataSet $nQuerySet $dimension $successProbability $* $m diff --git a/src/lsh/bin/lsh_computeParams b/src/lsh/bin/lsh_computeParams new file mode 100755 index 0000000..e92e759 --- /dev/null +++ b/src/lsh/bin/lsh_computeParams @@ -0,0 +1,38 @@ +#!/bin/bash + +successProbability=0.9 + +if [ $# -le 1 ]; then + echo Usage: $0 radius data_set_file "{query_set_file | .} [successProbability]" + exit +fi + +if [ $# -ge 4 ]; then + # success probability supplied + successProbability=$4 +fi + +nDataSet=` wc -l $2` +for x in $nDataSet; do nDataSet=$x; break; done +if [ "$3" != "." ]; then + nQuerySet=` wc -l $3` + for x in $nQuerySet; do nQuerySet=$x; break; done +else + nQuerySet=0 +fi +dimension=`head -1 $2 | wc -w` + +#echo $nDataSet $nQuerySet $dimension + + +if [ -e bin/mem ]; then + m=`cat bin/mem`; +else + s=`free -m | grep "Mem:"` + for i in $s; do m=$i; if [ "$i" != "Mem:" ]; then break; fi; done + m=${m}000000 + echo $m > bin/mem +fi + +echo $nDataSet $nQuerySet $dimension $successProbability $1 $2 $3 $m -c +bin/LSHMain $nDataSet $nQuerySet $dimension $successProbability $1 $2 $3 $m -c diff --git a/src/lsh/bin/lsh_fromParams b/src/lsh/bin/lsh_fromParams new file mode 100755 index 0000000..7ef11fd --- /dev/null +++ b/src/lsh/bin/lsh_fromParams @@ -0,0 +1,27 @@ +#!/bin/bash + +successProbability=0.9 + +if [ $# -le 2 ]; then + echo Usage: $0 data_set_file query_set_file params_file + exit +fi + +nDataSet=` wc -l $1` +for x in $nDataSet; do nDataSet=$x; break; done +nQuerySet=` wc -l $2` +for x in $nQuerySet; do nQuerySet=$x; break; done +dimension=`head -1 $1 | wc -w` + +#echo $nDataSet $nQuerySet $dimension + +if [ -e bin/mem ]; then + m=`cat bin/mem`; +else + s=`free -m | grep "Mem:"` + for i in $s; do m=$i; if [ "$i" != "Mem:" ]; then break; fi; done + m=${m}000000 + echo $m > bin/mem +fi + +bin/LSHMain $nDataSet $nQuerySet $dimension $successProbability 1.0 $1 $2 $m -p $3 diff --git a/src/lsh/manual.ps b/src/lsh/manual.ps new file mode 100755 index 0000000000000000000000000000000000000000..f52b678b8d56d04f12b43c93b6677ff4dd8275f6 GIT binary patch literal 233549 zcmeF)ZF3{nbuIq){VDp^E&s1n(G1WHbOTW8Ullwwa+O$PU5#wZSEUym5;MbyPXfQGA)$5zG9uDjB^P`i~u0>&uH9r}*OZd~^Ne^Gl~Mm!E!K)%$lAmuDw= zW9pwzKdFZOkG%N2nmEhf?GO9p=IHG7*xS#4*B|qF{=K~2o^H0g57vHkU-o}{{=5G3 zZU2Y1KS@59-@1#F<>h&9>$>>G>NlT!_WZY-7kS8!%MaeXdGmC;8=B3#&C}z{^AGpO z}Y#*zPZ_KJ?hET>HpsjhaZ(rakbrCUvCUmf3fH8 z_}DqhFE(G6m$x@pxBZkfPlxT(y85^p_s_i8ygI%3Yk^WR``_u};8B!_(WZ1DWfd0zYF%e&{r zv+ovz`Q6Fwm7iaHes{Ayy0|_&y4m>i>Gj!l`SR%OdQ(0de0zQRubZ-}%1K?G-=2Br zb^p%vY2&S8_tu%AxjMVOo(=E%&-K^-2ljS)ahJ2*eKjl}9aEK0s_*WeRAu$u>B~*7 zGb~un*Ttge&yk^ClvcE?0=n9Vy^~s!I z_aM37VcCzv-J{bz*&ozDTmMkQ!>xVsd2Z;@v5(KaJL%!mRe5xB(vQ*?pMO=HAKh$E z-
+bv_;y}Fy1^KzRNQ(oV`D34xTm&2z+|5ue~H+TIXTt}r-R1Aoos~W-{ob2rF^E6-;@#Dz9QkZsxKM`b<;BhR?f$I=e^RVXWgJtPyL(ph zR^|L|@bdDqG=eW(^}D+0PiB!`y(^Eub?Nf#^611PxK{%+*gwx#2IKnW{>kq0YvnF? zcbh>fpB4KjcHqg&>pS+AqrTi7^Hu-ZVd+OqHUs&reE2|jcXfG{E6xk+l!M8=-QVM4 zuzYj3bnE5W=Hk`O>$2%B=hY_DwtW4)KaCH6D)P6tcU8>`KKkhB@b~=8A@9ET@%sMH zRXynsDI9b1_In3ahlBF@)7_tL%;jJ=OZnAP-eu(6V?Tm^FLq1gS&VxzIL#108*Ja~ zejN8dUS0J+j(0!)v48PsLhr7RzGW~@w7s-6o|p5TLAXd+mz><@Z7x_ozqgG3Reux4 z;LFk=ALi4byyb9~Z_DMoa{0P{qP%$1uV3#mkmIA{*PD~gx2MOO{J~LKE)UD$$LogZC)Q*Jlcg8ar%{^Q-9YZ|)@_N%MHZ~LKUG1as@zdSKke(CSwDg&2s*}UBx ze|L6y8Sj5|^6i~b=eh=dSop=@`1Pwhv*YFNOG!}by{d#}qdbjLyH**JVe%~(xabaH8O>=l%H{DigXJoq0Q+ubJ*$D7Ih*@qq!R=C@bV}>So#mo~v*xA+D zS9d<_Fw^{GV$#-`xqH(uk4}->yW1IS&bwde z9`ybiXWQN9-00yx`d|B#^sevZyRYXvppv^U_R;UzV0AY*&MKIfKjLA&Hub#bgBNgU zyiNw;;^HAdWY(Yee|a7cT*W)Zp7!o^=g-*O-o$t7!+}||&Q;_xejYqoZ(b*Ti2dZw zC3ge4Vs9@lukLQIH`_kUdhz7-<>fc!Hy4+0F7ArMYiAog|D^1l*|Ty7kr=_yLj87g zb2<18SouTP;C8Ei-OhV#(j$W5?u{O9WNkd1&4M8Y|K~U6&rJ++)pqEG_k$8EABtmzkd9|hd27c<9HuqrY{GN zz|@yT1^r~O2jDRWCuk;K*G;>N^t^|-)FS|+RK*3{)unLbvim!7 z)LXaZ$GiT=Z|)ev=f4?z%lCQ9qrPRozxi(A0|yAULqP+0{^S;XfAe~?-Sob%ht6Mp zusXfIIy-vz;Rnw)Uw-jn@xd?Qp6ze9r~GbNwF7CzhhMYcz5O0evwwLO<9_c?J9F9L zNUP!AYwuld4+DGmEI8*z_NKE(q2KKx{m%0hVAS2+tKNRU_`FOM^gq9GMtd;skv;k@ zf5;f#WzZgiSMll)y}tX=6YV~}!}U4o`zI`u8_?v0<;^QlHrHDCdtL z^kZOs`f$K*36}kgZqER*n@f}GS%TUQZ}{sD#O>ecJ$ZS5C&KC7?mP_PWOH_N)Z0am zhUfb?zVz1i{-96qo|jjUUk=SW&Y|%pdHe9<`0mNL{4{60dot>O+1x#;`(IurkIJ83 z-3|SO*j{t)dE-=Q$j^Vd+kNDlyWKh7IPl@}XLnbnlfQA`d43)Bzw%gqulwIG^LEw$ zDn^@;9J}?~?$6hL+kNQ9Z@V+Sb&2Q2!$sb?g?b0v-~=CkaVgh$&LIcXC0*Hg!|Q`- zECSbyUADeYgT45ERy^H4-CP_OzbMzIQowTevOKyjFJJZ~$CKgc363Gb@U!R7*3LdF ze&O&hUY}me7|Wa28&}!q2;XjwA*%l1WG1}2xc>#jmsj4#7L?54#U|cTE?>SmWn9wk zr!x7RP=HPzT3+#uxr3a2f7ut(y|c|n<+rGTq_b|L^uPJ-&FjmM1<#-2V_x6<>qq5h zPs>k_j{o`S-ACnIW>@~x=H<)n#;e8C@_%1$FFq>&`DwY>Y`5O}?bGu4@$2)`lbimj zJ~`PO^-pzKzRsW~f6P4`%ZZMT!%CDFR)M$QiFC(jw~Syed>ei!~Xt$D>8VuoIexu9t{V|%P5^+Uf=YC<*q!<>9%})yfa;0UY~}A zxcD)%XZTn)+T3kihgiE>P_nO#@&9i$z+H| zMiMt5!(bJcj@_2m@2+n)=hqqSyz5;1+i{xrA0QWen6EqzgK~Q}ucf59N-lD}x$u_B zbb?3ISw4AEZf{>8pvn)D?VW6n`PH)zKk}IuIK*$Rua4Z;2UR=t-ZhHw^}DOrn+s?8 zpqh+6d|Lkg_VQ+P%`(_Z7R#$`Z+4juQ0AokmscJn4Fd4}R}wbQfHn|C{h z{K8O{^XJP?KIx}(zjmC=xj$$>EC=x_ag#eAVoaHdG>Dkr=YpCwM z*jL^jz6Du7*e&ss$B)x{u-*ec_`ewVqce}5Rv#8+Ie6;*!1BS#<;~GCF8!t)ygc>a zS^oMm{w~X7@cQ=R)zS9${Orh59sCkUEB%s2=Vy+*e06!T`KBC%BD+6p{$Ldg1B3I^ zi`(n_Kg64U@tRNXr!(uL_c1qbE=|~Bb=_Y4@X_J}_!JhEDDh_Vp1D2E760$?(KfgI zue+c9Kvlz{9PHosq-Ut`)7-=dw{A<3AtChf2;LijG5e_8Jbm@_qwlXWgO?%8iT=en z8NiqQu)RDh{pEgC|KKk_@t|KMS|bI8MN@Nnd3$Z?_D<}QGdaCr`Hx*!?+p6|6ED)g z>U}~{vgw;rMWpvG<7T(u<6Ah%tWp1&xkthIWzP=#i)h?` zBm2h~z%`qT{t0)#?p)sBNVwXbqNLCbu`_G({D{@|!JknU+gWoaythS@$Z#K-5&qPTaf3shx z`}glh>-hBa_;jl*Ru1A?x7Wu!Vt=7F7kaRxfl++nVNH=oGb*0UxRwJx;$O;v0r#xV zVkFds4D2MIzj6w`z~~rMmna9{mV-C`U@+eq_uY+d4!7yR+@KjuWzb~0y&G)$#wQLd z=ofpO@b`Pa3Yf&-urIO^V1povGPOBkoVl06nSg!1MhFKKZQdt&EXGk;CCmgmr&Gh zMCXt1@)zGjV$QHz$-PS$?AB`5o;>xHKklkb$L9FmhDp5GUVgK=c#)NT9k6Z{7~}o5 zvV;!R;KAkg+K-e|z)CwmaXO$NXX7=esaH@3^4-tvXA$KTEei zOZWIJ-Qm*?{A7RNC;J0Gd3@j}_XlQm_JiC(LxX4g+xXS~!@uf3>}>o?fvNGe&}xZXSm&8UY>in zPp7+m_~g6&0bvIW-MhnIimy}0|0ATx%gKxOWK<9L)QI6{_+>|cu>Gi9ZH{|pjidUSjba3Sj?(D}dP^B3|W|Gqu^K-_!P^#8W~zmxvoY5(u6|M&U~=sTR?Xp9Cm#+y+6fcY|LZoxi|d489blKl9)7(^r8*zZUFz(DK>q`~d_W{OWx3>ZsTO zSTDiX2OxU@xZZz&x_k548^=D(^dP_3JJ&-JaPI>jR^4vZBnBPKF)k=^bGNdK0cY6@ zRSt3HAtvsF-~(*TUHu`{SnMMG{xkm`(1?Lg2snVoKM^dt{a~5m4`Ri~QFM+Kk7emwztcaxUx(;(i1woSz1=oSz1AoS!E0pZ78Tybt{6r(n)L(jRW)&mZZJ z&aPg^Y47NU*!UaKlc}L-K$goIE#P#)8wt!d5=Ku@=YSn z&86Evds#fZ^8(k}XG;Asec;+3wEtND*7uK!B%;qPcc;nAeZ*7Y$I71_sEEVMU9byZ zL?nED^A59|)b8-22!C{VvCr8aUWQM4^a`)^@Cv_lcme!`XF9yxLx_i$$&DXg9ni_1 z>DqBmJAUhddkWQV2<4QgM`x+?{B+`pgZ(#<#DQ;mc@{G3f~Ej(PsljL70$RpC#el-xYH@fb7gsH8`j;hxTY1XAB@gaO(J^t$8wnyM)}brUS^ z-f#c=e)%ur&F*jh&v!q2s`h7P6hC%1d$jPUc^>oi1JA>T{zT6M#Xt5umS&9L4@Rng z<4-^KV1GLghq51pzv}V%NzmZMi=&gO8NK{7aT?d}cFe%LV1ny+!35XudN$x)pZ~v$ zOeAdhYD%qZK$@X!qi99jVVw=kW-OJVqS*-*3gk9+~VI^ZwU>k^Q^G zF#Md^+Mnf`6rz6}1k~U2{t0&Hc!3vBY3J~2k6I2dZ$-<)?~ksRg2#K8e4{9}KTUth z-Rlg28^Y5coiUKDPASmuLN`E4+XDrg$X!h88?|!IbPtvuemlKT=pjt&MHPEAa!xy- zc&N|>`Kal^31oik6RDXO5A41_Fo}2lJb&KTo>PeXWdB=mCk@asJ)AK$ODv1X^>#O} zY}T00|Lw9`@u;l!6VQ!2PKmEqDjw<=d5p`8ei_`~dS9m8KSp0PxZPf;`JL?xT)@%p zWj|ZUHj4M{ZXWwz_mg~9hjnnk@PQ%$Cbi2saw8g2)!zFux~?iLznmOT@YqL?->?sAVcYgzv>8dA4h-q z%b)ML{sK|seTano`Cc43go{5Rjxg6lA@tuJNBELM9JzPzdm}yg_x;NsjU$%cLmXkM ze|j8=ciqLS`#AE*l<$rE{o(tMw2nW1tsh?fhwuFPto3jA^MJ1J$N48ARcgPh>qDpR z_j_`;FYiS2lfJi4*X=>0ziS^!%gC-RfOflYWEiYJSrM9-^{%gB@av0HWs2Qhr7QQ`}Up2q>~ivB*eRjMadB@wNn)PUX-*92rxNJe{kOf z@!j25hxUWJTvz`Eqtb(ftTVg_|kIF!e2`=`#H~ou;{I z{|$ac&1qtvQ+wZ}8@r%u(c5x-d7WmJv>Fh{_kYc3Q1I{OHf_wB1^QETZy~yq{?GRp zzHbxikNAAkr!brTn24#L8`Ox-w#TnG$KTuy?lYWSvqe9tw^v{FEhD+BUV(l1P(SZ| zTT>KpF~UPC`f#4Ur(`!V_x)Uli`;$PpCRLVC$Rx3RH3)u9Oh7i)Zxf`6U$Si^W?{i z-+8Q$b^6;7K!^NcLxjBhqd#oe(`N{O@aH4h(7XKs^u1&_+wxO7{O!g2hyHe_)BE64+3D^1`+wc--!Sh5<2t_b zT&LIT)3=+G&rWZ+l6%|BOush(nautc2K_zmKK&W(!rn3M-pEnCqcZF{eO~V!$X98z z-}SQJ|MvBJ7r8gP`{pxE!hOeCdG9QG(>>UG`U1b}we>p}_W1qc`!kqZSu8H!YTIdS ze^58hJRHs_LucQ8|Hyk!-)M}R%trmAyzgxGuluHxO`LnNnyuP)-Ht|!>A0QGCX05O zAKkc_k5-l^!n^&V@*R{i@YpeBSvR+n;$!I!XEGNsRTCT=zx1Ls$s+bLD(^a!}=%j6jT|1jk ze7agVO+A~BSCd)SOony6o=ivGVyL%&(XPhpakm^b?rdGG>Q%Q~%xAN4`7<&^rqj`QI$5nI6Yq>?^>DmeE!Wf0q-*EX$#A-u7WHDaSXb@Zvo%%i=|{uqd^j7e zyJoqbRGy@&+RL^_p!pg=d~OD}P$e+sSA+Z)cs)tmo}~T-USpl1;ALddLEdQ9D{T z!)|6In8%{2y796YyGJ)LF=Xqin|9-&A53@JO=flHd4|ohnK#ULx^!GMUagm_dc11J z!+Nq@tP4x5Uad^kdN^)JD<@$5lliI{E!uH4uP5Eqg{IYPJTc14Zq&4GwHVKkjO%$d z>k7{_pSHuLRa7yjc47%QPdo31Y-QMu4gb(Bn_2(gwM=m``HXuWKwbv>QbqjuVjTz0~Yht}~ zX|=9~Z8aXxnrgMS7MtOCR5O?1dgZAbcEIsW7mYca)ZECj^0$@?vcA`rT{UI(PHe$B zM>AZ^yZLOnt`@UQA>XnrX01iiEtfOranF|4qBZ2rWYx{;(Y)m#7Sl;HVU>$&Je-fG z=3^8y>6qhkR7_UYbUIz|9Xy`LFlozf=vJD}ni!hL>2kShN7L!rxQ-TGHSN~(dO4bA zZYb5=ZdKQ-@pxQyi`8<-aMo^pxSAWyVKrKKfZ5tKPMfiht}Tw`Xfl~B7S(hzYU_fN z>)O^LnsRfi*{rg(+?Bn1s@clZt?F(ywyc>Y18&D2v>VQvwwt!qs+*1qSMSCyGGC2G z%Wm1OR_$VG&SUL0;~!O=-o$dBtfzd0JDIyq?cl|jWA6%vJ6(y6H8=0hfCu$}OrGYip#c<{x%U?uBj)?`fQCVRoxFWT9nGwapD2#)96O~u^` zR*S{Rhg!qQs9IydaTm++%J9$Y)*0*7sGDa&4ae4SYmv_;>qWEhTRWUEyV=;7u&vp+ zt(NYS?HA3M&m6BttEFYn6|UB!<$UFlhZX^QUCnsOx|%PmZn9W$F+zmZ886?}oNY0g zFu6v&YxuO)Xvw9}8mwE(csMe6Vu+Yo4cRacv=p2g7n^nqcD44CNAlEySNwc4YQ{58 zU_BQ)+1-3)`OXBpMYlG=*4~^u=$NKuFkCL`RqF&U)LIxObIw~#*xt-%*6VIkRioBm z2vvzbthAog?b_w0Ll4M1&YYxR?NiZ>V|P=-7~!h*#Pz&ugx2ZIfo^$bqWM^hpy9TL zZbIm98ji1MD+b+p`t{QJMx$b~cWLHF%hiHGTY|%?G3S%EYMJD?)$Pu4Oo|oz zVl9isls%0{)od+P4@Kq0dQ>}aJDLf^tD#_HrYFleXUV7<>vv+l+G5(SvYgr3f+w8T zYl~{x%{}?TUAfp2G+2(-qoK9ij=FZ;ICMO2J=wY*t&254vtDwra~3wO1=dAeakE^B z1yko)&UozMWNDN}YrbG+5yXgA<0*&P6zj@lb|ay{N?EQ=mZdp0!)t5Lb5t|-=utdp zH5&=Xre@R`fXt=g*Qg1F=n@5}f_6#;!Zo`~6V6Hu(Xs+k3%EZLtK953gK z^}1Tm`17V7S*z?dj$yfWi`{HAgE+=Zcp~%Hw8d=1z{TFiTf(7;o2Q+!hE}9$t7tnE~~f<5gFW-NvL_blhfEL(#%c zYXDe)1evGZ98xmBpd}YGUpK3zN3WruPTUbx)=(=~3wtoe>7;`b-RzPNSVIy}*Q6Ck z#V5YCHbYG}ov+5jRlDTPmaE}%=AaHHF-M^H854JZKSvN~?t+8@;uE1$Oo zz;n*|95h-m+BHnwEUY$9+%1cd=?2YXcneeJG!|sDW+Ih=v%+~lxdo`h%Q7SI4U(wZ z7Le-3vz5hU`88JVn%^-cA_8wST?3bu>Gc!>z7gV)XJQ@Dw3@@NV$6h@)RRKjq`3p| zMIz==%|Mlrl`&sTgdqvee9{7L<9K3c_vA7G(b$>UO}8j!)ojR4jb6t`)??38E$2gv zW-=Zw*W9W&%r2TGUtO&P?N}hU&K+qId@7GbE-^`I``=DXFzQN1&hSvIJ*~ zwe+episOpc8&0O{W^76F99<*yWt7|nQ-byabz6_%lWLhA1I`GUAwMvh4+WkqM(}zy0conm5=l~33`tyQ z+tq?k7t5N~@&c@Qz}153O*xY_H^HG>7s6i?@Z2%?wZO7$*0aT=n~$U$MsLX~yH47( zp0A*JQKdEB*1S}1GOmZB1H{PkI0VRJWehCQ^$N`3^jqs@?$qpjRWrlMO629x#OEoz z2tBZZF+|pZ4K7!$=Y?xDY-_X!GH zduEjnc#h?gYfb2EoM^tB%ssxTk&#-BpaL5+2{V2Q23QUg%cOszB9Ji#4Oa35E|ddr z1adBEozNl1z~)jO&o}}h`Rob%Ns?FdK?3`@mF29H!>@+xktty&Mpe32yUlJTV{J#A zYQrfC7Qo?XF7P*eE2HEZ7ltvAbtS{Ggsg66#cU=*5|=DcPA1cWiyE?1SYsjPbnDS{ z1cb?ltEG$!y;3#uRT()mT=6zd&BHU1+0?L$=f!fgni&?4H5h&u&A^C z;W*$sF>|rx5~uatG)&+jj?*4Z(8sh$MhZ&RE+ZMj?pP;?A6`{p=U6ZX0+yB0L7d%k zI$q!lDyJJKsKUyeskMona^q0fc*P-E#XtnwLGlHgNSN2$NF$yNc}cigxW}QO!zQaj z2!Q3~D&T^(0Y@5c8F#&4pn^P)R0&?w#3{jUWz|e!Z)cKrw$oL?$v_wJg+miuBfLz@ zSVm~as^PQ_5n;K>l5F=X^ekJPm~rBRhaDdXpZczZAdqNWIZ#r=aog}PDR_NBHY|e?aa|a{o zTqVyzOImXhI-Y%m8pNj6Q|UZhkN6l#|1o*84kOUXe_;X)+PaYMhYhj%Od(ekj;t}Q zL(~pt5mPHD$UQxuFfwBH?h(4`EF3E*8B4XOR*1|M6mA__oFk5i-GTweVA?de3NaVn zo>{i+IK(YVckjAn|2fi<(B#8D)RTmpkon?bCBpgC?524t}-!Bqw+Q{|CSMiThs zubd<*%5$MV@Z@|?NDtA*l!`|Q5VEr2yqn2fd)5;-2R9m7QJ*(3FET8bi+0 z0!I72(%*Q58VjR1j=s=U5tZ&4n-As98l52?iEJ z?fIq8284mF+)K&?u>6D<#Us`7v>3}M{M*bLk#CI0QgxY@P>U^*8ez^^OgEb@@gGU3 zQQ^}i@~%Kb1we2UG7$J~?FKDR+;K=t?h?OgQc)a2pgVL(o7L8g0U|~$@#Hk-E%&!} zTPz*Wo1%W#0QG#1Yp#8ofyU2@)$;Wvx`v^;CIq@JpcxYcEujxY0WKFplg+LG0g;9g zvmZFlh`UfoNl6^6W&v~^oXwC>))4zz800smT#?%lcwv~yq?O7cDCP1PHXb?`;}p;W z;x#I*TH#Q@sx@pYlB=sgR>&HpF22THnN%hwB*S#3nGZ2uz;Z|{34}@kOBA2uWcVoP z91AYXheJ3d-V2gwmo-}um^r|q9K_w4Unq$6`A7lvF=e-_o&n299J~-%Kt)lRX9T*j zF?f?e-(hXG!g|HX*#xI5`;-QHCj5-U7pQ(57fS+9cwc-sABNtpF=Giz=F>g4xK%^M z)(eHS}%7O@J8Gh;3Cr5nBJ^PM*(PF!<8yiDzP6bk|nkpC6z?*wVYmpL_1$5o-- zn+axH+(a>f^bVA5bW&XP88wKZh+$+EA4Wa1xm?3H38WyT7|xMcPK6gddWbi-7j{BM z4v+BENdZm{HMGv zX z`3L}FoD-_82sFIx9BJUeYU9Qb%!mo^-B>p%h7GgqQXgs6RittjJ|N65D^<*eX*Ec( z6BNK;vU@|h65=gwu2%k3%-IF+rl7Wf`b~9cI;+Y9td@q*x@y%Th4#w4yWiBHmI!wR z#704@NF-2!00D;;+yW55zyez`Q=kK*WAFqQ`LSt`MhdVXkf~d6p5sXY9VVKYLlr`? zWo}7h^N~>QL9Q!g@RVGVFgr#4I1dEu-jt3EsY;jQ`3y^5dBCEPrTp`d+lG~diM-FSeT*|K8X+|RUC5-}PtpjNs8eFDt=`M^S zx3We=fu{;R;0;re4s*uV=iyUP$40)6(U3u7FijfESiYd;zKcl>154}rsb>^ zQ96a?7LP3bf?}1S8u>o}y z#>XTDfvtVk=#QMtv*QPsg+i2-q39!^=URZTP~Zye(2#^lka~hFXkQ@!q(-=-A{fz3 z;Kd_xXXqiQLu7KQu{6UF8{gQMAg+uKyqhy7!~rNDV4GyEY6D7h6BSQ>XQa|1pLY6X zu$cI;L>TeDfl;a-!!UY0A*Uv@3@atTGZznom~PCo(PAbY=4LAK;35&LjJ#3#U6~wb zcDvyw`ItVovrQB}=v)Bnox5?L%0l=)T$Q}vv_M$gB1DdPAP;NkM6Kkr_P`eo=LX}Nq;CxvGKFbRs&;Up# zyb}H5Re>&1X3;|3LbP079pCM_N!%wifwT<5Puk~cq=%|7GL%|@kHZnujM&VQg?29fy}Lr*;+YPi$#VV!}Wyx0KhON zi%DUwz+R+SmPemIA;6tceNUxcrGH8sAQ=J@=O%j0lK}&$z+_g!IDepS#n+17(jxdd z=@K#(FE5o4?l=yi!%_k0*KiguhmW)3N2mr))#6ax6v**~Vf8FaC>LrmIamP^JS*K` zLL{b}XHv@*422G9G_0(4D6?jrA_@XLQVMlXg?vs1X6BUy+T%$EQEFk` z0ELkU14p0^%p~zs^`#n7#=$G_C2UtlD?sBFRRr@l8Fotj9u+%a0buk9OxmP))H%>P zXr#h{sT0~gmLS1+1Eu8!xD-nTITH1Ylme1=tDGgO+{lwxV0A%SZ*!)SY9u#^jlc3F z4qmFPV&FM@u$%8>jOvc=Ie;?S47C<@Cg(LjEFGtb_(i%4x>zl&4vtwONtl4sM11xn zA{%d0ZIv$xG_%4pE~i*N9(=)p@Fa$U{ANNB!IX(w%pIH|G=QGy#RTKJ8Womns%Ao?B`l5rdgMj$ zkJx9rEmjcU$wUtUr8PMyMR?O>ucQl#G{|G;MHNJ=GYc7pCJgbvB56peU#^Gt6Nyf|mm<_`NEelIIl0R+KdHKON*cl9HFhsWi~xk?5rhe^3x z9O~ryZWLtVvX+)vpq=b?$+tpY()>cUB(07PL#`=|8KWg+%P_D!p$t1_Av&65NJDa* zB-o9rDfj9P{DA@hM;yfL*(3@;1roDOW`-XX&xKZOhr)d;rif}>s$vKVJ0cN8RM zKNYun#GInj!JRAZn0ZS^?KXiQo(hd<3e75Rf)FG*PF+mV4J@|-kJeHIygZ8W=l$Q` zPB8S1O+WMh{tKMok^FeK*LY-_wkQ2XztebLL_OSpZ`hCOiv1^ugr@jOHGYb6+)01`UCQE2XD=N7>r@tx%O@y( zdsx@Aax!iIEnTs2`sdRXlP|~~6;A>|$!|nGj0FN*O>#_x2kc|jFs`bIgr;)R&=6QC z;k$w*DK(=a6AaO&aQ9*iBdgSj&`9<|Tv5$)IMk71roxM9CZp-FYfca%82t+uuANn} znYNY+PcA8p!XOA_)Cb6>RD&y*1YlKFI>b9sZ>lWN$r%+AU=$j_q=c&!4aJnER_LD- z;VHO-vEaAV2pt0WPhb;?5`+^455^y{tk@uLL=uzvh8YXLHELFEF-|5LA{{9j`7y8^ z4Nvroq9_`RwprDV>Q%XVB_&V}kViFGfLa34(N;sg@;2yI_By63G1L>NS5-}1l=PB% zQ~rSu#5_^vLc#9Zk&27765^#A3pO>CN}(MzqQXWRMNhg=P6@}8nzw8uOiv)J%2@N_yN9Wi3y>>HIf)duPmX6(Pm9gzR7|nyy!|9qGV&bw z1X-v2oNRR1YqGJ}kWOF3k~5)eAxm@=LVQf7h^9susk=x-@F?^tDwtt!)IRWbrl1EN zYGd6gkC+d^rL(997j6FtLmXFqQ5kt&)TXkAs(s$x@~#IXC-b{Hg4mA_hKE zGKFtz5CkAEd|Do64y6Vw?Q(nt0dDpeVk9CSS~ zL)c`Bq0~51&TmP!WAg;X@Vx?-yOZ66#)gSv(|CLCA(hHPl8lCudMOH^?W5H}Zm6JP zq+n!=z*<{-UXH@-;Y(AhVNNt*2t6u)^|`fA5oWDJ4JhMrnKb!^6%|B9gDsZ~OVE%F zFuAf{U*Us8uH2oJ4UXfxOx|NCD6$^1+#_TK`{{pH8tr{T#@Kk+@ZKZ+;)1)h|| z{TQNb%YV}KSC4`yo6%psPt9$A3_=m|j2f2QTyaE!nx+LM zBX|>fszViscu0}3K#UdX)sd8i!Y<4NWAz!#k&vtF0j;I*MdpfJR(}ui1|t){(iCDd z2vc!jz^JN-${^~3Btz+)!~qxpg!EP*YDIHwnw|jVd{qYpS^a**B4Q98Hk~OrR%$4` z7HS{!kFRNrt`!&V?oRa)t4BV42lCj6sDLHW^dq+W!0hnKZX zI5AWYpdi(1!B`ZGut0|q0H;S!$PVwPBu!aP2RX<%+=+a6td*~F6;zDH5 z8XoBi(DBi*C@E1qB2kgAD1W&`wBCS;Vgr_U5&6xin8COz_9>x(6T0!zEkXHB(~dG$ z!o{s&dEqm&o07B+LbrqVNAE#)L=E^xNC<`tkdqEnpp|?FDn;7Un-GDKHyC)eZ3Nj8 z7FB6XJ_J#8+A@%??hJnx-M|W~n)*GNf}#dsiUmwH6rV*zVBnY(6{M+9tp>y=w5w2{ zWJeldAt1k$3c_NOP?&0p36G2O>xEZfFx5g7beMlExr@(~QAx9A%P1lIkCk*vP-*HQ zEvW!Qm0#9NqQ=@K2qq*d6QUdyO0pL+E4dRw54DSmbC=9aih|zFJV`!Sud+S4%Veky z6@F2DM1c@~#Q9N@Q^B~=)C!ye91kt2GNw$Rw-42b?+@1~S#SkhGDHnQL!dGQ7g7O~ z&p`evWvD?E`Ix#UG6-Zh8qnB=yH_Dmk%J=VY;3M5M4(lx{V2+DDC&EZa|Aaqx6U_f z87yWj>XD{Uy2+9jazIihKwa9SM{`Xk6@U)ZN`Td9RYs6h_elieB`nd(#x=r5DlI)F``y{j{m!uR(eL@QgWrI3wQ=nthtpJ8A zB$|(1r<;@wO-~a;>q7&dPRs?Rksbn4d0k7GeC9~IHY=D=)Mn8mN>%2HmfC|kw#o(R zaUw@kW2QB=*wpM>%3ZSZYN6&RJJa*1Bq2qV$*6#O5_8N!rAucV;TOD4wUj0Uq6TB5 zrXUBGkSRMbH1J*LsoX&duL4ZSPW_{#9Kl#xd90d9Qm5P#-$Z~$Js6d}U7WJ_FDR6t}Z2{&ZUky@%a@n;f^8CC<4p>uJjQH&okCXrL{HIaF$ zZprLa)MTHU0OUz@5_x`oYp!6atGmgc!kLiY#tY)o*Tk^`lcbUsi}IH8Pp>Q$A@RR3 zr{-rpTE6KD$ehZ_CBynUj1X7oiu%&_nY)q8USs zAy5eE?IXao+BMr+&Yn=5G4ec~vOUUOlF0lrF0H&t4U%FF=^?{Xw~-;z7mxv9UsMXz zw1g31Qdv3XNs~;huNEOoZ^IfXtEhV@3rHD7vPr$BiU@NcoFybDFc=PcGvQx~uDi!k z);TqIukQ^1Z;_LYX){f}S*4NOiqWL#M$>4;d8Zhkg0?HU=iD8=7+K*ciftNh#zG51 ze5$5Em%&$XCKwilQ^$&rSS=-BfKs~ymPS2QDLL#$3f6d% zMN0oFN-)7tRgwksm3nu~nEDzu1+|RDoMsaRHhPfxqEL!fJ~wZX*6`)ZMgawirBQf$ z077mhAgv!tFxmvLJ>*4*krc%pT_|o-G^?(s)aIeAbU|GhisO?1&8$ zUbj}H`luew608QqVsnN6Vw}W~_$V?}n=bGMS_vpxBZ5(K3p#~*p$=pb1s|4^2+w4R zro?XKZCtE{r`0H&4h;ho#RtJ7Vkk4UMAJ}Cj=(*G9w|eTv2tO2bb*bu4r6BtM0QD-JgZA=iCMh_?$A%TM7ge+P^NJ%pfu{3fG1XgvF z4bUGVdZEU&*ApB>b3-gjzl&Q%`tir46EyZJi&jktFI?n;tCXFH7nmu!1=5+hiDOl!w1y&=mKq zpl@b&%e*)N7)Oj3q}ZVbFtx)%?HI(QW>Ov(R`70GSp+TcHONHSn>s6^Ni}MgLJyJk ztiMxY6EH)Yg8QY4>qSW%O`4!v-W%*mSCF_%{bLV@5jGd56qr$tMriRjatgEcH;&O5 z{4{>;^FKtdef~$PwZ2BNYCj&$KDKMYUw99w_t~xgG-mDd{~TiNXKeuW*G#EBVPrc_ zjeGyMIW?GgI{Wi+YJc1Q>1S>Klq6JU4@M~EscDl$)2LF+l0!y1r(RT`ptRXQe0HZ$ zkxJ)_5GZmJh=Y*uH_nbQ8Ape0iGmUjh^dr>q!FSjQb!Bk7R1TC0!LtZ90-<4xm8Z0 zH5(ORZzW|M95sjJ@$aeL{>AQviCV-+drnadImylJTMWY86 z_V!Ans)P_K^hik7kPCVourCCzI&jH%We_6`e0sKt4Pgo6D`#&sj6oo{HmR~424jjk z!(gGZup*LFjU5?7d4yto^1@U^CE1j5y6ANt+HMDXjI_{8LaLXR1(kKYAzX$|l}D3z z+J+&je4vQDmX-isQNN`(U8N5gF94Vue`$oo7eD7IP*=ti9j3cNOA(qC;sVi>z0CW^7zoMX@3M{rqvqQEpQG|~GQ!h8l5<4PKN1y6E)YM`QwFzf?5o$UmWSI~N zWSUP1PB8!^!ZuJK?$uAAIO@#^^W{?Zi;-~BMxcR_myk3yKqTC#C~y}|1hxUdFRFh8p%x!5z^_GDn#wuFK}4xc z#FiMw-s3U}cP=kSNg_?vR2U9PF6Daa9-Zb3?H3v$l0q7B4n-bAFlwgUiL^8ANgiB3 zJ}MN)pfk~`WfB-6ev5ushM9d=utt1RfzL)fq?a!{WTi{c?4&&i^=Ry{iP9GfF_jzY zOs!7R4Mc=Qkd^^CoBr+W@uklWrOtZIJ9Q9~L@E+aOH|~2(qDXuRUhKd{2-NPd#y4&=3$4rMuR&F-WUA z8-D0%!;0g{xztdtnl==SkX%u<%S`1xX)#L^E#@C_su*JQ8{No}(NOPbbChc%gmo(z z2PdPRM$4wep%t4XL()k)Um(8>8Ii0eh7@XWX=#LzzRN7r)tsd%4@SrITocc2gQsk> zCw+7g*9&t??K zKlG(pRMuQYO&$Nt`Jypp)wncG6h3Ho9409D#@JXao)R0OJY!{v#`Hf?(+GoYp)`jZ zqYlINWamINF4?Z;NqdOlO(@t^Dz(tz+)%Ux_!htIFXGC6-v9mW1phq};GdSie)`ED z+{ie8`soUh$zM2&{mUvu{*$&c?q$b6Paha1G#ufd`%T({fYti6wkwuR3(8KhSQQRa3?o4Uc0&_rk@0qNwb`VdCn6+H zT@OeE_!KKCh9!hPTHCZi)k@l&7zql?UZoqVv2q__nAK<{+6=5P$-Wo}jP$Xpq-Hl}Dh_QF;OYlJ$`TI6drVmjK9y z99QvNyO$!H+zl!*8miK846LSFBYPv&@rCS)Gmv6x9nrDMpESK?BVgA+rAsa-1wd|) zNoh>trz=lOdN#CMr-v$X=za%;h5#zN=}^78c{yncHD)chmTh1O#pRr4gYXHYv~#J- zI>cyNWJsO|jFse&P|235z_Vw9oGA&IWLje;&PiXLY5+K6KSF?58A_RrgkCKb4bbnD zt2jjMtQRm^AW$#`XH_1^NiG7Z<0}G8p+p1RK|m?X6X!~%WkK}%$uDgv11BY)aSs>^ zyRsyG?zg3c{g|XWX}Z7lnFru2Guv!YDj5}#%6U5c3?QX3P?t_?G(lR{&9{zlQ%P1# zpl@41Vj%()?bPJowxLzVh zCET4HMm7^N#12lNxV}0!B_qf{O9oW2Nf=7wKirhjC z6e5=7?wYVY2J5#t>;@zOw31n=PEafiW+0X$^CFzWKiCf~!tm_ABcWrUaxXj$c7%{a z6-iD7`@4B@BO7M#`Q#lqj-7O{o@`)kX^-!PN<2TLdHl=|gr1 zz(biIyCf!Y=2sXTT@spWmzStgWh=^_2~$U=qc38n9F$eaGwIy``4LyFcdkBb0V`$l zSd{`Kf=1(j=1X>jS&)C$x-M+mpyZZa8I+Uh*J1g3NY{c6;5J-ZYOc(uWrTNB1vaks zLZYU@zgY`2&4Vm4sTbPMnur8l!k}bF4rRCabTSNi5dKr1njNBSaD_3`q8M3`Vq1}2 z4eZ}!rWi<4OKGGglI&@dgf0i;s2Fs%Wisjtt*9?cQBq@*6q|!9JZFariF{Ux?RDkM z`Y?3J8dK91#iagJWl}CV=@E*I07=$N9a*bulOYk#@M?;S7#aF1&cvG7O%p%COK`&3 zA4(#fl5Y}GExp<;IKzS*&+yATF%a&Rj)V_Yj3k^SEm2QaAe8{CX464%-PWOWk;}tT z^7x322rgxY{ZJ`4(^y6rMcP1)hq?$ynp6&@)$v0FpUSstyAkf$ftUJF(wJUgdNhhY zsT#kHh1AwcL4mzP8!GN1HVO76cG> z!>*AeWSP6v>zlor=sb~%X>=VH#)xSkH01UcWA+YU$TS7nIwalW`iqCkxkOwBULLO2 zjCnAIImk=csh-urOK^|X!eSl z2w9xFG1pyTk@DbJVXU_ zW95JBOLqCNk_mjms6=gRGPHVoOxy*nfIV0YM>b7Er{V;$KB!?K2F|d!L`h2sk53sv zXX9ZhO+-hrf!szD4bGn7a3Yvbn@ig%0s{dY2-y*(ru>fXCbT7O7BA^d(xi>fvuBAS zyh6aDu*U{>Xj%b~NDs12MS9io`{0+QqYFJ+2UyAyv}LCj9W8GBj=nd@f>;P`WxKBo z5AFm|>%bWz3;Zqm2bQ5kt$Mxw(Adzq+7`U4-*3$)pXhIeIJ+3Qc|Kgv9M1+tWUBLX}@p<^w;c%&$3 zM?4jq&c>tJCLsa|QYRW76HR<(vl}3U7Z#8?W*z0!pYD$r2uLA2bMrK_QKi}X5nVSe zOlNi^Rr{9;BA}%U^3}LfJ0o%8_9Ot5kyIHhqt7Lv z-BEYA^GKu-c_uV~*)A}Q%}VAclv{|9!*U_O=uk0{{?FE93^yVm5?@gtU5f9rPlPa& zjphn<8;GbUp%INFkj`sUp==O|7A7g<86H84 zSUd`lB=QjA*rSMtK++M~`Q{soDbUS!1mGfk>|0^%cEm~B4#bM>s|8WknkV8bQj_TZ zvoi9P3=EA}javsnxkQ9!LZF~$ivz?dnGTNH!muO&jCuCYk!yetrbp4E7ml%IYk@bn z+KFK3OO?s^mFT}^4%uFu$=F}mjA^ZdUNNqoDBZ1y(CkbG?x*;riqVVQZMndQA*@v} z_(cUM^)PTsFDAh_Yap3}^L@vTo#J(%a$ zp1{PS`h8l7-jq|QZVEK1xTj)>q~-AAJxqvs@x2RbZP|r5`(8(L1t{=NvXg9HOoGK% zk8B@;-%Yf$uS7O+u~A6Ka_LGo%VFJU(QKk6j;27Dd#^%_y!A$JsK9soI=bJoP;h0)QZ^%|b9$ z?NEZ0*=8pl$k$$HN{MdKnr0?VRVeC2*^D}tVX4>p<_RSfJ1<+L5Q3fuFivrkKFjW| z_QVk70v5R&&TDGh$!r^!?pOscTg)ll`34+2{t{{7y3<8!?{Tx7U9;_mZ1*cmC4xtd zzly)A$wGz9m^YIO@|_s%)T4kdg;E)BHmWpRQfS5~Ku7eYd;qj?qh^`rix1?kB0&_S z^=H$v8*~_l=r{?0rN`O&$nhZn zM15Gme0V;(SHgO;mxLkGIZ-To%mPuQ#!(kL6G4JUK^AbQi9xbeG20?{IwcohDh0W4 z_528xDz1Ua@<5!9g+X^m7nu;F9Xgx%k+}L^EPKJqf@ldu2x9qw zN|A==buvy~0ar+sK&Ej%-;W%y%G}zap@p&wMFB%e&Ye| z$&Pj^hS_?}P)V4j3E+6XOiBs@Us68Uh=)BH0|*`U;xU;Yyk}D@>y3_C`sG6qBv2|& zGF;zI#rhE?zHNg4V_8hxG^!XYiJ=$d)#~|zVq#=@J;SOx3rcg=FmI8fHmRXsx z=&%c-QC>UE=U^%?V6E#vBag=@iR#8yo*{JeYHr0hTnVLCf%>p`DjO}}Qj17FpHgDB zw52BvGYCwG$Y4J~c@hBmxB@Csd)n`UKv=x&GW!{4Pdng+w-8#eEc_^0t$`PGNolfq zXLgE^GV!L_Z;*{qob$3MqJ~d|Nfq>=e6f&0Yu}M|h|35QiGPavSUgooyVUtw4xThM zKcFQWwA-7@KA2e`#HW}a%mTZ@r${=vt`tNiKvJM=4?vJ0D1}r>TZ`r)6UL?SD%s(p zkA*y5_87uvL2dkvfDB5i*2D8XeIy~n0mNAV-9oo2yiH)`rf8FVhm7e5QA|R%chFs6 z{mPdkBC$-VR01zC6flNiQ#7{Sx<>$KMQjCXx)u0PP?gg%6HKRlDzYiJMPZ;Ao;bzx zN^e9AAuiwSFttBfzV*X^QYz5Z_!fgGq!anL30b?i#?N^csO6X^c@&6$-G^!55}XPOC2onI-+O%(A~Z0!xXTn*vBFpK&en zX*-jKRKjK|wfnnS<`cqonp)y}yql7?D0V9P9qRxNxs7w0)1JPvK^6_e*@Ee8wY>38LGTVa?Rrv&Zm#d5)6liaj=|1-tj~Ba053wiLMP< zjR(?9C12+e7)v4~rXZfY_lh171_~2@J@0OosjJX!K-XU zNmoWSWPaHk-k4hzwy3RAwl~_akm+lETcH*j*cBoZizvQ;Mcb^f91t_#cIP(O8;F=4 z2L&9ho%w1Z>q-peIQ2)Q4-H_6+_tnuST#cd zW}1;B4g}SNcrBQE3ZjN#kO6OjARK4P4>Tv*G(<6pxO{^lF%ms#6nC`!0)>V`#GnPL zA|gWLfyG-9$+9G%i7bz%fp`Vr<#61h>&Odz$sA@sEt7mq7%`w4jVz&un8C&DJfEgD zyY*niX z4%kRS=nkaHmb$JvD4q}JHZfv2*9@s4AW;mE?-h_DSx71k1Nnj{_|GEHCu4*{@nH@U zj?(<9;(}+_tt9)3@nWRRpy?(KBcStKX|~KJAvLcYm58nz>pWZ9@Jz`*Y$6I7+Ed&{ zzEZTbn_EzlJ18f@5@}dg9=}&e3N3{cO2X@ixn!LZBG-$*wqR`ahvae|i30o_mP556 zTk7$|W4lI*MNyC9+&nIz6ps^!z$npm*?AE7C4eHiae3sUT;FoYKG_g?tHOZ<@nD3D ziUzy}@9H*95bO<+FzkGLmE&D7eh)?r>eBLvnp~ve%6-Xi6)!Zu!fwVQh}`6H2poze z*!MhPRP${p*<%kLB?5EX*co5pg7mcn$*)8^cZ9V~i(B+GX^9n3gZO+PK%OJE)hT;v z1^@uV_zV1)kScsyru;4PO{t4BgBl^@RrYxuuDUf7`0iTzF8*=rhbc#BGT$LY!l5m{BKrD~~E!twO z!#3QfxwG=!v`?y!xuWM(&9fuC8lo?u<=(i|WY4C}-ve;^8q4(`jS$wEO4PaSnM1ZDKOvAXUK{hi0OMOqCy_;C4Ev+&J0x4@(at$XgM?&SuBB(sk zKnl?~dAQrjQ#8|CE=?gK76S~KSRArG4WHI~zUwwE=E6ie_VhrTChcEXf1O#9NOX(k zr6L1h>?F9*m)b=Zf~#nsSCU|*7dy z%#pJBVwmVDBYGznj0SvHCBb2(jTv&rZK~IUx+qT5Zy6??4wo@fs8T;v*hdBN)|@@h zsxyZ1hW4mY618e&%UV_#}lYo-vwb99%YO zPn1)J@Qyo4IV`)9@WY;jyVm4~`SfL3*&8Gz2^LZlJ)diYO}zqzTA1i4QBbb1 z$%=CzeXLIpz&{`)?7_@6%iH48+<}0VLX7X-4*w~F5q zcEL2<0ClsD&oq!oq15GK{pxe(4%Lc2XXokj#hufCPi)C78f1&*>kVs zj+ukR;R)Y=C%5&?cDW$QAxXg;2^kn`DOZF!Qpj*`dYQosa(^55z$Hv#~t4>!CJ~2djzDnC_)ANo- zQHc!O326bq@;xOfF$2e*x%9P0>QoXs^e66GK(tAPnr6?7gh!IZ6D#;difo9Ht)A5q zhbs7W$)8>isjYGqld?eUH3CftG=j6bgbYxXlut0eX4|+KGeo+O4df)W*qJ;Ola(TI zu=R&p8=`~{kXz@Q{(u0ynA{P$CUoW9$jP~S^>@j-jVw_D2yw$A8B!UYW{U_BC#~QL zLF*t}$dcR0E{WWP?>TWcpxsbOKjlSOAd6M8#>p_L49QxQ(D}A|F4cZ2QK6$wdARJ8 z6B%2ZY#H8Iz-nW|Rx8%tjHCO`ZpB?(0i%&!`fxH9=B3hZIx(%yVaz$#@a(&>sB< zTFcoJgu=?G^P6#U3oa$rpr78QwUlyM^#P_N4`7(G67ZPRR%s`Q&p_d z)12~{S{6+Q#GvR+*8^-*?z@Q;3#7u_W-4E}aS+2yNLb(no|9gXQ-xb-{;WZ;&?OD8 z5ab#v>HTAd%w&gnz-&&M?<3$$8;;a7b9ws|?85SgefihRs{cpQ`aY@x^5W2 zYg-42z=4xW@a~fL2=UI&C!zxK1M0wZ)~M=6OoB<*cBjJ3-HBA7HEo`}#_pUx89|SjtuhRI z$gHzDfNvhRQ9hSSzXBukd~{LSB}AqqSB-PUdT=D#IhX=_=Ns*&krHNB5Z9J+p{zoY zFya|tj(gX`;@%9V9Ss0^&WD^y+Om^l>S2Kv&ag@o`6(hwc)iw~1sz~%fjR7Ha$Cl%mA z=|oB!kP);5WPvW(F$+}c zt`dwef;jT6KNMh5HlmA!U?mJxiiqNS^6WTdFAXVex(^YtEEe2K-yHs!qKnlM@d&hX zJSiChDt&ZPUtj zqElce-rCgdkA%`kec*kT~vivtF!*BW0 z1Pe>$8|IMIsMGY~OFXz_7(%75$!0V9dZD?rw~|^JSHwUHBDMs{SA=rHMZSnL7!ICh zZ&6G+klTeZ)PS*JHlId{d3?;fedMpr3WB7K$K$|52-;394Xub_cEZtSi={|@c6JwP?9DUy^o{C9JJoyh2Em=-q z>r0_ac7R*9)@)xW$U|xjmqEo*R!C7LYtj>O0rZESag7ouo`t7C)I`HAUzYud zh9qWaS5-ZnNRTX+g6!O* z#->__`2bQ0W<)Udo!rXzE@5pvM=xaZb#rnf%$X=IlT%}n88Uj906gc=RD02RDl&3h zO9n$id&4stM8E-y2;=aJ*eU^f1?EXt(AUDFPy;MWig}WxNB{z85=&NXj=CX@dg_rz zmE#jkQ)Y$J|ZY3t$bZi-c3anr9~YmO_lKG z`++S;8C$k<_XfTe2L|J-6sOHV_yzbxglriIxbXF9QuJ-cdPhx&+N5+7Q0qM;<`Xm_!3ZrQMfF66j=}o23k=>Ndh7> z4(o@Amq0QV<=93uI@8N$rCfl0eJJ60W@S+|%j{Q{G%p)-XWvfnHl=h|#;HW#>Wh;p zBm->>zVeKVyBl8sAL=iAH4$?l5TIOPS^QT}^Q|l_#ssr-N>ID`@yaM)d_S|d*6coH zq7_Ax-SE~C2TwtYf1&^(dG@J->2~Y5dKm=pSKv7lFDlSZV2N!S&s0K6^ zYbp!`F@(#Q-0Zvx5OHg^$%;?FrTdIDm!Ag41f9&Bs>%)m%a>I=u>v}ZGXU~DI_<7c1FQkh zq#5C%_&xsEet9spbMunLrg>Uv~29mqSV8{GSMB-i&`_IG-f0Y zR1tK0DH()QH2>JJBoOOi0l-Rhsv zy;{-V_(Xo+NJV%fdPVJ>43My{cYutG#t4`RtAx+<)P%OQ5~(>zf>EM80kgBhTj!&US2xk`4p!xFi``1ZB zIuJTT(gf(tjCvYupO;=pwU%TR5n<(k zxbu)ymq02 zt$@$7T#5l3Lqy^$%!>%^Mo?h*nQzpVv~5`uL6cM3a0oZ17ze?}#OdHD(`O>-j?Gn2 zGWIwbG>9{{K^lLK5jCIvEH4Wepg~lxjPJR?24n#IwjQt}#x~fv6KH+WANC@;D1$VR zq^CDTst1L#Qo2DZZZC#S@M}OVw=e*Uhw(s~d0~C1>=oK@QwB-(f&DEjtI8XW zSlmZdfV{#1Qg2b>@-m>Gp;$&Lc?|ijy32Vj-KiGDp2ADR7*O2UNoh7> z0vA+ZpjZgW`)qGpDfM5H7(I4s{eqfQVl|F6d9I?%>9+W+h@m)|A*iGk z%_FpPnX;6dECEo207+Qz0zxR!pP7wIKgUxupCB>Vv0oOoxhIuXxt*YT$iQa<$F%&Z zq}5DnUjr=oS_L(i15iUnK*%ROaKZ^<32hHR~c1^f&fk@j)yO(uSJ8 zAx)J+hXL5+NFru?qAic^T%qVCkQNXZ}#h%4Jd9=|NSt;Ntzojyx(kb1WR*iyT zXMhxJK^cANL6yLobUxz5*lcZe1sr$?1$rb^yXpb_FTz$hvm+Uo z48hK$(6_gb9 z1cFWSuCgraBYs-Kf`M2aNfc0`+6oH}AfX<1sht@SHX?d&u#2-VDMz_yJfQ^gGJhlTQWY+$HW`5=Vnu|X&;I1BKG>j5SW(Zu`P z2^jA50pCbP2~z@*psZb@{%Y7zcQWySZz*|LC6qN-TUv5XQv4Jz7fYn*!Ti?>Bn6tP=XCKTZK3YaU&=I> z6m^MjWP?rE*26vIO-GGZsW_EDmHJnSQ?s2%3TT~stIwXK6BHW)kA($eJr*cd73Ns+=VI^6n zprj@zX2NUY&{!-R9c_7RAlh&@VMS#rtC8W2o>ypyww?xAm=^_Dwj@>5%3wkXOBK

gu7uw+dv&Acsc1&jpc}9 z1JG06luO9^dwE3e+yN#}N-g0Da-oz!Ud80nk9|G_0&YYjjZLikke#Q-hQ7Vj3{(0X z*edDB8IYkqr)l%a2oidd>_Ruiqi)ahQWEf>t zK=CN6v*^P&iINBfJ5-71W5%^LzF-Ke#ELSO!c(kwKs*i}clAo1C7WN_D{zXbR$n14 z!`Bd#wFDyXPpDZ!_sCe@+j!60Yy2?oQ2S=J&C$Ub`w?{xUM?Dq}4x4Uy(Nf z4$>2`KKmg+i``}{0Wk2&4Kp`sK${0_<`^oRp(tq{XG!2!Xp8acE{BJPVzp^z+&}`t zJbri2p(t%ddo@FmIIBpHgK>B;R}KWW+`AxI371GpxWO9dmr4U5RMGVT_qr#84bYu_ zYEhFKB$mQvlrfg6KnC)lOBkB@P!D{b8Fxz8>;M5^;ca;xt+)yRvPw^A9UNhS!oZ-n z%S6BmwJXJf@kxQZ!wtk3BQ{EVdGw_~wm0fwBhMiKObeX}*$R)25aOuu6KQN>7*|%i zZ3Q)kkxZk}^+^<$sKsN96QHLJu~sQUu6UA`;^~XlN`LN)tm`()LO6{~MhChZ&M0H9#%_LLj~#6x1N&$g_~ID+Ke)U{O$8whlTcTdfrUZ+*b-h2}l4(}08x zgj*zM5ADVngcr4CB^Ks4J6K)4Im>u|`lgK+h1zgDn}V;1ri_n3tOlylb!+8W~nYba4vWSNj8 zeD4@PidjgK$vh=6P?AAU7)@bbX%4u!4YMJjkY*Q&dvcg)_9KfX=7?;V4n>eiN@=uN zObdWNSX(W8!z0nht_Bw?kzFcmG>CK}5QV^q7pf-(o|9CMlLF0oBi$x_V|G0AsU1?{FG10txLsEf+NM}exu@PQtu;t&GrjKPr#4r!!r7aU5w5Uv{NDR3np zr0}ayl^>PeOeKV2``zP}1p!8BdCAkObg;x^OFacWBj5mcYQsWw_+GjO)ClW^~ZhD$s~A2-Z~229Hvr$fNI?~5o^i~l`jJ{ z^45|oojdnf1!1uoJ_C$ZGa4KHssNLjUD5CH;*Zekj+R1BpdbQx24pMhuriYy#%{IC zz&7y^F82??44Kmw0U<0X;Y66!hA$f6+c={%#x)F8G{*PHtvElC&G*TZj9*kD$*&kq zHm@KxU!j~23s~&Fz3G_Pe~};kBmRu|U}wPS+Kj1DtP<5w9az8un<6SJax*)4^eQT; zP2lp`0W~|wXsr8%V5QjM~W%%A)qvjjpuIXA!kqolnd`o zbhK08hnlZw7`tp`aD8!0b7I~|nG53v-zijuOT`=!Zn|QW;6zPO5$7K}gZzUy%z$u= zl-Wp^lxDoCmLvmmIQ~HAx)n2GMLF8n9D4+y^E!3-S zENh8T&P8C!(;f#x1p4Qmh8qMYH*bVLsn2`zq6j+o)Ve)t5Lj>y4Q2=xk28KpiG z`C7i!o}1SXHN^}|+42}DFiu3iKq*PD9iAPY9V!)D%W8}Hp`|Di0Zt$HVX!7q;f&-kaf|7l>R>+#Iuk5LuFr!pjv!v9oI`$mKM#R|f1)niV|D;C!BTk!nZ0qsHU=);tT4S~QPf?ogOl(nnnjRdQJWXS zU?^hmsm6y%Xcxka9AUq5(kW?JX&g$K%m(1hDE0#@Y)y=MwiJ;baVfe11`-Ae4)Y@p zcp@_kL<r0Jcx|YD|cHq!^)r@NMNoK}n$*yt6YVxM>eTsY0v=YLZb){IXd9-zJqwrxne$EZ6~IZtdq%s#COS2~d+Vp~5_kU6y$&2vJXP zspa>zf%v!lpARqa?=%bdi@zMc=B{KvF$wo4Z%W=;9Y24(+S==;AOr>n9ht{^4r%#r4C>_3d|`u8yy+RuB4% zXR8P6XX~54tsi}gQCdB^e0aP1a{c)7>GIk6_0!wWpMG_{I@tW{N6)|c`;Y#%zIk?e z{dD!Q5Bc71bMcT~eeZwWJ;^(Js~KZ^`Rx4i+x4SAU*10adiB-C)wA{d`}NaDtDE)3 zqwA+v-)(-er}w9eC+pQ`r}taV-}4Y$x4I}qR@Ak)CW|sI&0x7m5<7?95pSZ zWFZ!VMU{f@0FdC0$iKuw3UAq4{J$x2WCPjn7TntMarw= zO9(9eGHQKx?0jHKW>8q%tHP;D2zHaykkBo)2D@rukAA`zS9`P(G@%7W8DEHJKv@JQR>-z2 zuwvUhr%W6qA*sF@wnK{~^uvmDidy+ma%I%)zbK*-I6ENNpai1+@f0TZLv5<5oHvk* zHYdEgB0--O`qS?yYpibFQ=p%Z(N{!7*F8pO+|h^V5#$^(2ePJSpuQo0r}`h|ypgVqL-uZnu*uwdg! zTuu6cWIY$9GK*_FS#ICCj zI%XGeun^szcr_a71K#A8_3uE`E;`WCsBh9H%i)5FgA#306)Zr`2LT#t(h6ep#X4uK zg8cM&fCNL;g-&3f0NrD+@fN6W$Uhz-lF086+LVomg+#)}C1lXyEX5ItF{&HvCLov_ zHY4n~wpP;V|L}6_7|q-4w>JdPOb%nb|>lcsZj>&n0)EjZ1!U@(E>n-O)Va z7oN==VB{4Cfw5Xx2R0Hk9r;8>Pt9l&11KIH2QMpE*g+$Lj8w#|qDeJ=;F`HZZ9wyw zERwyWZb_4)_Cm^zTDLYJ;gmKTOB5!qf<_5}KGH6LI&%Y~rW0IUbWB0ac@R@=Idl}H zn@XYBLfndyn4H{umejGRuG2N>-|L^^X~Re&U6~9?v4~~-qOMVEqAx+bQt{T{A|j1w zpYxYEiX$>4nS)c?iwYVEgD$H%SMLb6MXQkEQ>kh7{^Z^ye<+%GD3fG7wUnov#CnTa zCy;v?zWi6zl#oqn)iOxpQD57(P4Yr1;!^N)B8nKFnTYEEDXq_*@b7(WP|po9fPd3v zh;l|J`hiJ}4e6lvrC4htChSEQ**d6u4cdr?3UzIxOn~g8*(@a~RSg#pzDzO5GL2T` zBCG9zOph67Q?;9eY6@ra;bhoB3u9e1lPyYxDkju}uv zA+5z-xh2J>dn&KFU*gdfdH%8iTiAZmT$7r;x5TZ!GAu*6kLMKjbH~{d2%f?mQckY5 zE^5WtzU)(_V!qf2y2wb+N={aLsxVb^&KG$aeT;RlUxLTDmnOC{N69>NpZE(VG7dPq znMtee9m%@H92F1O#-`TlPV;LNZZQM>jiS-j z%2pw`fmI|uVGrx4cZ+Yq^4O--;LwVs!?0jyk$;r5skamd%&L3KDZWTP{5t%=C6vR{ zj2s#?VFZW8!;y|n@id=X3lbm|b-}8_^QHj{?6Ikp*T&V<{7id60@*$Gp9S*Am?Q{L z1fU)^NWsWELIf7sb3F$z*znPWe~+~X@Bw`dC1rZd}}Z7$oWDsvwpxD$miz(uxJnp z6Dk>9BT)C9)4Bu}I;k)!8Oua?dLEH7Xm(Uv8Nn_O8~Ff3fdHC-I7pkAP6TAX@=y&& z_ki$yzLT};RwP8HEzJNWba_$;V4wyOhlR4UJ=XEUH|v8byQ`KVQZkBV=V%tBoOCWG z&sWlIACYc)ZXz!@0fxx_MEOQ#%7xT1dBInzjgR>bUju%6`Z{y{T9BCch#ii{skU?& zm7+pR5^TA$qFI=WL)H9@8EvIY1I{o<(!2(bNE;GllWFRaVIcv;($(@?)CSLRb7x7L zto;_4ifyfjk*XmaaZZ6q0YDBpzglx$QnbS;>C$*&BN-<0*CLxERlNb%zXxDAo)nMq zG;db0rP?&~qz1=^7~rB1#}>n9XzZp2*6@kgAZY5(A;ZyPpk@wnSVkf@2gRrp4&|sv zE$JCTg`dJ3KcrR0IhKl~KUp}3RpO)OMoh9(0u^d|Li;t{grKGu8IN3e@qR?&X5A1K z_5L!&*bt4~OFM$_0k7t%7~VI z|MS{QpZZZ$*H3a?+pGV2ezn@(S#58{b?u_d-Y?hn{>ZML)33sI-Ptfzzli6W#r%yD zUEf^bqpsFp-F|ubc=gTm+a)@4YxUXu`yo$X{=xR@voCKh9{zQG`|6+UeE*Xl{n6I` ziET|WEf>f*cae|Y52F8%)Fn~STLfA(V< z^yK9F+yDJ!d)HN33au3f!7dtc`{!sxbl&X4cS0kAB1|xvwG>5F9e|v=f$lAHZ!HvH z1GxBOl@yvERDjVX+GCBC&9__XheI)XYE*n6FNh8~b+h?q4N5RT7^$cmC3gt8sPF`U z6*&oJ+OU-O`gGOc=bXlDL5ct_njiv~dzsn5(BvsL7Txx2?W_a!YN;$M-P!9YZIn2* zZdMZTMsWbmGpJ;6;*QZqiB19P;Y{=&7{J)s&;l?Mc@RuAx6|F3{*4k%t!}cb9vzM( za1f=+$;np1o8-G;Q+<2<6O5;#LJHNc#sJ~ABYcB;i)#i27HH%bK}ezdTtuZr;)_kOWz2XENtK-LS!q3CZ0#&??m_&6C5)YtNxiF^mr9jq1@jT(y zS5d?-fTG>j$k{E#cw9gUeDm%AcLgwSwbBfP(~b_xaus296%t5K-MlO(m^k$cZCKet z8V1-EWerClUe{*{r0IsypOD_&&0>lIURK`+ZYKyMT9KQsW@dZIfCs+$`VD;cR1F)b zl~4`hj$XW+h7|kDYLHrO0i`1H(Y#wUicetO4KGoi<64-(fk;~Y^agj=7-t1vs>j#L z;4f6>cm%G9yrY`XOy5PzS{#TGM^~bBYHfz zD%nVIMo`NQIP=Dls4-yNrw9Sw*%>8W1?2=QSIs(6AVzj~QjNbwHRw64=>f-pgkV3V z3cEK(^SYJ9s4|^-dI%ta#Z@-qgR;w`jt8#T0lGCU=K1ySYJsh%-VH{hQ{eDx4Gf#B zI&(Exf96<}9oK+?p!KbJcn&Q_zA_PZly~21j)Eg}Zd6s8E)g_`>+NSFgcer&}6V$1q zs=cGt%Z!uS&m8F*I{14r0=BWXDs>)uo>_qLoDYNp2YKV;;Q%z?iMFbLONf+ zvX%Bod6+DwV~goSjgfprQU`=yGbOWk*9VA;rWRmQ8@-E3#2lem1zCiT@YuvzWgq`c zqjqePzLB`09~U?5{JHe1=vjxLe1o4+g8TPP*=8sz2pl5^6J5CfSl=xk&YpX%W<DmQcv4}7%q`q`*Q7EjvP%u;Bi?LF4e{mb^IKwqgRfx>6)^-9}Za%a-bGQPI}W;9v}$Z`b@DIwC)Z_`sW|JFWII`7GO} zr|VH^XYfNtXCFSJGm^HM9iE37BX?{h1c!jc(Uq9N7Ij*Lz=3nZVR zCOUbHA;wm21RASh5gZW>nbEWnMXTHJI!oX+1iCM3-luFK7!jRvP(mTJ9i+Rn z-8k5uxYJ1^(5T?_3BFAH^)EN<$3y^3sAR5}WO`1(7Rkihp?@GMRzfPQE*uY$r0vY2 zx#?3D#`{`NQFCGNb30T#Y*XkmNpeOZzud!yzU55Ty(^vg2F|o>C@MLrzM9k~%!r9B z%JPVfhe>7*=PCm-Nr*I6d_9*uaOz(fKGBqbIBaPTtED5aNd=3x6JZmvOD+=nQGgUi zI{-@$Bnc)GF7eEx^x@5nnn4_BYH6G@;)+DwF!|ereZTIbIn4>*cCLAmnzU`&*42)z zgCc@TPA7CA-0WD>a|$3rG;6r80M#M?Inf$ljl?JSqt@F~x{a}etyVqDC^2G`RsaKl z7~%5VIrI<%yoU+WS>aO>m9f}t`BGpgVY_oT6dW5_s1mK5Moh%PV^+E6Ms#)^Q~ zL=T%=`coNz{D9Mfup@D+GWcP^1n^#p@k;`8C@(J5VHSs|rP-Z@U8EsB%Ez)Xs6&ng zslunBfH?VH$#HZ3O-d=>YgZC~W#5d}l)`RraM;9Fq6y>)=IP(n^Chs!kRUBMk1uAg zb}N`Y;)1r=NEm~`(VlE-3$2F#xIsa*8Q7C9Te~S&EF`Rnkvd_;(W}CMUE=#jf0Bhn zBT1h^#3DfD@Fxs-T8kxOURqlhaqAmHaAnhGpvv`fcRAS%kC`G(rxOTaYH6SZWRes+ zIpYMPn@`x}45!7bsByfZZffxiU)AdcLd!YaHFU|(iX8(6AB zx60~D#%aB?;kh|H15R(LuZxQ74-lR8h-OrGlKE&;mXmdSczM8R> z?m?0W%;X`-cK zX1ejDl{&e3VPQ})kyAjq)@V1~Kba_?b(NK@suFeimnDVpgE~qaO|eO1waY=Z4uz{S zK{-Iwr%}_2W=(&k?RxHlw}0E<1*Kd(I}OyTJK{~DHU$NW-_w^p6M36IRuRyJG>!FC zICM>kyhc>eP!C{u=?o0AL=JoteH4yPqsg>VUxPTB(Mw^+4cLGc&+Gznbpuf9n~{mA z2{4e_&4%$MnkI4XGA?@~Uq%gh+|AS+TToDI=z!L&%0OeYO6}F*@?&k|!ilQg(>Ia4 zi=oJ3p9;3-2o~n(cx2G04UlBT@sOeDF=m~Mx+Q~|gaK@iu zu#9+4Z>Wy68+$+5jFU>0rl{4zZjsch;-|uzPRGWi6ybg5mS|A7fCh)ln@&)+TrtMh z(y${Ts}*I)aw3ac^KhQ%R=oAPLu}*Waz4nfY8-efnChSa%D+FdSmOntcaJtCS(2nng49NZDmPhd$)?lQj1#uwQ zUC@!Dz7}EnWUY{7qI`JGCG_75fp(6sqI__{0N~~*vw|@lpn9yPWoZr*>0xm_jWs#OS)z1LXvS_k7rHUm&8f4oWiwXOm-$UnT{Xmum8F@g-^25PQi+qIWx* z9;i+eT$XW}kcp21go$pHNOH5fS$iQ|obYKoQa9@*M-tM6)gE=Gd%#uE97%COuyIK`CVxF*nGJj;(*V8YtJ4am_Axy)qb_c&v zrzxDlhUC}?*U-MAj!Cp$wC=95{nXN8x6f_%7`q}`iX&5g)+sLZn1~W8;x5%qUQMjp zmw(Jm)GMdJ#gb@Ra~4H+dCf(4a(#g0&Tcq=Ffts%Z3 zf`llGZI38sCBP-rTR)H273w_6<}zqQf&PO47$X~8nAQm{-AWpsgoH0sik{dV7t@I} z&QUWd1saqA?}ZO2!I*H({{=|a1_pvc&M=s%$c)0xi!reWK8ULHv>gJwSl5oBVbZ^o zTMwU-l?%)!F}KYJb<`bQOkg=rdz(mcq|-zEP>VSo;%*HuY9c@*A4)@b$66LH*H|qW$Y(RU3LDT!ci36jquI0jp`?_!5+ZdTH^5YyNYD&AMZ6dA zft#Oz$aY2qBw<(MwGV1gW1e9E(esuLL?Xn28ldi@ED`b@8YTm`{M zMs|9`5v?+m2o}p3R#f3C0t$XSI4mPGa>OQ=-UnB=i3ss%=Q*jON-P`bFDW{@4fd;I z_plZGf~Zzk9EuFd5oi)iw{BrUH1uG9BcG*uwUFLaB}bQ>K<7{(+i?)PgO@G)PN?*DYHA!n^!=UsH}y!ztj&#$hg_2lEtL;TUt zFP>apefQ!wuAdOY*Eg&C*N@gWPhb7j|6Xs}iteu;T|R&E>Q6tvy|}u3c>MISrjAdx zc0awd5p6&FMdpE7)8D9h;M1qqw?5iv->=iRcXY`}B`cwFe`WUC>*a0czr+1^CvSI! zf_hwMB>bRZS5e903}815aEUwvBvo_M8C>qwiOIr?KUJNf`=xMLN0SA&UE^usUv+EF zUS&29snqAlK4ZIO)%4l45X(_6AGwfa?TxwZak99_^4wpV}SLQQAv(ir|uNN z|8dkIP(c_8kf9#7AhK{qk}1^c+P|A>tCW$Sx)(!o1eh?8LoZ9N1mY1gfTSvW6L66| ze$!yDChn56EG%%Oso0068j%vv+&cL2gfLa3HsE;fJul4f57Tt<_%|~gTfldv^b`wr8C9w?19p9HGXzf6G^ksUi`q6i33?2O;F9I}<$w0U?>Jm7^ z;$?#8%Ef*V-wbbKz*Y5uanMW6>9o;s>A0t%SM#ft|Ih)+Tqj0sr&qQo^vmNG+{CCr zR}4cY*hg>MC=!ckxjF8EI3i=97CsyTit`cy%`~iV3lr8qjs>IHqE+!t8~}K2J4_YB zDlxNCYSaMq8qG!9i0C%f2S<%~*bF*u^J4l_Yr*Gc3|JNvd~+b}5&PNXuo>rW@(D&; zp^e}I$v`=`2>5jMZ1_uVhOVxiel8u-l@f||&>u2o9`J~9X5hENGgd8jhn*ULuI|rk zZ?I-#7fp+?NiY(~aI6TkrE<7C>!>Sa5^s*r;v2w}SaUaSfx>L>I^9xtoHzb-G=Q3t zi8VuOvpq>Ls=2;mo2y4`z>Eu8^p7im?UaD{LDLOmg7^l)sQ9c^<4|!^ zl{_gEDqmmRG9t{0)}oE#&+t-yR8(4hB?^^{9ZkQfeL`27Jylt$PNM@fBEh6&oVr?^ zPu}1fe2LR+uh+5Nz~eqH3QJGE0Ayey3Rhu*@rkyhtEeIu^_1QRQ;@)LGcpZt(>iCG zg>Db539~9@8pG{Di*lU%;5)xTq9UV!LPiU`ZDRnlbx1hmov|H=iOp^I2o!$o8mKdr z0yS3&mI90AL7t-L*s1-Fhl7-?A;$A@cOA7t=@Hq?oS;L*v^*mT9B{$P)M)+cehw2i zy41vMU%W;}Ojz4VStMSlC+5x*bfy~ERZYqRefX_$6KK4n;GG`{gPdf;#H=Lg z>0`BhHlGR6#Pek7&D(LDj2-dD_>D%LcSV}7iX3AmhBN+I(OXR+NM5>o63$i=0CN zL*|e}Ov*uMs_DdiXO)RU2q()+^PvOceeNKNm#k&1C_{|FDPDvHrOQBYD6%dWfZ|<~ z7ju*3j!3oJWPDkp?pQ7~#k?X=NHdfO1JJ-%ei6_^{U~Q;m#VwKPmvay2)m>eS!&Ii zv=A>7oy>d&ofr?}b7oZmSsKx+d8>Pza`;ek@U{#HH4Er4$-`qVLu2ufJgg3h%aj@v zoLe;zUpA6`6f!a1LX2rDy+ySH$&9duM22%sp=ZTU%%lMrQ>QPARAy8J7J^ev4@D-u zhTu~R@KiQMQ%~Um)P}=C(=06*vf((Bg-D+wKiQ5Q-J--Vy&mrR7OvdsFad6|AFh2;x#fjf6{%KzoVbQGA-)5K}4|x`ald z;EEGuogNs{PDG@kd04;?e~4{i79;i00YG1o^qcwOX@+_bJIzl?pz%(1GYO62dNKw! z6+GUws1Ce2WcWRsCDFx7Ak;!l&2Si>p zq@1X%DHk>E>50=$iL~P~h^|aA&IX(2;74ocMKg$1`Y`qf-7Z!^Rf4nzRxzXT%fv7y zJoKxFvak-KLdhvCXIg0W8Ub6#{YHD5Bo|edH7(GS$G&I^p3UofFzc$BeU&k9!r41 zTCv6n>-_@HBpHExgq_THsd8}U(3{DCX}>m#{^-uhS|m^0?ivXcj)Xz5m_D)3#K1-3s5v~x3{3kKzDkb8r#D>K1I$QP|73FyFN+J*0I2*-nTSw6ofU;J*qF6z=X3~^pxsbR~iHuxyXC6s%?MUkoM;0ysD7=T#-VFALE z6$i*Xv5H*<9%wEv@Tj3^b7M6%fm65kL91ILK^(tzYs`li_(PDLZ^l)A@t48cS^1us%J;`p_LN?w7k{(Ud@Shin5f)7GR~(y92B5FoW0i*mCVB1o3Hv#`f&c8 z$NA(Jh680}H$uw#avT31AV!2Yl!2q2MMh1{eLGdHL7q(FDswK_jOq|2`jICPLWllx z&cg)CS(PL>0*>pCF!bD8o1zNQYS>Zx`sWQzua4Zj==K(vWA~ z$zink+wzwgyW*XL(#mtBm3DEzggo-Xv5;cqD}fLM zphDMaq2nN-%v-w4`wCtrcLypHlFZ5lKjj(W4pV<{;;HPqQWv@9E9EEbvI^-M6hJ{! zY)ScyyhMG?)VYZCZlE?f%g{Hf7{r8rENqP`$J1=~WX(k?H#muHWWg0G@018#PnEd_ zVa{2oHUiD@+AxzE16hOuuqRCaApaX3LOgDY2=D>Zm;~Wn`7KyC6kMEvC>mBLD29k{ zDu;qnJ!n-`5z0JZr|Gg7^`mXlzsVH%2!=qU6I|(Cc)h^3%mPk-)GFA@HVe&mu?Q}@ z!7`-c#@jF5jK)Z>2Ey)>l97BsD0YB*1Yeg`0as*}Ew>U1*n>4Ra%R`VTy}#yq*2id zH;);(*XV^wg0$HQEnKCztH{)uFuYO_u@#TbUCuGuG)%RW&5tP8!2tQQld7=c!)Y2r z@_U}hY}{E+O9gdc)=F0FaZ@3F;+y#d6oXcPVIitl?W6M@eQs(4G^}#f7qINuSJwqE zIh@%Gv@UrvYP2qdMJw7O?1hT6?oQ)qY3d{ft|6!jKYSAQ#|3F-U&Nj@y~n>-=BVW8 z2$sxVI<&9u$VP%C*;&}udu zrod0wY&oOt8&QzKK!s{2HG8SC^7>5ZFX$U*-aS|t+*~-ZyZp_9rAkv?^W|+zHX#-& z>t2EGc#^@?tKFVrt-7&@=Uj(GO&=B=3uN#UgN!P|k;fjBT^xG@8FIolRvAB}LTaw4 zN=uRC!#hku6Ru0N#~@%fDGiPwyPdkY+7`b^BttMW%6Q}EkScjRsRpX@kSN`q^jZ>q zYgJ<;wQ8D?Uf@s|cHK}k!rUTjf}Ln2k6 z8C=61No?Y!^^r+#8kgVca_m-$AJ*fH~{e^WYI%%lFgUV{%&72?q%oR@Y zsR7gtqg3mN#3t9p-f6Q>7W<#YXn&6ZW9a4bdd`)`W-3!;GlW5aS{m#!kl{-Ysg}#_(A|EZ#6N(QN1r;~p+T zl=oq~vmzIs2r8%=?b4)enaPwDrb~&mS$I;WKp<@Ru=2T>EFY(%GT`ai?fwbti=kNj zrK!!?KTW+(-S8e?B(FZP9g+uU*`bd^B(g$Otr@V|B(LEsC3`#aM3p*M958sd$pJEj{t1x0O~KLO>1!RJ!@%h>iCboLa#lOA zq2st^5XXe5AliCKZI^}#*lZS>*VYQNF=(&Ul#+RqN{=d>CL~`$QotaCc@P_Zww^=6 zsF-RU;FI7r(vq`SX7Zc?_QH49V>qb9AXP)chCQ62*-(++s5F^Q0Oa@{{)(;YHXc(Xf<4Me2U^ax)lKq5q-`#qR zC37rjGi2=Mo7LaK?;*4pf>6_}h+h-EtNl1&*jXA0+D#(92<=ZKz+QUJ0HfTFhZYyb zSQiy|KHiHP945t1-aA_SzG9`>Xs~{lylY>ek^vz#BvY!G1$1IKWI&$Ag1Wpx`$9Jx zuTJ4&wXN2TlE`^XHPbEWk89ndQb0{dv(aOQqvG}LEIJKnKm-!_G?y$KCzK9B%kLs> zkJ9Mn0hR-`l|ZkNk4?iqV6@g3mkw*^k`Ne3e5fN`Fxixe*3%BF9_1$D3ZR*(p1M9o zePrp#WuCm3sK<)oR5G^LnRPV!#v-d!2+&dhCO$KZyaCmNe~?98il&@Fp(3J2T41ME z1xhb*^i+Q)+#v+$yK?@#rv>o&ve=S-R8wjX6|G0tr6kV1uag>SC2L^wJ`{DbR^Bk^(YY~gk_Khyq0+s zd^SE#La{_ma!rGMCnzg1>RGQZrx|}^iy@@iAChxjrC}qwoBFH18>3b0(v&{krc)tK>nvPySW!lqL9*WVyjp+ zby>F%bn7#_21UjJBSkf(!yBjyd^@HicC@gHWNvx_t)*qF_fSxP!l!H4d=cM{sE?Fr zIh)_Oj9`&mZCf&n(nq>la*a4PNb)njzzf*lYdihH9RyH?pnCMk=EZL6ib~)xA(1T(Uw3MCT zCLPtlDB9TnWZ3%Gg&$QlB1#D;XlBTPH)}!HMrV5{dBk_}nm9*(CHp};G#<>58bDuG z#0@N}d4%!^MHuYh>?L{<$Z$l9?FhDrm>L)4n%QH#9y%f5`}*SQ@lr1Hb>z6nd>?Ag z(l2@R41`R(PwKPKqA21>LJo8x%B-9km1CurI!-St>Nx6}NT3IaHg~!x+SNA#n*^=% zL&fGIBzAY1PBZ{A?0Ej4NGKr`{~+zOPwhB5+==Y05QIx|_GDbpmZ|bln9*JXOY(s) zE};r{1hH7Ysn^j`K}uH136dbFeR^~KlK)Q)=Jk>Z0-uB3 z>Qh#}mcs&_60}FIJ62b4*3#++1~?>?aX)}bF;Q@^Wt^HOV{l=Z#*TT>kJo%q9~@S} zPWHutJSdd><`KfJj9h4-Xb2cnZ1*#y3+rNDQb-%N6-_mv@vd<0u|?9MqMb!c;)0U!Frp3egJX}1Q?i#0Uli*nEmLe%Sb~-Scw*>Uow-N=yk6OXQ7=@du z5l79biMI(zsLDn?jwMM*YD})c4-8sUD~XyjVJgcP&kJeO$jRuP;zhdpoH47^=GHX~eDV!Cmh-Rw@-B{J;^;!tmA0uKVU`0`GAQX;c zq#lIW;7ds#Jn|6~r&LwYNuQl=CFGZlhM{PZh^Fv*%B&bOmHL+_(e02>z6cpgMd0L~ zv%f_X7ARe6I&Q46(1e5$Qq3jNmfhqJ{!@s&L`mm&w45<<3+o0sWF~}v@j2pw53Dts zeW8ABy1!woFnYIKsv8=yZR&^=ZizW24Kxzj(W_yqwm+$~ z`Sun@?L+0pf#QM$U_n9sqqP=?qBdZ6mFb(xE3q``m>3N`*zq>Ild1ckG%mzr7T`>e zNd@6E+Dh=K3b6ZSZ4X8$BWWx<1|Gq#eUEQb&DG#hbRlL9)2^1~ioha(^gfIk)DS7+ zT7RDCSoIE>pdbww(C>+!?`#BeP_jOLy`9iR-oJ=a=2uU3L0Lw%{fP(gdRf;%v;TMGR)mN3C_@l})x%tO{(@~nrmvCPV{d_gJ- zNmQ?4L##{&=!DTeJWw9w%*hH*DaRR+FAyjkh5}JTM(_<9nS5ozs3<3G2+|xdB|%}s z(`CG4#~|#JOw1-xx7ge~bwV<3L_J{{uwn9PgK2VA+o1vS@F7g#fXT9^{i_T-{u7}P ztOIBm_Lz|1KH*z56%@fAHKm<6lg;fTYyvvIlx|GCiM@7RPmvXY3CbtYix#E`OFIzV z>Z~zU0hoImIhD5%m5C!W*M^5z#}?kp-|ZnfRW(a0za{ppzzQoxh&^U|3fCCD409EES@kjlA}^p$iBa+7Uc3z~gUGX0 zcsNcK@1Qgzf~taG_zaBfQOXJYm?H&0vBO5pDbU?`Ff%xE)jOCTHFyH2nTroOqgD7( zBzKNTSxDx?M)Hq=Gwheesa;ojDVbSJ4Oa=rL?tP_8_}VxQXC-O^sCAtBC0nOcC9`} zXRdg#-Hhe2{ao?N5EjkF3|sh>xPf)JPLfuf`7dfUg$3kUtgUFnZ!#e{btetbjg=Pc^CoWR!UV)!A*709KS!Tg@bFV(eIFSmaQBCag3tJq1Ao*Aa*f z5G37+Q6B`5=Nrj<>3|fqZX>fVJm-)YTfc@3C<&Dx} zamSGhh3s6V-4ez^4Fzg6*G9|*TW-pr#9D*K)n6+f2tS}1ZYLY8G}zR=V6oCy+~-a1 zZUquka9yGjm^hd?Lea@KweJc5V-tj;sjCGLLvg&Z${21HoaHXj5^j*&MAJ?HZ;%() zXD@A-fbHe^pIB7q(g(+>o4D9@Fb@&J&VdKUWyHGd^kcRCDixF4&I$J1jFcF+8hn8~ zs^es_FBEmQjmr<-@_#o3=JEq(eHZ#B5SaZ0rVyDS(W-^+ zt_J)fh5!YM1++l9O7zleW9k@gR*{Ax3sjL47Ceknnsg7(B#*0on+mMh#^HbtF5*;y z(Hfx$WQTW9NP$auXm|Az_6@5t7K`StT<0bN-Jxw729gBxsIo`NK!tS9CW`fOz)Cq< zPk}kP>=hslE>C6NSd2Ik)rK2BH1{YC>Ox~;Nzn0ylzJQ(Fl}PJ(h_sfT?F_a4U%|3 z)qk^H^4C;Z!jTx*?jCJ^Pt(&h394z--^6a^N4|MT-oHv|%hd0xP^BVZOtDS?<55Xsvie>6HW;`oAEEYu9hSS%nG9`C6U$qgt7 zYm}W@8$=A3+*{Pz2E)(>K`N4Reh+mBwB>mMy_Ql{?|acVtqp>xNh|7DPm*cE5HTPP zSfU7Ptc0h!;|7**;!UUrPPHrQIMkx#zY#*%B5FA`SC4O@W;(!JUmrGHmkPH6hy_wA z&?_MgbVgI!%w$ZRlaa4_+E{341eMi(IUx@)gcq}~+ol+IYecC9u5ni+Kg1ratep&0 zx7ao_RHs_UL^Jy9eW`khf~pSHIN2o|bUtrFH)~Q0_^9;n@(~n{659^tANR-*6{=dF zg@**-5T1iAtYz^BEM_0{mGUr;}C*i}R597O{`72Qo;PWyBl zj=HJYq=>0LP%|f;Y6V67B?(BiZ3-nxgrFN%O;b{PBR&y|2w%?}AzG&c!lYZ6oI{9T zx21xX4(1B5E)1zYa}A$Lh&a|=Dcv3Cg?K$cQ&J!{swNJ~M^w399y}NNjzR;}Ba9Wa z^p^;^+CiKCN?*aAYK}7NM)NB^%SmLi2YS0}kX@*T3}qKSGp)|9p@ynW$U0twhTsXt z?)CI_{ram|vp7k%Hz%Se@8l(0VXD=Z+B%7J$QQwdU~b1+1?Nr-qxiDN#ZeGGusB{3 z%i=_1(YO&&D|VwUn#C5Z0RpE)MN>3F^bQ->g42sR(rAS6LAL4@Bv2uC6B1FS&F{`~ zv6xV$%gt;B9y8Ak`7f}^7jDUm%P+LbjkENj{i6q43cIi$(6(kY4pgHB%3 z1L9oyX%R+N;P0FIGqW)P|X)lhwpejs8g7?t6U zqVHF+Qa47oHqyzBJ+%tB4^vnh?j&vtXXp|In-`)D*t?Pg%D)bZ`iIH1tZLoi?p$ie zARTDNntpCw5`9M6nK_|k8ZFNL5;f?7(SP_zvcK7)rc|mZ>1~QxB@yppaz369nO(oP zPZVc^uhHM;8fKLzp`*JL6`7mAIE0x~lfnNW_3Z!NR0;y_Zwk zM;BAfAZrpJCJZyL>(A)P^SaZtvZ<7vMVo5jZ0C2A1o-vk^#rr}~0>)H%#HYp{gFiAo=N2g)9M-cyZ!o-N0q<$J z!^D##wBxG`2rc?rfMcc>IvpI++#|}p>ejU#%21fUK^$a3s;Lr^!L8XiJk2#qs3LWf zXorCpym8$eZXi_^GmG*dP~&ruQhAwLcA9)dN2%g5dbknLU<)r4p^dhy6m_{UsLrR| zL<6`NSxygQ2pOq#K&DpjL}R!I_ED(UJ*$&x}F?4t(ouAUg398!;SzT>?Y|dQfvm zSPW_2dH-~W@G&#YX@o&6QNZf~-k^M&wv?BO3iA%dUchEiKJo6>QphgDXh;>I(ie-CNy5 ztVvo;YWq1J3UwY<=IsV)DONpco#-krDlP3g#e=&PO~cZh3%L70)wgs9Mu}!Ub8{$_ z^ZjPgQG4JN#)yYIaP6JKMFq>rxVdS)fe!cuoC7hYc~{x6yR%Z?2E$XjyyZ}(u&|My zO?x@6D$vlwzCtWz_EZ2g@mlFkb~4w&gR^}VcFF~uMb{hY7{iKaytjD_Yf%|DbEgX> zJ8+a*kEQ+|4iDsq*CvLmSUvvCx;htFz_R|G(Rt6=wmp=TdBV2gwnRg6Ct`zasl%M} z-tWcYBxw6yddis{oO+6q9xRB;dMo$3^aj0PrwPIeMLq=rI!V5&)TbfGDM@r3=yGc4 zIo*<;rLDke)PZK*Q5_^+MKWu(B2LXuo+DrL!P;9&cxrZxkyEL37m}>EnldK|Z?$?+ z>j~?S;R&ka?qM}5IF+^lmIZ-bl9ul=TNBZd%Z=07MpNOEOOYFl2g-K?3l#H0fT~by zix)J?i7A}aN4&0`j*D6SgV0N4!$X{ViG+^cj|bXz6ya2&Vt^{E`2;2$J8qQqnx{B6bYQU3;c>7MNC}sn znQMUju@>r7NrAp=YkD~m(xyj(d5U#}H8ensP(9-$I9Kg0({$9Ai~*Pi(w?VQO*id- z_PQ-=5(SPxm0WLwE-K0(K^?iH9OMP)lBXSBmJDqC2<&2JtqV6Cz$|C9+T2s-Ga1an z#HKtPmaOSOA{V+Lx8gqOxdA$)O7X<75Rrt}Q45M1)aazfn+n_cIk{>0p6oMT7kYh@ zd+m>tQ!NZmr*Gw(*To_Np)-lLs1>OoAF8K}5!F3caKfC4Fa{-Ln{0!*m08AU_p-mY zMWk09Xx@w?mxaJ{r2IF;*|MWWd2-OUvRk`RBu1S&iezb?8!Ul#Fd{uWlp2~<%pokt zHTM8DpxlkGh*nh-WEP{a2Eh005nGTBvS)@ZB#XIXF zu(ul&C4(~KXb8fXSpBXkH%lkyE`{7Z0XP_o+(>TpuL;kdYdh7U6cfY>;RM0aBs5HT z<9tvMII+@wvoQb@28?`Ojj26d1V8DK??mHE9Z%i-VC0tF#e#-qi#w%PLp}VdwwILW zb`}oUh$<%ArhFh|tb9W$_}^-;(;op@2yQG7xNYfxU6FT+gq{DMTaNuU`4HQ8Y}4y-vi_q*!~6W038ZPT3VN_EzFJ~Z}6!BF>s&IV6^ zu`xvm6Nz1yJ(eH(gmqyyXqp!&#fgJ}N83w8+;ZO8gM}@D8sw!e4B5pR+dDobOR+?P zc-47?<8+MBWgZNEn9s+OdZI!cyksW`WAH~k(^i*dAxas55j=glx@kOA$DAdDnV>dA z76xQjSpS-&-Doai1t5l9Yj_Gr71U=HNHiu*!Kgb7LN=T8 zjG_ZLM6%j+fm*O!UJ%a!5u_^wU1tUJu?_C7G8A)sB@hT=qWv{!!CgQa%*))&t_{S( zH+e*`-@kbFPU7>=^G`(5P0H>+Tv8Hwo?GqewY3*r!H1XOtu#fWJ& z!YLw{MO33J6CCH7pKG?|tOUwrbPAU4^6m$+wP9Lyjdlf_Sn$dp$>eI(pp57rYkmTx zO-VDjNCbBfMH5^%F{<8DWAYxjAynJz6%PgQepT?g~Ye`cy z+s|x9Fr*HDCs+3s;47-;3PR8`0Xyo`Mgn^0!;()^seD_dLIBS6CuS_+PvBQ#^v5D$ zll{C$_9w^8NtJ+wOgPPPB-lVV&9SF*2}V$w)ad$GpvseNaz0E*%CIIRRI(e1$=+4+ z0S2~brAydG@ut5=!?7si7n+-|`GbZ)PvPlR0f?#Esz98|OoTr9rfLOd6<3v6rXk z_XWz5@!(l zWNn6ebu1{9IXR(2HGg2wI10XWPuGSfN`cgTf&z{9f307FL&K0IHc{N)gW|9a;2~#% zJz6`ugi_*wJ{>-phkk|~@MyY=q+(b(=X=3qh=h7*lfBi^Gm%5@PU^gPI-N#fxl+E7qdvJQV?fq)+4)G_QQ~a3!|ZxZW1OqTCJJ^r zAbAYIc|Bep1)r>dlCG^ukV^nE6#*NT3{|$g@k}tTrYYa3*luKD0}mN5n{`PAQbr0$ z;(q-=^07B1Cd(UId7GOo4>dp%Bf+(((GueC2X!{f9)KQawD8TcEHkC6tN-X>XcnEm z5nXPu0vVan1!s;1%554Y$Rxos^!s(~9Zt0uvXQLN^y(RKsGyW6`#-b>i40Z%yy{c- zmQ7GoRVzXAIVKaTjx8flo`o4R#XxEydKd|NzCESXnKo>?X#POLLt4{BCPY%M z#lKqf3=B5GuxnYYr8YMnVaafYODjE55aNSPrwG-UY&ehoQ(rJozU4OUqLz?u$O1qh zC!xyWjfoPQ0Ykd0Qb4+pR3sBuP%18Uw)lO3*8gdx$xkESp+2mm&#(CF?G=G=RK9HC zCGA(urXPq!I`&YaP*K)~=CtP1j>AK*)a7Yr4ydmO%Hr%&!!>XbuC&LIU=dl&g6L&@}(IBI+nA$jk#NFVTXf=uPQUmR`PBl z1JcFrcKxCVEPC>`NFbIt9fe3t7wq)VmMS`bwYy{BAltEUOSVJ0R(X{+7BhJ zm}OW`QbC}jXdy_@Kwq9x2HN0DR|D4`;9zq^!RQs=O>A*uy{@NYLa4MOXtMzTzuw$Y zKxfrVnvgHdvO}OR0wK*}=39!SL_iA%-Q*_r`I~wK)JI;?Tn}ff3E|v6Y)S&2b#m&N z{kxKGuSuzJUF>GJNGW4|>bkKQWgIW=aZLep%GtZCZkgmggYkP@3d|((|I7mfq^Ng0L-=#7kIR_lNoZrJ@^PzY zpcPD!AXk5nhAC_$BvLb8Tw2G)z68vjMua6#b#WQx#Gb5Js zpE2cx;e`>6AwI`aS8L;UFs5AxCiT{_&@kb6RY2D92JO=mj33pp3f9jUh@2LYdTM=) zMTQn=sSC_dKoGUfJ^ds_RB25aq4t!V2n`Zc<7PE5I%-lP($s#$lQV|sW6?!fa6s&$ zszEQ`9E`>J3J<~<>?B}lAiyLyGXi-&A%RJ!!B2J2*B2x+=^>Na{#6h7$IhlTt)9_jtmJ9pQ9Z-e*^#kf z0)<$l9m*jKB%)3>cUsV3;IUp!@IG>~=B8{%_HP!V;uRWBtYaFK5-lyhO|Es|SwS(; z>?^c}@pR(#p)Yjwf;nU(6VK~lD66qz`H08@1{;sN;@NzXAd17GHnjlcG=Rx`DTTUb zTWHkZb2!fTl3}S@JMl+CCwpkkX~^Kg$6cx6pxRh5z9~}NV=$ojiw^WyJ$3;!);BvQ zlTPPSpvDw*ZKp^97zj8lZlD2V=~93iD}-uckcSJXAx5B=l!ab6bwVZqqy=hCj4GkT zA=rk6YorQb8;JwX%i3Lch{AEE8Mvqo1h|R|Sg9FQy4SkbSs{!l_7~y{#Cwo#%on zbL4GFrxei)<0CU^10Hd-ji92w*5$*~>kG3F`xPZVqSFfcrYKz{l|fArA@$z))4Gy8 zZ^xSP%zsk#$OFheFxzv0^}#J47uV^b(`eu@f~_NBtelZ*_@2f9bbSlPBRbrNB^TDx zh8^%=AXsnTl!v>FWH4`oPo5CYb&*D=R|4cVKWRp+EY0q5q7 zZNik?phZcA55i)$$uV}aZ!T~&`^3q!V9H{)IdMpRDRlWJ%1ECU7T^_ysQr82AqO4b zFpx(x5l%e3>E)M55eVi50L)#YneqhXrb$R-C_iifGRr<&FdQtZND@H`%#XRNUNyET zz8KRQZ3zIPJLz*F30}em-PJs%o0%~>EzBb(vDNsfD6~v|QAGl_5mgYFu)HY}uIF@G zno)shY)p$xe8@?e?A*1Z9TfIg*qqH0B`gD?-6ddctAe5E?cv#UT09|Od)DtB=v;?e z>0|M&wqm5CVngv{71E&*G=fD+G=$QerjoOd;XK*0aJ&A%#LP4!JLHl8TQiX3^r`ii69;0E}W=#sRpKXFIvAzeC8U$ zZGw7R+-ZU3x?2F&pe<8LK>;N{%}hn?V|aUT8KQDoOjr3SSs+ifp+G{MrCZ|*NOBSb z_D%V(z;plv8}L#&#BlF8S_Uan<;qFUuZa~AX($Y^HhGt?odV;N^7ua?hnx5iGd5BmWy6H?sbM!GB>95j&Dx&Ei zk(r7qu-jY(r4~Np}@yRY^92unXSVM;@qn+{k^T%?S$yL0Z@t z+r2!VS2udb{^QFK-~6vVrWz2Wt2;y%-OQLYZto}%2O({;F)XM)IMk+At_!ePW>BwK zR(2dYziI_>(5#Of>-t*JrUc7|D|o=&Afr)3NdR5ZZ#;LnLy%MjLLJh9N)E3Uoh7&v zF&KIA?ru|Ua;-0?Xdnpzf#_PhkI-N@kP(b0YK|NZ$JSJ)V|1WxSv+dQ7)h*1Xo{wd zt!zOWh9R&si6hYzWg^1#UapDCI^yMOgVo6=gtN$h)xL0J>whbUSdtB9Sk9YqU;+p_CAW0rjcsXBw z;(Bwm)}i|?WDcT%yef5AIZLYU?aiHRnW0N1q%SHciPB9n3ec=-2s`Uz2BYGFirFg9 z914WGy5FO~rZrVJT1RNYpr()W%$rIE4f*(lbC3v1!|~u;d~o^C3>x1S3oQCIf%jK&qCqj~V3j3w$_+u1cytXI|A^DD=)x#hh0~~T)L z1&HWrI^YTpt0}y|X-HH>eq&TJHd!b`jFcR9lM>(SNS1UvRS(ggR)xe-=aC1R)#H3N zc0+DdlGv^o-R>h+(#WyVT$$jUb3=wyzL+s5h1jaHS~B)Hzp4xnMr8@+1$R`6%2J*i z=U&CpxDP5o+s!I+cVQ1q3DO59y{r2W*C3vFF6Q%Fz*QguAD-_4d|FsmxGX%J5#SQA zALWNb0GJn9O;v6Vh>w#m1gB|QmDuPjfX_;9$JyRimC9B4Z^lH=Ss4lvJF4734TOD0>7lVF z_!V4Eny}yvl2C8NpdVtd4guhINZ2oQz6L25J1HAJq3rlS-uluYgDUc&0B)*lnj(d zH#t)Dpx_(kV?=RPU9^p2a4TF68w=Fji_p$M<)@`LaE?Yq4k0D z&M{5Paun7bb}WG0X0gmAC7dX^k>$w~KAqv98PdSa^l<#( zg2tSv9~A;nk8|ArM#I2t=S*GbQUoXUDmVh=IS&X?XBaNDF*+4^Lb`#aZPaNz4|j!; zQH~6)O&*go9Qh{tLWjfv+$3X&gn=8$yh6L8@j0Q&{?tGfT?1xpCbAu{t@de*8XiZv zq1vH?dS!;eX1CzvX7Z=CY}}spV6(8uKBhDalA1s0kb_Qk4SX}v%oi@jr(`?!WNG_| zBUK$~dZ0&*eH0A&$8eq7EEJNpK(4}TUQG`h)R39v znroy#-eqbDkGs!?h@$yULGXTkG6}4=xX}_g{veCs}6@(yZXRE>}Qg|aF z+~rYLpG~9~f-+)Cf>NMu^%4+31QXy@pSfh!#U&r{QiQ>xi{$eEA_5U&Dp@cq4imW( zMN%hTQ(%!J7RHrtv>gsFvYj@;&Yu9$>0zaDk`NPl7JV!f0FhoKQ_2F$;tHIMpo0Qd5g&@O^QsO z;Vw{?fGTmAoIqv;1KjjaqQa@^yx6Mdzc%baH4=nWs;K56QJ%Y~2K<3?2ZKz0#|F{C z3}*1}^+s$h=pK{?F%z}Hib>67a0(}4ME1^5DISd&;n?leC2IzYTA zrWgOA!0nn0tc=g3LN@t?*@{ex{8vWOJTa+ML?EaBHTFnCbUGg_JFQQ~7$7a^xVT_E z1)vBK00f>8D1uR}1+ES)l?^)EJi8PXkHN?_pIRk`{80U(r*a2Tp%iR{s)$oq^t{RS z2tw-vTKZ)QCIvK$7@X?ab3Le^q49CLP#l%6unj*9aDnc5i05_ zr1dZ=Dh;ThYBKd2Zox&JOIaYlT70myMC+z)WLvO}fI=V=WmgZBZ|HR~%gygqxB#;VN=1Nf z9xP&m!iU~yq0-+jQDyrCm`tjh_g3FCdoD~G!(Sp0$w;Y(*l-n|qQq#>y64aclx18@ zZg?0Qxv98RT+nL|&4KhmIX{|&%@4*epmNm4#{Sd5kDPxo)+#`Jmd9(!eO9bob~FC- z9d|c3&K)J=8{m_%bksom#Nq5!)KZZbNKh1-1JG3wI5ONIf`YjO;Kw!)@zg6VwiosF z*!9#Vkwq#HH7eejJ5GUNtCHh+IME*HB$c(o580M`faallNX5dE=l`oZ3GK*2=1tWB zeKG$&a$f%oD@HPJlT@(lHYpMh8zEfa%{1MaEv;RSvouE)gItp`s3IJoYJ)mM`LemH zksqdB3n;V!LU@5zTK(KU)Wx zSd|`w!=Fu#Q;{=FJC>dirrji08mUQ?MzVMw!Y6w$vu@|75lFZ|W-*_Q&rp@XKRmi3 zDN<37S&PG|sfpr<9%n#NTB$=6wOqbx7WEqmopTBUVT0OpHliX4sqZ|q8eqz@gBYfX zimGF+ID=#4Eq>k3h<`H_QUQ{d+01%;Y}V7;m+g<^%EZA1){d6d`+Of;*aTokp7 z{hi%{q|0lg2@;>@$@!!VO`eyqNpfSl5!1|DiXE~El=GRFEKxl=DT>rBW%ou`)HTSPlvE+rU+JUo>T=>)Ok_kbqmkj)mADZ8@Fv^cfGQ5 zUBLIg++?a!#GYbTI#eV(uP6k|#CTqV6~hnG z59LQM)4?JDqoNQ!vIJ?-hqUnp@qBX#8Ql^mcEfSwI!*P;iY&(lSPNG@uwuIwf zA6ySxM#x)Cv!F#d0FpVqfS6WB+N&*0q(%wnu0NC15gA;xvL7D}vy=>wxdRa zE)T1YTusaVLZ7JvjP1c?J48-A4zkWNd>O@r8tcMoOop{^!1QHdTM#{tPbrUkzScN$ zEjHRcSdq<{MtO*XAK7$$7WXjmgEMl^>r(@T|C0n2d2bLFKvTNCL zv*p9a4?79sXp|?SuFbm$u@W;$e_mDaZyD=L~i{MH7cJK!e!G zkEFukNo8l8JrkPY9>imQnR*n081BIA4L?;MW|*=#O=Bl0sgYpZO?AuS#p7J z9*FK;3*x_Xo{BtZD~2i%5*;ffvM2&BmJk|kqJuOhTsrRDx*k(bG?Jh05HBPmP+vn~kI!F9S;7)F3^pj~D$ z`n`tdT0XGGjx&Z9Uq`-2nq!b1XKGsBpm3hd4{SJ-x_NP&wZI^qiA1yz1Sqh*vAt>p z19YL`mV8Njv{yj7Ac2jHBOQB;;NFSz1Y|QW#wG^Q+XRA4=Af?o6YX5ittAOmQZNiD`&spoL+;=dJ@Ds4?5E>jg-+a%7SimC4>!B+yelHV^~; zhR6UKU&P4(@Cb3L7F7T+uQkpFgl~yuqzTWhTE-1l1QD#ahrrz{oEDqtTl8H0OUX^A zHFZ-FT;9iRMM|$NF{?0Q)buqGu!}cNVPu8S;)j}jaI9!c zr#^O7_>?Hv`P2redJ%(yRj_l}y^L5K5puP9P&WM1c5u|rW4)$4QUoM#`&GpM%KbY7ePa6!AGatPI!ovMy^QEr9Y& zcan!ZbWdti2s@yxgQ8NcISf4pCxE&l0U3ov%B!nND$}VTDmq01{~(4!X`Woyq)>+V zEhuU}zqV9v6wkp*Sbc%GZU~OpapqSRGWM|>Io)PGcGU-*pkYM#WD2kwYIab-hxaXy zs*Fl)M}!o`yg(~ulu=3#x27yj-i<&|PjN+f+@#gOb%0x_$sf;jNeu%68KT5tt~-c` zNd#Ww09$%{USyY@s-OK14^c&L-_V~3-_C_+#ZGEE3LP2thb+L0yq~Ie4G18nJ!SGB z7@6{)!|E^|0B7(Kv#OPGINB;oPpiacJeh_ILJ;>9Xx~C<%`$v0&J#S< zZ5wc;kZYgly<*`qMka%v8duIv_J<9H4h_J$G#Ow;9F`ujTSbbo(A6r+fs(Sm%_H@y z&=`ww1_F`u>;v(eW@#~|Fl(K>r;72}o#J}=d>I*^$xkE{VyW3V=}0v~1^l@gb%3hU zA4mUp%xmLnqX?z9B^fUJYBs>`)xfX1KOTkSp`c*ueGgl1huwR zj)^#s4XjZ^Pm?S4kvQ(s4yUPY!Q;*$XQ}JE^Dp?tAzti}YMc^o#RVH%g3(a<)fwpi z0EdZBNKR2a63}cR&y8cmw^P-#Yf3_}roU5537$&N&YxYc>&?VXfxY1ZV4rwyoBEcP zC1;FMFvG4gg@0S170Su%tX3NHL>v{`z0QJ4Y$p2x4r!E84xFlXeN#QqA2t^;xlU@Z zvwEn)nV-R%ID)!;p%HsdSWzKyNkKk2Ye>NkfxZ!@HN7UfM?q#z7k^V@jq1d31aZSd z%;t)%u(l*}WQ{Z^cSwD_OgXpo@yTv<3j#QD`8(Bic`gzQg@WC!i^vi9xT`% zCN7y3uV~sbt1C`Ig|9ScmajY;p-mS)bl&mt=4x_Od?#GbB`}EU~%TuGYV4mKvw764z-%)BIiGZHjN zxqdr{*^B{rnhgo1ZO<}&Ko|B4`ik*~Tc<05SZOp^fj^2PC*18mU@9V1zQ|msYE>-T zK0>-Rt+R_nUH2b^ra3%usGBC^Yl*VB zBB)R0fkTQz&JtjAf{BcELW~sE`#}7>vRb{Mq+Z)$PE8OWC9mpZ^NZvUUTkSsuz*yQ z*(eu6ghHE5POdVe3A(ot)O5{@@=Y8UDcQkB-#1X(gp)#a2TntXZH%N2ci2_nsKPS> z-j?$Z7(h=^9jUv=wnwB|Eud2AfON7NyjnOcH5Vd+CBY$5x$?t9=4vR=n~^@q=MF1X zN)z>13uFLxUB(X%NMus|0b(Z}&PMW0T&c=_VQYzbVuBq3PpSD~&=qoDBYh*;W7uuQ*@8VXjr1g4tW*(g|A8?=9l{Yz~cAyTA!*OMNTqA>1e& z`Z>8|l}=8QTEx5cjfPJ%|E!4_*3+)Ga)mkUR0@e|!iwyjqk&Yi@n@MXxK?z#QBJ@e zkpm=9ZPjc(^VEwo%zc1m?Kv_o1^|bmTN9fpy_gdr5Ho>oeO1~3MauH=R8lf|-Rqsa9I*~ID*yIzlof07(nUd9PG4(Q%Q<-5A!9jWyZrztxubYXcyOaqo1Z^Wo_u^0Nq?bk#awMrG z(EQOee18fnazJ)5j1j|ZKg0M5 z6uLR^veJRrs*aQXWCH&z%j-ff6c`8LbBl9Mj7q)ANNEwp3gK;a*>~**5-}s0fA6@7 z@dA*K4RI-;lDAY{Hawyk9$_}rMTPS7rkb{E!jdnceost~5_(oTpj6?`G{z_dU?~=w zLm$bLbVRxOnC-;93S&f+xP$@`XiyA3nkT_b)O1Xtj)zpy0^y@DVP3kU6Ub=<^5dEQ zkG(f-uIxC{G{56lpslTJE~dy7_Jt}{4<(Wkrzwg}W>-~DQYOQ46S$e!h=pXTruw(f z^SmB$&bc=Ml2v6rHrCTp5}CO7oLD?Oe0kl&164ywZ%si|Akv?F9>gU1Vnm=kx`nPF zEf1*$Kxzg$5fs2tc9GW=Kuom-1x>&Uac0b<#(yP8lZ}8g4BZHHVOD_%Kstm4$xLML zk%rEmw+b2PA7Li)1M!YHrhy6P0V)g63*taoa6Pn^!$^>P^44O6$06YnK3EyC9;5C74vuF7dnJ=89m&`h# z`Pkj~oH0L>=kgcK;IMdMHjhGjq& zaIbao{+<8(Uk&`vkxcbVFOu4R_9Ch6Uz81X_z`i3|KDaqZEgR*QgHm=l@7JFog|@O zeRaIJet7xrcD~-4^WL!hURgxH^7lU;Kin;TTz|Fsg44=8$p8Dl{=9xSKR&y;zI<2h zeR_R*bGEo9FK%yt{rK+g_}v;M!TD`({hzni7uRR=AJ^wUo?fi~>c#cV11VYSw~r71 zvi|z!Ztd*xmX@cVuPwLjP=5Q{n;-xA!<}98p|;l1<3C+$9PjLCHzPZJyt|uUKb$R2 zAJ*sBXKU^E*Ne;f`h5Q5{FJ)90I}B4!s*4X##X_l6c+YG622dB(*H__`ky8pio=iI zxPrYwB|ompCN4!K7{EnvF#J{U35zs?J6pZPbBp9l6DuH^7^4PgiSUgfXDY67+yS+V zEkHsO#fgAY0Kve1IDAVsG-x~kFV+$kn%C4~su4xOL`1VE>V(&Vb1q&%m<41QGZmD^ zxR`D;VFRpBtX|8`iDeUjAmaxF0$bLy0C;=kgBbDUGRzH{k8|wO0KA&Rjx`R{#K9;~X$l>Ej7<$Y5X>T3pOge5sOHS$)DTPp zIwKN60D*JqrYBL_hCu?#YPEYBiBJ|;9jBs6G~yJKr(g6miKdYo02J#XG@KUDN)jzF zA@%YNo&wAgJv#DW9RWBHCroSj9QXi35~m9=$#etrdvFqfu!$R5q8x^2fvI*t;Ry~w zP(J}Q3ZEamk8p=B4Y^f>TsW8ME&xCVgC?}cfm^F|b0p8=X)FK`0yZ&|;}i-Tgs3sU zk_Cbb;kfm=IQD4;4;)EO0~_+iOTfRz^}(yK9yGuaEJr;Rk0Y7n52u^Bgf9UDiE8i> z)3RicM;dYI0Lj5RfRCU7SsJW6SH^=l0P+GGfFi;$xNQVT0Rl{VgSqfOvGG9(@fj_1 zkz3A^((9tn)faVq1@xEs2)R=U97 zp~41%Ighm9qZ#<4k_kEnKH}fFoaQzVo|8V9^T6i(!T}n#9Dz(q%Cg%4)J&*iTCxUd zIrwy}AvsjIV0DK2RNJ=A+ zIWaE6G@uDqIW=NWe4F4h&MjiWle&0MoWNm@LYVq(1hpZvDW0II9J?U;!n#shg$>Ir zaKdoLJPJyFokT$(b z2p&;rTxjHowSl~60aVS}_%ax750NrTQ%f{9DuZO#Q6-2NH>Z)vIq>8z%ub>U0cYR` zqC^CmK|5#(blS23q?3S9m}N-13V{oJcmyk85V=I8x#8CW-2hr+D*#m@<*g)QQWqE~ z`2^*Rc+2mjXEs>DCkxOp++9&jldH4bOZ?;K0&5NE(3g|K*td&*7N;JN@%#Pr1CD|I;U0yeISX#r3+U zJvpVdx$c=*k}vkx(P5kG1jh2;?CSXX@%Zw^+4qZ$wg2P(;rQ-h{a2qIbHruu)$aEC z-sY=+UZ=s=2PgYyCTn$5|Lb}FnJb6a|HYHYFE})}{wpsY-dw+XO(Q3(jhvhupCKz> zKikOe(F@UOTlwa{?N=+O>)@xawh!0$UU8M|f&bd#pNH#*JFotE&9NOH$q`nj>~ zu1}i%7PInyN*KK%nhPNZ72*6h>KQo*uss8wRr`78; zvv%G0K$-(?v_jzTVdF*ek6BFcKC$Dzeezmi=?jIgJ{O5n!+J*9c ztv~&zK7os(gL>=L0gwDLVr;J;9laozY}YYDjqMubVjcVS)$VTke{}$HU*Su#{5z0< zPxtbNY?eL7vMz7z@kss6ka@74YR_g8%xw470j?vp5B+<~S#DM1d}k+{uWHKEpRitf zh6nd38Bk07+vEHB)-znZyV>Wo{;QXZtNHzhzrDFUTg#4k1Cb6SCkuRd^oxl^P41%? zq^`IxMk2ipk@j}j#?0f8wPQy!NqFX9eGfx2>zU8k@e+lV)Jxm1 z4tAZp^_|(cLGR^c)+IaRye{%J$`h3#=+sb9c#R+M zkk9$>6MuY+CL9iX;XoeJkDl8TwkvvDnO@R{~Zn(bqC2f!hR=0N&+j^Flq6Gbg8R{IgBS*e;EnN zb5Aj)HL3tVM_9Tezl|Lh+UKEOuwYf`}& z2eUChl_NX;)<$pAW6HhJ7Cs;|AnZ$VRQCP>uMy*L58tNQ{w*hu$Y&=yLuk|I*X-SY-~vDj zubRbYraADIjI91PKIeV1&ZmFlV$|pe6|!nD34ViB#4x@w-`kvdg!RjmJ!Ma_d^Oxh zMwG`16E0-V`~wd+^UY%tmid{6jd$g`kwc0<$|C&k?~gXuKUx2D?QcH4K3idQnEEC< zlMS@%c4PA)XKUM)?99)-=qZG`#t&64_jY-V6QNr>uMp~f$-clM;^)5GlRrD{?=;JE zBt;x-{SuRbA&de96O4q>=&f6Nqp^%9TC;{*cEMCkpiq<{o5-+1mgIv76V-u#un61n|j4BmPKOpe4&V;FBFKnDv#xFzUDa}jdF=R;|I7EGv3BHKv6}><=x$8x&D-{ z#!nySTbk$dhvUn|X-UoRVj>=3lhITx&7To)4E$jG1!+oU1|QH2t9%FXP+~AG_P&# z{vtCt$17rWzR#@vlnvv0U<9+z_WXGNkl!hfOmk{R-e%sA zj3>gJuq-Cz)-ReW#{0l!G2ohi^a#aRooL%^t?k-l*&}QRr&1l$ci%hpVU4N2b${iL z&ybkQzw2R^#*MDM>7t;OTtXM4d2nE(EXc`aYUocd01 z0p-@$`L{d`Qydl#^I3SDtn7KYm@WT9aj}Lo#DpsDKSq+J zk)9EI{FLqTty_~Ag3t0`?b!BHd8vqxIiGhiy0Rup|FtgcU8AfHJ?2?fiT)xgnSOo= zn`Dt1bJ4AqK0kINjDNvmye~8LZ)z=mWZt{40Jz~ij2$-;oAd|?1*GZ+ewDv*w~&bJ zA~#g~{8c^Jp3k#KEM#0?++pm5diE8tAQSU@PSuCd4;v*6=ZF+et~`Hl?Z-iRCEG=PONlg5 zo~Cc9o)Kt2KSnrsaF*ZO1%HjMV(z~fuxV|Ts%n1s$C9r3+wiTkiSrF7f)4ERE1o(t zAnP0wu(W$n9wa6U!Dri|3HWQgl=o01AQq)Ne~SEDtlsX<3l70_pE(;`#O5)zJi^W-{h0c^^+THAi&#o6CT%(*FXDs_UVSF2vEL92kY?v#jQ-yLL-Fm*FXDl_B)(0V(0f8Fg6h| z$M(Z)YvX8Z?PT_k7sn492#u53;+}3k-Jt9MPoWZd>U#)5lHlYm=bC6> zA7iN7G)5E95ej;PYJZ!AyY`HIO2zGMUj484EYJoygLb^T!Fd3dWHWs}&ak^SFse=% z_tov<@>oXLptk?X>;)ryv2ou1^WU#0zx(-9pDlg9HhJM^AD%qB)Cixi{nRM@&fPT3 zjoe15^a48VGmk%#wNVUFguoJWa)@*P`l*(OAy4PAI<}u`VD;Hy9;;vcTqDennY6Vd zsFV5a)WWshz?M3h-OnE$NGM@eH?LW)hYNZ7_W17jYW^_4yWe=T<`-xA;$r=?huPN~ z->rZ4IQ#I+FE{41v*U;3jm7N#;qLKN>97GHS)>X(^g;H&2lD(38GU|f0vsR71a{j5 z=2tg&?=}#dc6K&t?a7|{Vm#HoFFf`6Q=erqp8Dcn)5y7_Del|P&O#k-W^#KqxsT7z z7VfFz%MEnx$qY$*$p`cMjYGHfdwlp@=g)oa1lCqJ=5r4}_j#z$>FPuMzonTG>Cepe zYi6I34aB};5+o#D9V7An4Lhs%)jNM`J)ZjPxtTn>sLwUhXGkV6RpT-9l^px(9T57H z*(Yv^XcM*;d+A%EluY}4hMIxXzI&X*xfkxc@k?rY*0yoI{|!>}-$RqyXZssW2M6Nl zs7(hIo9X=N@qrYwjmU~Nm3dpfjW_fQ5ghV!#Cq!Ur#{QBc=C&98h`HDwOXEonh(ce z+kAG~d>Riz=N}di7fs?_OeNlh({qO?3(tRE2hyTxs+2MV1Y@x)QdLr$&v;^S9JjBen-#9o} zoz72<=)z(DQc?HZl>Gb`%gg5)sa;kR^vF9ayAyMR>HL0vJ-=I=)*idOIRzJcc=zG` z{Q4fTzi~bL9wT+GBEC4jzff1tXZNQU^Q-wrsP>*hcy0{&e03ou-%~F<^;u(4Hhh~1 z=5Qv`?257pKAnw|75nc8JWUWvzu$Kp4^#zsff76_}3~Ttbd}C)3iEUU>7Y;Siv|; z_Z!6TCe!PT+y8#5^QS)Z`6s1{Up)1~Q=j=P=ViuOaT2DM60R(1^9Q)Cw(4sb@c3$R z`ECQ>x@M4x^pQW;@pGRK16pOq;R{ChbB#}*U9yt_&tX7;hN7@F&~G|YCfI$W}i8wdu;Wx ztWI&t9!pwcm}9!nBQ_xC>&NR;r7d>klE!*(MtxP@`=YJNQ=dQeSyrV+4^O!oq>yFw z&C>n+>h=;bcHM>x7|5oXKO8SE5%hY$OQP+mE`ExbUG6V2$G^3a*eCWjuE=(+2#2m} zPJKmIEQUDL@sr+6{G{{QDJv1TN%uKaTAJ;9GwZAJ-WP39&wai+mKUI|9^a4cCs#>I&16FO}{0_ido!Xd|WO5+AIr#8Y>g~MHKxX zv@GSgQ?{q1a&w^m`15|RBinQzL%rGs&^~`A<%H_2Fmf=>k0WaM7(zlWK7=!fuD(KHVUh12|iY zq|wVbTb_~_gPHj`xanPtlhb+891;5ObfFpZzsk6JihTL(sg39tHS+^Gh7%mcG3Y}* zq|feH$2kbWwU{q_@bUG-;&gF){4hUbC;XSk`F$yH>mTy`_9lQXT9YH-VZBFCv&53_I6x0qZ zJ$&za6Sp{-eLK_2zMXygp2gd@Dgys-3`*~iq}Ds8?E{G`u18H5ub+{znfK`{3Z1bnahtZwB z`(JFloqhbeOgzYNdpD>3(>Wp%XYlP>jMvk`VgoFFK$i#|>Jvg`Vn?l2s*({*8d@!} zQ%e$@S7-{D@dKTAE?i}95ASXlxT6~W@&$9jI(|I4Tx8OV8TxZ>XAMf3AD>=`yyx4C zDLqc+4)yhnk;4p7`l8-BV+Af|r}I0EK2~upggZVF@O-!NVQpnB6GIo2nUyA^P@ozq zaZLtVU_qpUC<1HG1jqzo0w<)8o^71YFtG+3zd6|S`_0tFUc<+JJ=5uNr~oK(a+qoG zW(%@7p~W}S;&k?Ed*i#=pI^Z4aBTJiL6jR_7wsEaonvCz3y;wWBZ_`I`>+VPHldXg zZo)qI4E$^^nsY6KV(Rn3>zm8V8)g3e3tTc-@Am^vK;S~(Q{ES1yeL@z_J(t)_Zy4# z&u(U4Rix}&0%RNHhMbTF2&fNp5Fy{ynAJZ&;{#J4hLMY2&T00X{c^T=Wb??oUo)b= z-*~UEpw0`EVw7j{&znb@-@2P=ABU^)}=Zsq#l}CMJd$;vD1X4bzUVgZk z6Fy<^)lE&}Z0(tD2yv2sHR*>^j@40(o;?uq6e*{2Qc&Xf8rgF(JDETHFrOozsx$cV zF(CE4XcIu=MM3mzKla}`XRk(S!lA_pz1)5rOa%M6xr|wYNP%6%0g>JBrcySMw=!K6 zrjWjco)|S2V`F~T?uUy7tAKoQYbuENj1`yNN*9ERvtJYPm5;B`T;ij{;$&vulFunS z@Xkg9zK;%Xtk-yfzu+I- ziu>2^WL`Jq^16~AI}UP%gE*f?ju`n>O=^=-&OS^x#SO4fwdmy27rvR>=7t|9QfiR0 zZ%WF#Z(lFV*0(dX;pyyh4s@VckaVb zy)L}TZl$WAIz+vT++tj-dV%-dJDEd>%b9gnxDWr^4Qw$@1qwA8#V$L6Q)o-!>-y4> zgax&WPO%6Uk^(V&c@J6EW{>KD3u%O0Y#b`hfPI+hZ4K>ehJnRX13N4^pf_O}16n{Okf5xQ0dtBT_wfG8VNmQlAsJ~;l zD8d;WiYHRyD}=Y}^;i@pf*sWdhhE;?*F2nU$*V}i7kL961g*)|Dmp0!7$$iS{&n(> zAE(p>NK6N8) z*ZLdxvj;w)YC(-#`hBnwiBu9eeE;FvxtMA-j+&o^&QK8Q~1${sPp5*bHnN%+=#wN--)5Hx-us~_ckH>ee zD9Umm^M`7A+#T(*moP}cv*E$yJ*bg_ix77(0hwEB^W3;u^ z*mq2K!t=_v*Ef&XXT->e4spg?>#hj|j%asbspC*zc7cEet|lr&5Hun4e>TYSb`oRh z2Him_Oz*1C8q109q znI2}RSg*E{omlS<@%bjj{ThuoTfBB4G0_<8y-tn_t@^K>poYh^jzNhXg~V`c4nx5h za*i2AGo3SqbRL)BJ13`_ZQhg}#DqSE47FHIxvCgB76f2`>_Db6DUEe?ErP&qWs%Mw ziNcMZes{WBrmf-w&>b?*!?(;n3$>#O}gkmyTZGByuP+;-uUSOSJ#|mLy0i6YD+=qXM-FkZn+cRdcg6`vtp1pH2xO22b)C zs29RPA&1@If^gz$ZQ3vO*EnpPK@B}?(L~nYmv0<>tjTdc`<6>aIT6iYyBbs(N&I8x zjw%1*{6$aX)0cz0BJ6ALbDHa9Xmk|E6a z=-L{sL)f0qP!Jan()jV=9z;r_H&%+T8n>vq+nAzCV}vtJAQ;m^36+vn5oPevi}^U2 zum*!;0jXrd0y|le{T8KH(ANjOx}yT3LP5M?L*ERk!tJUj;;EWQS(=0_mqP zf6Z8x{#ivdwCp4zLn$)`5?VMaxk|+XJz15D7 zaztJCEtEI{%9!P(q7KuUBsnIkAgk1dV=FaY?Ks5;8;}99id*fz$AZDe7LH+epCJYY zqv_^u4)aFmH?hJ_ z`weBY%i0X{!HsrPIjOYVO-Z&Kj81U5QqaP-bkO-t`x6(D-_LOf@3VeqxFZ@DwH-QH zu<6;XB+RyG6Fo$~T(P<=@x|@ zJM?B?4G5-21@cZJ=vBpXr6>UkF=VQ$Lgh}GQ{srPwI5F(?=>@Ol4z|ilNcUM(6=Ua z0o|>9!akRLicYrZE+-F1EQy*+9QBL4lwI{vd;bkg6a%^hA52WmLi;R6xVkxeL^5cp zXWAnP`gY)kDOfm-p!SALlQ*Hre1wAG;Sb>Lwr%egf0e0AFtOfIxPvT(R2j|$FK(FJ zXS!8HB@!XPd@(P&)9v+aj|MTE4JjB3pe`_E{j99WkD1edVb}GrnVDQq$Sw{Rt;jCS zjh~E}<+Q~RGlTS=k(va_gb$X2WE%;o)d2&-Wz=jdS;r3HT-t~?w!mDOLI9z$m4>bI z26=*t{lRtYKo?6-NrGYkN-S|?xngxj);QWisA=d-#XD?#uwE0?K3kwfK&7?C=nqr2 zb#c-AYFl4nY(_Us0Y{f;DBfc}6w-z2VTf$_O7?VUqu^=+=hp+D!MCb@;a)e9?|`qs z`6@f1yW=r89ub=S(I`N_E6F6Lz%RmC;Kw3YxKFW)m0mFN;a=ZhdFvD6Ur^=y85Oo%XkyVfzrSjmBlk^^*| zZUEdV!v`5B;XCR{6VK~GL6^=TWCC?Az}b7tbN#?cE~9FT2R4rDxw>&QW}r4K0KlY? z232q3@=Yln`1JLozRPeJp~-?K4x3z1?!;n9a(|}Mgi$m9)td>->EI(#=$YvjxrxnG zu(p(<*Q>HSKStPA8{>kDn#pj;>2s3#T-_-iHxJxWQAUNDAtvPE`e&E36Ml~geFoNg z`rsjwSn_u)%Ouw}o2iW)F-oB|s_jr7rQ5xf9U%6>07vs=*hOfsyBlz9rpkpq%wX)Q z1vL4Fi++9eZ4Oh}@bYOY;tB?iV*8BluM61sT-OGlZg4j&VHgwZ%5K^>PBXkMmz;IN z!iVBg&x0*2gpjc+tLSxg{1yqIPK@cUb4>1m9oS(D)^opGdelXVR{7sHf=+@mYuJSv zZ{84zu*rAHH$NUkDo#FSA

eT z(TX8s5Zog6us6~=GI4kPdrzGI5J7&+zGQ9l-9OaFnLwBr5?7Psv7EJn+qwuU`=>yP z@}J@kA#R#x1M>unco&2cy3p)pgYItuy$CDt^?K`AsNtCVdbhSeS7`C>r(C zybaAE>A`Zg(Bad7d*PzbG_n%U2H*jTW%>ohyA<#HU!h+ERmk>$WEiD=b-gW8%(H17 zcZVBmgLC%@YRAD+!Fr8p)p`K+p;JYvFW|A_m0m zoK6DuiAsOGy}ew_&$KiE9ubRVy$xa!6PC-sE2Xjk0&$q~S+XIL7x;K7bl#V~tPOzR zmaXzYkT5ZgqYk~dUpK$q+}%^)fKU`)Fb2KC2>=htRka9DCBHewYVkjoFCJ>K{S!uA zv3Uo!DO|mFG%WPSV2iD}QP|SU5ER^-jY=(G`+S^zd=L9f_B~df8^60j7v)$hLS8pN zIxW5#w;pr4p3gy$bmH#jQKG}5AeT{hbC!Zl2DlAGBnQ9UxbN|xF9I7o6nQ?N{#6-Q zgfN1PvY=cj;&2~k`AR5$2VCCtPAW9B)SR2a)&W)YSI(s?%lY+3U-_c+n$l}A`$OEE zAwW^saB}Nwie0Zvbs1?m*=Yl6)V43!9djP18V&ULK&6vlcQ^pSAbVQF7>^`%hFrqf zwGAVrJP(DrrgEot${?cPcANptu1F#y8Iz82u!Z*4W~abHL7%}*6?569F?+f&rQ4cm z$2OsP(>$yn$O4hJ0BKKBYH)ZN>MnF$O%kPo$cn=Y7Pku45#^m%E*B#Sb~%{}MmbZ# ziTAbwhSrA(BL&al%hQ@I`b@7XH@N;__&>%V1g^M2azb4g5YB|N5DG#0GM$23pK@&3p`&9fBfuVGzDYJ~*f&=4 zBAJpc&N%>qeP+uZ*ho}NKaz$63yqfFE!w_5?l#pXHdWg@U%4Rn2)WXS?)b8+r|)x^ zkFqGVi>2ec`V8&Gq~;0$x(nQwu{A`iy9JEAneJUi!*z$$U>jOn9!X4IlMM?s8@ISy z&5eR*>B#{&2IFCzI`or&%9L1jKGBgFxNAwU4;>Av^_zJ-Y{R+TunM&SJPjRE8PV;3dRiasv zw1UFK&K|Qz<%6R-k~ys7Us?-r_gBaKxZ}s#jI~d-g|AV>5Li_*Is!?_{O2X_fEV4g z|2OG_`bq!7ay;cL@YqB170{o}_K-zlt~Qwt7l$HciPKUOE#?k3&Ue94vwgv*RGtDi zF~TOmP1CZ|^%YQ!%5cH9Tus%Kq>@!bEKCNS@i z{tWsoDSkxk*Py{A9&|`SW@fTPv=!aJBQ0Y7vAk_A1@noQlJxbC6(EXoe1=?MhLgDs zGhw$+V~*XZG6K{mQoM7oi;3!Zyfd`dVp8G61pKNB0%n1}$qyW3r(%yw5X^PhyPZ>+ zuo4pDa`6SjRExy+0L?VNv0#KA%S#gcT64Z2`nQo#mXsGOq&CdW>@Ql`GPGf0OO2Lo`VGcp=6m1V(1mVj zMy}{Bt2LegD>wK-=(*xGgK%K8)tj%U-(jtZhGxA*R<3!H%9h7UVz*#;Uf-ShX1@ zY7lyIGpOt~@I9`C7EC^We~~*3sEFxMg~GnYNm)YyF@VivHw=ybr5Q2s`SBG$njVxQ zqXKLqa{Pc8-0(ygVs*2a;vES1v1aFpYK;+0qe|iC<9c$cQYc&sc_hzuBE3!^7W)MJeJIk5C$u6g$G$6{Td5dPd)BQY;aY> zLtK>e!TDEkmeD=GB6ahO9xItT)*_Q`WmhGER=y^o@ zeVrFK4Y#9u9V0L9&K5e|M1s-!GRY#^QER=3TUe^43plqH|@i zu_o>!jc?H>8As0a-&cC!m_Z<4h;r+67LvI+^JZ?!5l^l^sM-0 z9Cagj)w@`A^Mr$%ySqyzVB_vdQ6R5Be3*fA$Z^<=tRS~>U*1a|VVvg1WxzdBFi5uFCEb@#h* z1E(S3k;&U(2Vx{>g#Pj1e()q0G%xQhX^K2fq(zC}><2ir;jpY$`eL~>pWDn>?4C;j z^;!+gwP_7Zig2$$OVrO@f|dkOt#K6qwZh@3aV}V6A3uuyqSR1%{AQ?*KxsQ(!yYr4SFNPL8iQIAHE|CRO3~j|#?;QJ~KWapCiG?CdtfF*Q`8AgH=y zIrVCwz{qE-Kmr5?DrqMta#nj_B|5oUoG0zGo%x9_gMZpUqi;!7C_>PCoq|i=IZ1%+ zsEJ!AhH34E<1Q8{$Z6e#H?!M|cjP1E=g~rIYluJZDOly^C15P66Q@FvH%VwLr3BrY zTa70@a(zq8<@Q9d+c6frNg_pBcja829qWY*Mz!Z=vpI%EG8};DaTWmhx73Y=uqPh% z;BUSnEh=?b+Z0;rS)K4=(Mhf}j+MX!&$Ylr{LML>Ly7Zk?o1h?23$Nq<>t~5H9ixn zzz~zhousnlRo*txQtXc*#XJ)HB-&J|U1;M1#lv#1bv}P^z#e^xjce947qCI*tK;+Q z#lzznV^zO!;d{}(iYYM%XYA2B_`rE#l(lG6bMaX}4-&bQOFZ(rudC0+v?6G?501{= zfIS<43x8UdeL)D?(aAztNryu93(i;$85HdZA-pQ71}--N~N5*G)(JSj>ec6u9L^ync_gU_s)`BR}0O>C9-^geoBQE^g{VR^AuW`2J!^Z z!=v?5rAompm7ehky#$&{CWRVSYFLtv(Yxa@Bsbo-Fr({UDhK_turQQ~#;wu-+$qz- z{yi1c)dybT`avl5IotOwZ5W(MK)h|7=G`Q{K$|AxlxwHvVZ0Q9P^dn!9Bb-Hhy(K! zlT!JO9$wJU70ATx%}E0}Pt#Z+70cjHU`Ilzzz4dI;#{&YY?QR~C2Cg1Jh8(=y_b z+ZEv$OXUe6ZmWIhv3`Tnu zH-B_$5p=zw?ZU`5B7N|Sz zbZZw_D=jd{p;If#@8R#DJNS6HQ^wvYy2@2RM2!h*f!M0zP#8!^w#ux<@e-Pu)I-?E z60xVUTCshGG&nJd6hUaq-Jx!1$6P3=B7xqhPn*|*#aWCITW}a=@mmvEv3AfcOjfHW zTFK%BM&d7LfBMhA$(g!GqV;4}Rcaa4EyyL?6gF*HZwHi1sa8p!1Qx! zEW_981Vscq-Qj-n389nK7Z1J&onu&NtY&#uaK{7Ad%DUiiMmW=*BS^@j`ya~d^%-# z4tkON=Cf0N7x0YO4p$YC+C;~5;9wfgA9I+>6$I7qagy|nB#23}_N|!fnRMm6N1FWU zKbJVE;DiN`(pdIwDhh#{E86_lH_cWwS^UOK?&fsN>Cq1V@r@xZgyDUEzvF*eFk&hI$< z6WdX(+C|1>FP1IB1>)3&t!yCbn#Gy@O!ze$^!3ISGCXO~I7}^2lBkH4-NY6?G2f?x zUlaG5`AhRLJy20G1OiR_GQi}VetSb)pi)*Q=Bd2iQdKI9S@L=brfTOdpo5JYe=;aH zY6WhnCNqqFV5uR^l4uWg7^WXbD+FtH#0lnC;1G4lh3t8@=Rigy=*Dx9Z_@IpSO|92 zkuTxs1 z60({+{LCn}e`*wHhZ?WVZvX2>!7&v?L<%oDcAfXcd47IKUos@vbn@UTVH#b&{P_{B zO$qa5++yyTY=G3KRuPM@1;R^$ek9D;Es|!-0>ydZA228|FsRGK;Trg{eF)-$$@SV# zcaULz+BvWES&UUKMzYh!)r*RgphuBj{Fk$ih?$ZIA0~}q3G3RY`PDS4gZ0A|tm|MM z(aEhaBwFOmW9@lREWxfnm6k7KI}tur58iX7{9*$3Y`>St#NbHHFlZ4{(*H3ywvC0C zM&C!>4KI~GXGUzIL=xB`Dj2&?gVvgigSue4_asZ|+E$Qc2vG;k#3x(LPvJ=gMeQnl zo0uhvLeRR_yc?s97gc2@por?6I9iH{TuibUp54291uk7${Sap0Ob7{dt?a{sWwOjm ziU0ZRM<)8^mw4>2ou0>hn2Q47fNOu=xmkZ&VvQMjdhcCxz;b_=O^?5U!-`&MF>LQm) zcy}WuH(}qJesK+D*3}fzD%l`G4YbX~WkzgHl7xwz4&M<`yr`h_BFUvqA>dUve4B?& zF5PObR>dTH(3#fKPqU4dIx0Ff-)(9GLbmtA9V`#b2n8~z&KfNs;AexSTZbB`4$s9y z4vKW3Wch2@xBR59DkF_i?{oenDp0)>+q&B4L1Hy$ALcwQnE?ZVs-Fg&b5As75^ZCu z-u#4h=Ho$dRDUQlZTCnR5{UZ>w5SUGaW z)m{E_l)6S=f4jZ4iAKaHBEV#4*wlwdM@I+waEo=$4PuP??;ii&T{RqR<(*hw4x8Fi zu|nS3(ylo$R%YejMvuWyA@i&M$lKo7VPOmdm!vjgSw8}EH8^Jx?+fYQyZEy$?57;a zj`f8pJtdm_h5d0AF#p^;O5Nd2&D_~P@(4#Dw1#$HP?)g(=ZD*=cJPw-c6U-+fC)$xbO92qIgG`&`ip^s1jsU zBPDiMW`!v}H)jIDX%9)*YHRjXo7Y3a0LvD>Wyee=5V$h3$}^Lvq;i4MNtBj-CeGcO z4vYyxN}#(P&xs~R&f(n{XTyl$l4FKTKT+;oZUOb(-;&Q{+dEoa1O1|42mP&r9loM* z_tDoMlttS23WIN_vrHBKfWC)q;mL}(=FskKS?gwYWz^#fF`a)TN4I7ALdYjHOalav zCm`=sYFS#<0i`t`Yntq3yulL%^ia;S1MUM&0rsV#8d)R*xYr3jc*5Y~zS0$gi%Sxh z46aM3n50#v$5Ds~tUz|o%u^G1csc)>*l|BABAX@qtFPSH|NiXWCcq;(2hBbpSFA%22#lcyQ3%a&lULaJEGUK(7o?> z`2;bb?vQ{@U3CJ71&%^w#m*Xs9`6=nfc}FZ0VehPAEZ3PY2If!yzGUFPRd)JbJvA7doWUqFxvV70ulEfSZJDClTDexGYf&pV+Qg7i*R_Za#v|PUCN+yGR&f0K@ zhw2zDXxPtX$q;WQ+(79lA`a5x(z`BUbfksKYNAORr7V6_Ntugt$|qn;NT!Zyvcev+ zH7ZzQofNjALJ#on!;H9`+h$NuJojytK@B}hq!To(wea{(z<}}jx7CUfsiQj9em`3< z;-humDbzFzjp^Z&95u4P52Gbv0)%8$Eh9~;E6HE81DLRAlWW~xqw5T{{GZ|BnsDW6 zOms0&1K2A!ew_})z=ksxoDgXZIzyQNx?7uZXClRN3aPv%`5N-+N+dyO(hV~RrZhdY zHIYqgyX1_+OffB{KAw)YLF-7f^6fD6-1RM9s3rHkEe63SylhP33DBa(Olp zUp0PaEZF1@T+1SSVeoD!(|J{_^lNAnNV)nRM$}?&T!>oo0_kctlRL|1pX^%|X5ZF7 z;2}f{eWzC2YS$hgZpSHsUiz6y8^yvJ&RHT`IeW>PvLz@e=U-O{*9eQ+OQh&)BDj-s z_J5R5)Y!W0_K-x8{YLws$80Q(0*xSko8=5g)(CoC{e&oKRMg8cJ0aUxZ}4 zT1B})CWMtrNo7TffqLL~kLarF2binXJ_)+mS3n~sOtGm%D>=Hu zXGLHWc~X6SBCEw@SR>QaCu?2Xe7(g?@bLi}IPsp{pJEBxSYaeDfY zcwq0ANkSu4Po}4STRL;X`>3hL`fuoB--3%ptD)-!<;6k$61$6Kln~lY!#fivu4C>v zdrV`&ph+jVTP`mR_19`(I*p#NuO&lwq3dP4_q*NUDjrD!bqX%sI`OI)ph_aU^QAY_8ZfNXcf`96WhxgKpAz zX~V~ZP6<3YRZ^~zet_Kl`AYm~Qfy}JjjX|e?Xe8YV2w9ARmAxc@j28teXpu%(MnE% z#>1er@MB%!3%qXSQz2tBAew$;?_4g<$&_M4X0at7JuC^)y#W2jU6%!Euo5OS(8$S?58-^!+k-fGO;_Yv;#$RS;c73 zWPHG@qU4F;zJ;;9E>4yE<#3xJo+u&2T8e+H0!3PrXf#z1=DfExGCH|J~ub?iK#oXBXg#4kvT5bW@Is_@7b_S?Q zZ^IzV6lxTy;Q>A}O@fF{!;Ae$fFRC0yyC@v4KrxDprvoBSYk6k4O%|1X`L@yS zTuZQHagpilca3`wt?SQZ1wrK-Sy_(lU43O6dLuNPrqOx%6}*S<8ABf~iF*C{N5MLs zOG^gTW)A@`Hx9_!n!a^N^6mImE-zeC36Vb*n5pEK?@Q&DWJ{0}9Zi&q+sG3VcxI@Y zqT9g$inOp9gXaycCAG2+6~Y8-YJ1P@1|Q>O=8@%T%mm|u^VvV~fenFu_?|`=H2Ho; zgqR~ctW_I=3$QIBzP+zk+E#OG>#*XPq*L)VU_yUBwSc!N#la^Ph58;igr@Ijzu|sI zPXrwiU#6i5nw0kFhjroYrxqt_{Wt3Wr@jTOVfsgP4k2N^O#rg76KGnR1BEla^PpHz zSOe8B!1M)*wf4tyY^+u4MA15C&Pv2U@En$+k1Re#MT^&$L0Vi&mccGr2AH6s8Y{4G z7&e{Rk|pj+8(-?g$dK38ZO#&At|^9^I0lcSWew_oTxegR-7&$xE4#L(kUid)fWzyE zYr@_ylcSVp1>fR@iDRW1Ew|~nJ?Ka-+mRYJw7O5J2A#nf*%!XRUr zL3Vea(ONygy`HXOl$IHB76|(|^ zQrIjaN52V6DN*b#;bbqE`=q|hE%6MCkac|DTO>y4{kocpQ}l;+-%Ru2Zs;ls5r;+j zL-Q7L_=E#E4OW2(4=x`ClbRH8ky zq>cQ~ZmmW`!fuOGeYn7*r-6#<@&x2?Ql1`WdF>XN8esUPnNiu=3wWpdon8X~gZ8R4 z!WbU2J|z!g5!WXlbPuxZ$?v2b=8?p*l78}e96l?W;VJQ^Lo#ET%#sdP4F&Ckb=3tz z(J2^t2^{RzZ4B(*3rmZ`&&pjk>8w>H1ibZ=XC0mvKq-`9W_5n|y4g52+3uq;3lnJVToGFaT2w zHWVa-A~z%NO?e2(-+0LIOC9YCE5#;==^FOSNdF!=^b1;oFCU6(y{9yhAsY%|_k4c+ z@Hn|z97fdN;m2vtpe-Vm$2ziog@&xllrv8np1JZk9kJ-6wy~#53gS?VQY7maq(`zV zk*)Nlwr$e6+bV{g2*f1@FTZ$~aSDJU!(Z31VrFNp+Jy<_x%UGligydpz*!2|e(ss> z!p0oluk#MIwh$Hr!2N=o8wOtERz_`zC#&eBf?b@frP~r}N(_)ja)%Flqo1Wi{}6@L z@Q1=?tsy+2fPl#{@V7<9WEDY}^W$5d=(lnSnq=0~?c8IYhPqnph=l}#K!pTtsyS0w z$5eQsuvbZ(^!}XF&cJ_`<#W-55IbGwB?;0GrdNvHSi{-4)#btp1O=3|SG?5Aen#d_ z6ZRzo#}`;)KKIL1YbH*BrB_84TqEhgQsUsxnd4duOqs9+ned4xH6Er2cdTRL6=lSn zqXcs2iG`y+zsCKcUtqrOd{|32EGx|-z2BBXn;sQ;-D)qa`83MC>0>FtN_SA1QH{Dg z+xhni%1HX(B%RM8HJ!_4jgso4va-;*e)k{RQ|=aGyxYuK@J|c-Egx7DJ<&OzJ!dMI z9%D%C2+6XF9bu%S3t!Iv=@XR9`pNu!aSfW}pJ^UaDw~mKrSVddN(KNLcI1_g(v(_O zFfsq%z-Ilc3kp?|AR&~hFS)gGz)ek+tn$?Rh@asOr}1{CH?Rs)6I^6(XUz);+qA(+ zdMNCP#tPgmoq$-bT4Rw2v}E1kHYy!m?tV(<6%DH?-pCm;KYG=;YGcM|R#cMe7fyrF z1biZNTMe?VbC-akRPCZz2=qhV>VPB!|3d0=IQDu0zh9Nv5!qWe zpEkGclgHq>5phd5jIvi8pF(2Wii#st<^fqUw*^3xO|5qr5JJxJa7z*W3^xr~q{OKz z8mGrDJtj?SG)Mb}x zIh1sIf+rVHAA`RVnp-x`Hz_R;-B|)Q5gx^PWd}AWaT(g@KwK3M14lr;V-~(cq!^7n zM<0|uHN`|l*laT`@wnR)mE*EiQ6a=%4h2^H=O%w@OEh%tJM3Gr*B;kP%KX+A+aQ_QRzejYeCI zyMhndt|*=OCu-r`L>m8uyY9$EaeC- zLfQIifqR7A&4^s7aW1R~9`jdJjKwDyL$ToPN12P0<7a?ilq7F9IInnw{J z)z~!HTb|{#xOY1d+Lu6V3&~nFB=A#3=&5ka#vCFDu|cpJ3k9YD0e(!;i!JY~+_=K{~I)`!_OuvWo~gzd`= zi>#RB#)Qx+anCaMO=Q{|t1^~*CG~|4ArE+f^g$!3LV7l5w3b`=lI%@FH9$#W%;X*I z9_&*@ta8Ye0t0l8{AB@y;WJ9ZIL>6dF!a)iWwI;ls^|g(ZSEpdXix?`O;~~VpbVQK z5egD7F=mpbDTkB$Rn5wPn%g>Sq>eKIZZmbcK{mVr$s?{o?Sk~lMnrRFW`6g>!Yh5T)>ulGhjBQ4zp{c65yig?M?2m(FhdMygv)f*X$+fy2j3<)xDG?uM z-_)kT>0H?~EWxWi?Q$tjUEAhjF6|5}xI)VmW;_t0-idz1^q{u3 zZsw-WcXLFW_#4vHTOi8!U>YAwyc%C%Q_;QwOkCc z-P+Q8wQDF}U0?WuW@>-IzDe}&FW7mIs#HK-T`deVmn!*NAHc=MaplLsNs|GB8xw@! zgk6pXo#TS#24u>UCNi#XyyHzCeafNWE>Aly?dLoJ7BP9^pTJx}cgK;_hV@0N4b4{k z4u;{V^aTynRt*L;*D_zSyQ0u)0@HEhX~cAo*~1aqh&#u>Wg-OsWkT`oEq+zKNEYBy zsr^0cfxC!{*h2h(0J}terw@`RZ(TS1>r504M9muRsjbLz91ku{@|}8QDXzT_1^c5e ze~UokCMz8ZjeybpO1r`ZW&2dp1EQcU*2U6dAqlhSBnPh0DHu^(@zTG*0?m@308KJJAcNmg8H#@=nEz)nS>~ec`OZY2PCchsGl&TVmf_}98my{7#loK z;>ZqX3M8>0_trrRQ?a?CnT$95?)dSPZZ8Zu_nEIX>#yLkPcVo9nK)O`mh*RfKr&2k zI-rMyA2l3R4hQ^K;10_quJeG|Ax^mcSA{z|i9fbqjfw)??Nhgy6DaAkR-cdHHn$Ak zs)v!<2kcy~GR)n#xU*CSTz$`Ut-R;GBzdtu;XKOmZ0m3B_nxC`xs8TthTqRH=e7?y zH2dB(S#g-F-IzPt&QP1GeA+KRLG3~32@90QN6(dC8nfFY^eTnMbl{HcxOTnv{p|N_ z6NO~!=HE>DV*HwRh5>aGWedSKvv>o%-^=6WDM+4${Q zQd|@_Wk0j7?-8Z1Q%gPViV_2wnrx{A{+We|5rYWK+f*AszruVCgwT?iQ(wL+OQoI< z5=OK(*<9A}_fVxbhB;8fq8eB!O zySgZyD%Ri1I_&v*%=%}l`t%aru68M>=%QhbEurPKe}<0KPkLeu^beD;-Y{k_6uf|rQ#ADm3&HH2RYO5uzHv2Zn5-QlNihbK!vp7 zUa+A3#zt+i_HLTS^lv%+inj-10+d5^D3w5VKRf>Q2{`oOHfQJy3?2nYdW+KbIB+vt zw+|5^==~N-!J6ZA;_$Lz7e;zL`@hNry*lRRY|kb}H6&M^)bDvjeB$1fq*Op85MUr`uhb_vT$5eOW|fP^mc*vOPrpQ4c@-jE-|@ z252hY=3riA88V%+Wf}ir-8uIFFdKWiES2zEdk~prlX*9G{xS`cRZZ_%XTSzy@%Sf-K>ZM38|k>A z(kFwJ=gPGp^>_zrZRiWM2WfJ<7Q8?}6gyMQmqe9TLMTQeG2j84bmy_gUNEC5u1$dn z!hJK%Y;{Xe_J~jFfu$W0D|!Q}lTE2fa6$x%`UuUzVE~HbXuOuP`MpH=euh1=#`sS7 zx?aG>#yyGJWvm1v%{~Wd+Q;1OJ>OtWar$>}a#(LmMfzT75*5C^8k*!Yqy!}r1eb^) z=3;0CXx5p4oVf}ZCBkJFYy;>LO1CWhQF5p};1qy*`FIqIcyGNVK~93!6$DKf*7_&~ zvyu7q4GiuXwTLIR(;(U^zuL|Puhcu?LIlkTaEeExF-%FX0IXD=k0Wq6vSLCDQ{ld7 zSXQ9Fis9o32jx`z_pXh_D6q@Fq3|zoS*aQ=B(kA}&tj_%!4gt=cPQV;V>)x%DWog& zdk-KY?xQT6O>MzrF)={ec_sA(;#wfkqqigqi@-j_($fE)>JL+@TZk^>0r=q|`)%1R z9$%7yCG#$?4-Zr4L@TSJhS5QRXMo$fOiALNC4AdXxnfM=eJ3AMh{!NH>(#j=AJ;W=I6 zr+{sAf!9ZV-fBu}G?B^8K|Z9|?LGh4<@YdoahPZw&!7*iv<)sC)v!MVSwakZ zf*)o%=e&D(7v)1#Ie$9Xj;Q5G$hS;B_alXKP2X1yqt^yL=}Oh&@TA7MU2$zT?ir(HMp{An<0 zha#6u%L!spGrLLpUB=3gYXdLSuB-*FKHFJ4C=8?a3VV47ur-xp8a_g0c#OTvdv$@3 z+}0A7D$-C#GJwEQbTKKWx@UIg^*!R&jCItE{Zecw%dz3I!<(rs!9(fM0%^rWk%ETy_tkiEwbULTX^CN1hCmAH9-Uigr6g45NB#?339w zb4$4k4iaNl(FfT%>lL#=E&|OOzm`Gy5EV85<(i2S2no)FnJ!7Kb0M^m@do$A@Y3h$HX4HJRp!Lf`#?oPC%^=gayc z3f`G?5JzQ9Hp}$~tv?oNG)4`3;P-U)ho4lyZj)bRgc=Wmz(KD{Z*8H`z$OMn$Y4=_ zq7ec88X~CXRKE3ip#;?47#=S`FrZaea}zBWopt(8m_Makn$AfuOPwPK&~;97DiM;6 zJngDWM85Gzc+)0*%1oc7ViORXX=vAk-oPhkILV;7+ldn`b;4x*fzT)Gp4OKrvhC zPT@p!cCXIgmBr1Vu{K*{lhp97U72+G(Y;OIu71Z3*;RX3!(0p!$jHsdktjbHp4>=G zrFM~y?AD>%Wx4UA81=g@)nkfE( zC#;>cZ$TK%zfGRN_W;ogd|vpWA2CPr4KR>{Ozv=!gTqV*8sS2u$anh5+z|KM_>la@ zz4T4qV~8So#;MP6(cKoUmlFL{=8-;BCZMhKk`L00U96JtTTR;7TV1vs9_%vA8hLB; z5VJ8ON7|&#d{AvNc|&84%Yga}f4j$pyev4BjE>YXv z=atW^8Vgg)#OWdBzU{6S#Yn#+mnK7SEFu4fGpQ)Uo9h-*t+=<@vl2fz?{DmRYW428 zUU+0#NBIwo(hmThnNycUEjXndf(6FMfib;MyptdebA}Yr)>>2|?|AEae?(0I4;z;Mt$QG#BWs>P{P+ZFDMME=3~Y8_Z2N$WEvrX3*1 ztc|ISE3l&7T1vSeq++?u-1=Z+4IdF@Oy0nmikKu8j32x~1WFxnk0*gKct&om8pL`zOw=B?75(_?@c#s_@#WV(&{19f)8l%jrT##hdAxOFUUbIU2}#*0WK zz*HfT%y++gT*DEoK#7$C!^6w@>ZBKmZuM&kn&pP^J9Zk-?uiU7mlRzh-u0OZxGqJ7 z3~>xRu2np6^+-|}4yOLWkcDl)IXNu8$*2!tBQR;+K4k^)e(vTvJiOAJ1eE*gF*i*z z6WN9;d_B}=KA*_@XusoQ3UOn#5f%dtP)MST+m;HG62{u^(brI{64rscHC^up@P?La zEGAMOlZeLX56yBkQRb}h3~s(_2{JAFT*i(HYe~n@&7!~%tbH|j-AylBmnVf}i zVvP*UwP)xR&Vf9LE;kpt?1+EWkrzfJu;KC9(%&lN9S(m}ps&!y2qdatx9Qei3Q`)M zSVd}a4s}?jI(#Y|l?=lMT*W-8GzUa|%5}gTNeUh`JQaS)f~eZA_av7BV7;Dg|DHmn z03AoZX8-K~*hZ?dEMayN(Ev^*sw7=t3)w%@EoAXR>&Pk zIX2ojIs)}vO-4FE5v8RVa58L_{egEj6P+4v{U}AYr7&zefqzMWUCw-Cbu^3PZTnp{ zp}}Dt-2;*}Dt9zGxUwSp3tv#^+6L-@NC|PYud}c9+Jn_~k2-3gg~p0*(H4y`O8CIh z;VEcnE&{uYqER=bg`s77a3OFvZ6C5sy%M_Q(P9`z+pt(i64!3HZA>|cX1V5@O3M=s zBPJ0(n+gjSC9Gb`J5UrKg=*Zpv_tZy-l<%(>+j^g4OBx)*kQZ3nWqZFyowb*^XfDabe8@b%q!^kJl+pgbBb5(T;J|NS_4xCX3PwO0x0j zdAG6;JO~rv-E+cQ8O)cV*l7FI5)h7QyR8Lq%;kiDntQ{hO=zo!aVEAP1{F1VhLv%= z%cHJ!p}E3CX|ql~Y-gPUlFOyIX|h=8+`oHea2rF& zHRz=RSrPfhWMJ2-L!;Wu)YUgAs%a>1#Gi1)srQ+$-H=u46q@NmsCi$C-t##x!sn`e zec^hpMsGYu?)p=!#n1vLXUw2@GqbQgt<%bWs+f6!#g4y81jsZ!(bbeIH!hQqGK_!( zvm-=cA3`-*#{S@;U@X_+5(n0eHSN+|i4dmOt1y(R%9#wLJQ@pzfdvBYBWs(|ZO6)X zH@#*|QY=8`hHuA}8%>?)%m#!7D`r8G%}7GLJHT}aD%h`6u4O2cz7hWb=wj@WxS!SJ z`;cTFD}+5o{$_B-h}hD4xtjgQVlA{dn!^8bZ2jI(YmHiX#Wk{0n#X~sq%lgAuoX;~ z6SbH{&-vxb#o^Jg%Jai5 zv?6j$S0w6Zaxe^LZ(|{5GZ~2K0dPRQFg;R~Mc&?zUC|eL+q>AxumWSJsBgCr+V=eU zBU79~q$1F6YsjszEAq3{l50F^$^G$bV4p^-mbgq&=(=TU5avnG58U_Min83H4;=Ww z1swtZFwJ?~E}v!q64;ewK|s_h<}Wwn1_R-6Yk!SeP{FrM4<)}ty7T~h!r6EOax9%CPIM*Z)2FhUzmI5YMV>g$n zHf6<@YYzIw;#{N3OcIUiN9LDInW&$)Rp=)7C2}mj!zCVzAH&ksgS7z$#BJimBoT$^ zq-8J?Pc;Ls#ZwD6^3x3KE+6;$0AO4s2Wcub8(C&U_my_QP4Z&OG5;y}RI8oS&Y@vV zMGa~gF)gR8B{=o4YSkt~Z-z~cM@y#U1jIJ(QJ!K3Lixl75k|dY;m&7C2SNZMEl5rR z1LUv;g3T@SB+{l0BtJK;-#n{k)iPqbcgSL8WaarJTqNXzVb`*#Z~`nzU%e@#|F*ng z*-h>m9yA?<2wZW~$x(Xr%{*<$j@FS}IL0PbW$3(9j;_#l zc{lnqhz6Gq%dUFv#v;r|h4QL}KIj^8)VkZsLbKGeX03+Qo!OS@K=0-4fa&k7=It2% zS}G}EdL(sgtNik3Zt#0$L=K9U?yhmQOtlyw)9f9<&8i)oGmX+N&c<-05gv(i)noRM zKs2lsiR7xyo2oTi)ghq<0;$Ys%6jl#N_5+3xEOFH(lU(V#Z z-QO!<5cidyIfKuozJVx$m8f;oRoUBb2Ooi~O)e}k%d%{h@*R8!6SyBJZl?0bT6WjO$IJnD()50& zuYAAPeUCSUO4!4#uNXKS5yxHv=dFCicr|0<1+cL3&K9nvdy(zt>eUJnsMW)(T3}i` z!*Sva|1-XXW5R{HcvD$wrdFJc7+Fn*T4^ADS}M=vn~7iKvQr(jA)ztS8jnVq7h5XF zat<806>-&}v?Q}hu|w|HY53Lq3!blwOYb?NvrE4Hf$hY?f8TbMmmRXoI$r2xu&;$SYtJ>NPWEM9)1c_opvH%NIv4bNd|0lQ;#! zih87&Y3V`=m5?c^)s&DcKd?xh;zi&Y075?qnp*$e1EBTR*kM?RW%m-L!}2`Y*;&bK zbkiNMh4TG?j=w>KZf)WuM_5@0BrPM~1rR)bOZ_>eoR$13^BEk_=DgK@d{Joc)B_N{ z-HNOu*(OHzNKyMF5B>iUvO28ZkL|D$EfRkgohf>+kY4$iuYs`c}~QPA}B!q{*(gLlpQF5nbyA! zU|=p}-z#QHLuK;ae0+(snhL4!NeNJe>`Kj{tM~S4mD0J`@NU*>GweBXYiqN<_c*$X z^cEz7<2`=2O|Swvhd*I>l4R;UDTBzfW%DSr z9>F8=(Z;8b`M{+44AJ%)SMp#|H}*CI?3iPpG@}0C)KrPnH$cY*qWd}R6mE%kX88K= z`0u0#naCQ>dbI6m@v{QArAyqs6a}GX`H6Lggf%q!5O>aKe6=GWUaXKDv=;Z{2}Xi! zhmj{&D=_!m92G^=i~|@Yg%*Qe$=v3sWN#r)h~tcJlrrQ*w6;Zy=G1s~MMs(1pw~mp zM>6jQY3hyD^>S7au7z{(*yeY`U0WQRlW8%62&iy;&I){JSnhoG1sKF=x!TA#Vsl2 zD_;p`HAW-Ll5vssCChm7hU?UDEc#f(hESWYI&Pjw5?GkS zMNTRP`Q6N7uvyuLBw8k5uhAN3yxq|E9xI?MFuF9{mhx-FmWP?eqnHM)1B1oqX9I%L zpAvm@D>S1fK1%ZbbGM+_J3Ux+9w=e6Qgu7~9Kk+Rp$U)`c{+m0n?WJ8$lf7CV(jy8OW;1mJ_i&de6C-_e^% z(elYw!6-K!v{cnfR)Y99AM$J)}ZW~nAG#f zg34M%Op<_FuXN=^s`Ge!+2Y78U9p85lOC!drF4UZyRsqvgx=z;@HU6rhehTHlMWsU z6U|Jn1P!)hvMsR3dX`2##o&lv~?OEYlkx&PXis5;}pU)`YleNOs_R>oUCS zH3_@gbN#=o^ihasZXm$jpymA*fO-=X9(=Ja4j!{NkJ?`Cli!rvH-Q^`FK`2F-T;yz zZSDQX;((L=909}S-xx_?;s+FL;v>vKla&3Sx=ecY7aKP+A_f&Sp$txnQZ-k+&>FB^ z6Ckk%L^207#*uNZwaC=YLpfJR18#dd*q6vhq-i+1u)V3M% zz2oSmO^Mig9!=FHjO7V1u}aP2)$eeKwYMi^gOqk=5+kdDZ|IDd+H-;tR3w~YIUAp) zItOZkLl37&i*uL_bB!n%mQV+sfvY{WS6BYENKa84_&rFtI+a6)k<{k%?!lBg-=8lP z*(=<#-&({Q-uc5=h1*5%*+R(Ir3;>8@Fr2lLRfd8*kluX$Md3Gem@DWy%sBi#)e#Y z*$hx1Pq1g}(z6^%=_ciuU$WSB5r-F#RO=u(wY`mG^t&Os@ovcHXw2&wJ{g6;*qA&V zoLMijc}iEmM)DHxZMjIYWWCig!Jz>r4RT=U9RR}=u$Q!)4Ap7l?w0o(-^qa`oW8_! z+UQ)x7Hu7&Lx_g)(gO*T^rKjyxwxgaKo9{ZN4D|t+2XPJosDk7Mal5Z{K=Qf@xCR4 z8EZ18N%YXv+KVtZt;KLc21so@nacsAr~2S}U?-6nKCO^vqB3T9z*-GIHVar+VbZo13?Z<-QXZZkBu~^-L%EH8uv&OMGJh;GUj<3D>Vp@{5%f5Q2{HNE z7OiS7kw&&bJ5IojIb@<867GW=o6&}&M}}xxRsdt9PmhH(I~YYsT;#9>?+SL~cifgN zO6l91NtK#(;b+)|9D`}zDTac#Y}jYvJ%5X1pjF_sy9vHk$TZ5+DEE6^m)T$64PEa$ z$jqc!yL9a()hjbL#ogfLX*E8A2hcbi=h6<@6Pni+Q9Ydp~80@k&xW zjo7p>xb>xClKr5OuOr?|1-i2m;!tu~Xs=Xx(~$0gVE&QlOc`)YFt3(c25V*dj+$DBk z!oehfQ?^`-7I|$nb$h1`dODyr=y=O=Z`N5q814#vv(eP1KoTqK*}Zm^s@WNBid6z9 zOq5FwGD7a9sgP*Hd`eBoJyI*v^}Qa&h^lDW7kt~^MWsz}V|L^;fI7GZG}uxnzKuyB z$bc3*ee0E+Cd;S-uv6|$SOT$P7xnX9k`I8CGz!+B=j#|F&kmJ8z9~n8O&(tdWmmxA z(zS3+c61X=xhRM<7z?=rR4zO8;=Bga>Jw*YM1%m5wVhnz3lkg!f)kjd2eS#5R4uq@ zZrtcZ7>1B}9xWH2eFQi$c@#YJl%ltYuzriIG911pxHeo+GSvA>Br@u}$BXn(FBWHK zY=IQ1X@IC4GVzQWY?k^madY5^N27O@2Ey2n1WT0ay^`gUH)1bx;#vgC%Z)?x_>KtI zVj-Ahp_;5F`t$?GG5sy9yYWe;zlGm7J_>Va|BN`6_;B~GvW4J7z5W&+BmJjrA>8+6 z(|lr2mn`JON#EvKe9QNEKxs5pp_nU`!vnsv%JRywI++0nD)iNM7~0irvw%ml9>bde z964uoeM7edNf8%cWim)tR6&z$#&amzGsu4G5>nEq)dVdTYBZ#<=dPuP+M~$+eWLi| zqbM`7hN$)d5`b}KnY;J0hQRrESF?ur4BD?$`}rL=zEJ&^Jw{wDX09fCykcZqbR=av zm}qc5wueMXl%>0E|D9hctw(eK=7_&>Kyf|rx4N9oW= zir1`Q>eZ7#=8Xf~Z0oXC04OKzIa}k*u4YUWw7JR;wV6^H5lk@ttBIb3qxju^+!mpq zLnS|IMJXUYi#wXs;5c}@q@-nx?h+mw8geJXy2+{-Gy{_`l|Ri79NIDrBG;P|V1qdr zsu5%I3e0NH|93RVPA^LKl9KFXS+&jWmgU#_QdG~vjR2F%v(&Q1L;#U>il?~h9Q->W z_K4MZihSNh){zbDm-lRP%bT~;Ht3)Yc)EW(Mwin>@)m9(rH5p&n-31xL~SImdAeXW zuL;)D$vvGs)NlJ}{x(dF9b=QEg5jF}$!bdaE1?c*raMv1D}3K0wS02~*^)=v3g#d$ zM#FDK>PY^jpwE|IvS-4W)bj2Qx=M7sYMeSfq6ls2Iu6tHDI0I9t5~MymlfpD3LyTr zU+z(SY&xGxlJLf+=8#kQdhQY47^2bJ#VGxTuriwyOjeWkyrKvo_F;nJgnnuriPr9U z=6`4##>T+C(bn#_stib3uBG9N&aDLR>VkpgEc>3%ez`bZ)RpS&N01)F8p>!40nL2E zXI#MI#Js=x-K?%+NjlbYdvwU!u_4mGRikZbR5OMjTT&xu+}+`BDpEJB3D^#?Ybdl7 zb5SQ{oj&+-M;#lZ9-_G=EM&!O$mREg+8CqHH5_A=d53uPn8ESJ^unWE#9-Qz!9ib+ z0Lp%s=QX%SOqD7I25B5200g=VC{4VK+Kg0xkX98ngMT`AWw02IpSPM7yM z!z!<&Dul_BQnQbrQXM%U4zD{+&b$ULNKgRx|EbKC;SQ1X2dT67U@gyNo%ec5teHIv80gL zlx`?LEk}lPcWc>~EM{|`bAoo7G1!=|-M7|63sYvc{5&^(19<8F=G2;DM#W^av;Oc%}kT8$j&@MRrLo`hqI_)IrxD+VT37rdP|s# zKwi!q7X;Q={B^_ny-R>l4xq%^^rkd=-6cXAIpk&*6Lkd{yLFJ-X~h)AJFs+0J?qHX zU5-5UQq-kw;GUN0k2!AK-U|ae0ASngr(FI+O$c>Zm!x(dW%Oj zoIMNkO`0c5g7RPw3qfUU+9n#;R%N4u&}0;-Z`mdLnz3;ORf6r{i0(x*#B{xt!OL=% z?<3E#<0v?mBq&|}T9SMj2Z+O5Krf zCGD*B`pn1)HnKqvE1^3gL|C&AH6xE_sWQcYM$ZhE3F>I%nK~B`!5)PGhJMD_5Gz@lUBE?7j+v)Aq}q}*o5Yn$2^hya8Ud+2X*Orix%_eCNd42A z$g*rY(2PK;Jds!V;|3a6zCD8Nn}j620)n)+GMlPQ(C>w4)>O%rhTmGhv9`;ocjPn{ zvz>X?*X7iWE;CGN-N=C{KdO5?4syyMrgG%e%<(9a5W0^zu zAocj%7DXz0%bgHQx~UROf801kbn#jPF5iMx5OvLf`nuLhkVrVyQAg^!>FwNlCh&BT zRnmBz5l&Ik8Um^kErg?z8O!iJU&U=}4k0H+_Tn%~o#nZb6@^zd5XF#$vHmvfTK1LM zdQXAXXD3tL=DzmfrL21H694vFM92Tr*ts|7aa{TNe|!q1Qk%Tom=s9>1YwsqMM<`$ zwM1z}?rxk_Y6}EG-~~Z|#zho2mG7SC`JL{bnHQkeRW>OCGt=GYoIdx<a+3rlGx*vNV=)mF^_+;No#Zzj2@ z**c9k_m=`wsF09S@n-2D9HBZza+9<*j2!=QP4)bSTQBmel=2cv%1=Vpm$ZhZok3Fx z7*4@9TQP8Y8rcDT(H88Wm4%_US#*;wJ~L;fw>z?QJ~G_ILTGr6x7)zC||dO_2J^Vbu&D_srWj_N8qHjKFdwN1&=?GOo?^yU=EL60V* zo?TqLN-kN zsaw~FHi+gFRo-QD9Sx`btMxG(pm*!dF@u)fvRczQen$^l-FM1pTJr$b?5wqrHx37Y zVHkYv>GD`Dayf+3b)#tWaB+6yb>r~Hj44hrAYcl!tRZ&g7Nz^ZZPBQsaS1YWAkWnH zwqrU&*>xr0o52Q}vPg0$*KqX=zeUXKZ*{6ig5}s%FW4s|ie(v9?THKZ&SqZ?Lw7JN zgd@2o=0Z1vo#u%Az!hEOCu2zZ)I$(NGO5n>uvrEmv=IP-$54NxCi9FtBbSH;?98M7 z?bP3*!)(w7ExBsvQcR2o9^)e`+>|7>*|e;TPt?_ z86zcbU`E9Bgd&Cdy*-K!2tv2+1}FEPVwlrHs7Yp;+2W}6_e*JYaBm+D;&Ywt>}OzC zJvX=x$@dTS*NyLyDVO;{V?lFEHf=QCA`!;7h5#q)jpf=~;R$gO{JtE?cuT^Ik5`{T z(4WB*C8_=1ev@WkOwFO6ROzh8bZzAaHYe|~)K(Gsdo*Cj&h_(N+qV?GoL4I`?G08j zKXA?!V1h_uAU-u|p@E3*w7BADu*nnq*p$;InKAy54fj{DDd}REsLK0G(rygET8nAs zY&#Zj=rCWtkqdQ~9sWC_YPd|HIw?fL1L!(NbxoC8&Gy*C)7AOUsLmH1!BnImo=fwZOFXb7Wh^ zViMEn8#D={gGM2OSNAiC3WbP61Q_>(4XiJ zFT9IHeEML3ccggHOy;dl@q%HVppoNjhaipJwX-OvZlM`O;n%Lpqu$LvE_ya(F3a9d zuBB9GCk?8eo4JTGgs_-i702zxT^xL~7avma1{P*-?!rH2Z$@6U3Rq`_q#ZWuh*36m zsJjR(o&jkqt~q%c*(O%lr}Id$!p?PxM}$eMIlIk7HwK-Dne!DB7Zg%SKIj=qi~y2C!%1l}%1b5w!8O(`x0!OO+(A8;#5A{vZiT73LHSAl{hWn5wQPEhto&Vi z<+SgF!#U$lEdj(v^@yhW5YbEcYOHs856h1CiXy|K>;k;w*hP-1>Tg2x^a3&(rs2W^ zE|&5x7J;2(q>Yz~N0G0$R7o+Si`Q;KmKnV1^1jv*%B-9#(xk@B z1D2~jA7yRRqx*)%2PPjNT|xl|%^d7{oi9Gf>v?tl*~LXAt8jKwt(k z^6+iUQ=1M~&s4jCRjlwwM4%TdnQ;#r>TemqKIZblfOG1WkVt(5!d%}x40!QAJAHyu z(GQ-T?jbyvJ2g{$vD3Xq4uUs3m8s9^ig6DcD{Z3=Vf1hal^H__!)7JPZn0md8*{{klo5YC9Q$ECf0g zU`~(Tx0eQ~_|SYe6rBkNoSp65Xqe`J6`TbO%{_(O1R4Ax;4M!Wzn_7on<$p(G{CT2 zrZ@^8>RiUG>BKiLGX{u;OF?STq}OW4$X?ryGh6EmMSP^5&{(8gB%*28p7m``q#_9Q zq$cY4ccp_?Z<8n$;3H%-Uy*1h(0-zupy}1Aks05d=a!NCS>wY3)dZP_%`vkk{H<(( zZF;x@?c;r}w5V@i9+SohHIL1Wdbz*?T)L0G|Wo4;HmGFVR`&~_qY;Y!GqqwgSgb~#`BRqYyRg% zSTR@k!zB&dk?8giPkcIPe~bpbv&U?-zLsn$-)Wm6Qbh(w6M4AP@JjXH3rHJ^`y`dn z1^nKzTd5~SRbFcoa8Kj~ZI@zdPhQZr6Grq@iQ1NL5(TwQO(*Q1ues5DO-Y8m*q@FX zWQ0FzQ2e|m+~~cibR{NLl2BE7I++WB_#&ao@OZa~&?@>9b8(JhLX(;Cq!tlSnmt(! zwfP>Ho$5SS@F2|Po&BQlRN!rMtEIW}=trZjAymRJRJ8__C`6&-U5>$!y2cv0$k-cD zLi=EY&rIU(B22lbA-49pE~7AG_edgVszxlHh7Wv+v;*P|p&B7?^)#6uxsxU2tzhdP zZA%8#yWTSz|IEJ_;h8kGwufIiX9*e_dN-|?nz#(eo0iULH-=Sl-AX|A5$DXF8`+tr&_?-4bd;783!3om zn3MMFMsUHxE;A1e)J|W5$2=wAQ0D$kvrdM+*nE|y7*pD}eoq{oh_d;NWxzW24KuV) z9mAW77gr9Ft_|hjPxfb?&%j*rmlqcAFqxVcpJB}$zI+=R2-xMg-TVHoPpC-(`)1CN zehn27MlS~1Qc)VIeOk0ViK7V&N**wTJ-(gq4F#Np_jju&609i?p<7eMbpr5GWsq%G zkGN;82M?HwAe?uvbEA7@oZQ4v;x@Ddfof7D46XldCZ*H?bf+&8)txoN)45CBv-L0# zt79Jdht&@$j+@(@FlCxDwf-igXux~7>>dKyYnkKRqT5mxLr#o(Q8M@e+rB%X@uv2- z6EC5!`R%t^#a!C?DwJOI9pAA^HR5oc$|#Pdj}`m$$@a7wW|MtUZ|#-UEGn3-#szrz zH>^e-SxSL!ro1*vmduyC`e5Y^dSh>6*mKL9juV15AYy}OTTP2GLb{ZeVU`7t;%HKT}<&F4-DC4FqU@i z=^*HbdQ-yk7X`r;ALvla-8Y|fG~vmuyMKKH%K-?&RLwRloYj-AMlwi<88u&~)_%W$ zsTaA!lm=Zg<^xl9nU*yY$`a7qLp_;PUXKE^hr~ z9`b6%D&S$;tO6!p)rJuwJ0?8*%8ny7Ky`C1p@21qU3l(&a`J^IR!5J=0>a2;d5t%`)zZgJ5~} zFUY&S$8ml0#raD6W?krwZ7U^(WU(X3alTSx!f4X3;C1QLUVXHUVv1I{#va;f|7w_& z84P{-+Gk@h0Djtl8?`IO*r`!>055k=k-`BM;q`sc8>8mZNv>ty5SvO2u==K6A4JL% z)^$G{aoKODz--Xt^i~dEo&O%N&eyTXr`n-=XIqe3WJ+D>aANs5bs$6uau>qLA4&Y< zds_MPswtvlMy#T4b`;Xa$@yW^S8xX!!s%Zr*$I*J;J$=vP$H!Y1RD6+aI3;pCSf(+B4lI)uAZB zT#`q-v}ep-X-f9cLp5)Op6X!cl>B*V#(dLo*<`RrZIJ4^x~*0<*8sTpvSR$hsGgCm z(=h~aY(@yl#9sDL@H*uO-m12T=?;*Dl1!tvTgeqY5>U4%`#u(NK3=kKC!cSO1QlD$ zx4#s0Yw5pvCaf{`kt7lvi~OJ?!%P%JS^wfhIL3FhG{xXESSzO8^cKxYckMkm zfiIoA$`_Xl@da3snFQjti?E-mt)2$f@$5@KF`|w7_TtHjaXgW4@JBycQc;`HJ;&Hn zvwquiBJ8OhirRCZz!#2Uw*REG0dq!xqKlP9A1J7&$q3#Z7C~zL?UaT`UYY>P^f>pNK}Y`a(TWU4D?JtApyiZo(*?3*GGF|G}-W z5WE8{yWO`cYQEcjJf)BNZi#o0o;k2WHj|^}=evcH{lN?c5iTJ_T8A}4RFR$Df~s&U zf8woNeF#yn>*AF?oR58>XcDr>Z*W#fkWdpfHYi3&S$qCbR0y{e*OzpH;6>c}LhENp zI{$teo9FE4&GkLk>6)}KC+rjz?P)t0*&WMftPe{xD1BJ#S?lMtnJudZHOf<%d*YYzqEBorhC!*wlqvjH_d~AxK7mtQSuT zhPhINU!>>m{RDhsDR-<1;`+giZeWd&XajHj0$ebO(IfzN7!yFvV88dGXu5TN9|NUf zV@kz+6E7+>3%ekbwmH9{+5FoanbknAuCl&XTRvR;=H62il%4kwIK84U7HQLVHSyrF zYS-z#wCU{f4BI#dA$zic8eOehBp>rt~d0Cz&?T( z>F-$1s~bltX$H~*9UfhIWELc+1-gCYqKC)p3R;Mt@zDt~?FRvk%^u84@QGTsqXlK7 z5Q3*?10%Rie+@Y0v1mK40!)!iW@!Q?*HT8xiZVh`>t@n`T=FgrNZ(eeqdt*cgE>E7 zx9?ogl(sNXU65TjLorUUg!CeRhC78%!3+J=LAl?@cRuaDVbZLZW&RBnu@A{C)KjdZ2f_p5PwtGiZO&KvvCAbWU0KJC@hBcCBHg;4XPR$EfH~WCKv`}(-3#>^+WXk;j>uX>d!VBEGYqUUjNC|wCCBO?Ya^$?2>r47Q)Mm{C zNRO7DhFAm$+{^1wmmr@Y^Y)CS^w%HnZJqXAlR>1w;?J?|1iemE3oL{%jKg>y*5u|e z+CH*5IX>R%q$0K6xua+`&m8Z0P2MfvXGVV3 z{=8{+ew%mSj6TO}J4ma=Yjg@Q^))_Jm6NwxZSg5xwad!(~ zy?buEeS~ozjd`=N>gm;D3Tp(SY<3O#gfDRSnr_8{T*tF@Lioe)1oh~fB+hQ)*hB$E zxSyr~N)HtKxIR-4`JgGjBH+70t zLt9)0QC7*SUiDLid~Rdp3Q+5EC8*kvD|ZZ@_#MEPguRDn>^x0XAl!^ytY!sm?BH~! zOb+}TNw+R!5_a{{3)9Gu9h>PzCdH?!tM;CMvP`tzQ{5xxedAL!oqhw|nW;vmvDmG|B&TwVv37&Ux64}It13nhIbwNn9PcO2{VRY&&k=cs;&Dlu1Se3#EJQ}e!*MF6+F;|sDzxI#P1m?%G;W)#qIa^8@xd znVR!`BL9AUlZ58UZohkz5lp_NRCS`g2+%@cp)Gu=`0UPP;RK z_--h2XFmx=N}^qOhEgl<@IlO>2nLs>lIXmqs?W$~o3JxEqakHmvOdA-XbkSV>d(V3 zcPc@j_$Lgo6#uE+KAl&%-3QkgW<(Q5>y|JFp#-%?$y*67IsjU>J}WdzED#$kfI&XN5iV<-P%N}>P_|Vv zE@(bXVC&{G1Zj?||H-;u7)k@MfHZZ?jGo%O%3G|REic_iv)G#S~Lr?Ya z!J>lKO}HUXOlvYe_K}8|=w;?qRanf8MI+-ek&?4`3m~p>Z%JFhGsa#u2SDJ#N6Qc@ z&A0TG=(Y0_L1b%o#CP&}#4fMJ)5XWNm0o3grYljgk?tCa>lOBF-B=fbY$DnfZnRn3 z0Rgw#tm!!4%rP?%H078*vU$}Fgi@)?Wr@TzxSG0_5@Y%M@Hq$WpCpt%B*qq>0!J_rk2V z4-Y^In3{cK*pUz`Js2Ws34)OD_E2l81cTFzk5-Q{0tV;ICu0Oc0dH12{6EAZmd`HE zkF`j)c_H^f)ZXqvgIQPp_B_Ll*WV{wLdz}oJg3!aNkXkTr~7{C_o1Zu!rYjMjs(v6 zN&>sY%}*QnrD^XcDqVvWh`H#u58Ro41zd?Bjj<|Mi*h($EW*LR7i)x-iuE5;OyyKir}s$aB4e;3L(eYzM=4(>Mg- z)8>3Xg+NQGTTXF*C6KSWzp_6IjvAQKtt3;Y`~er1<+!k|d(ycgrrb#x0HWb@3c%>( z(;0e987^0UNt?{1ew8?yN{mU0dfQq8h3<;n>+uobVC6a=K`^j`o4W3{ua83XTHmFV zz09z{ayDLcSAT_Yx^YXO_INHPR)shhUZWKTu?0~!Z)Hh?J^cxBfypIJ0#ST3g^GP- z!=3k~{}4@3`mL<)!8zmjS#Ye^bx6X4!^;b12ICTRckR-4t{p|rWq))*0blO;qS(fD zxhqK@MgHQ`u^_<=O@}<=TGh5KRXDf_i)a-ei zb-OK21t(2U#}HT!RE9`v)LE=I4(q=#-PCIm_UNsazBz6PJ0&}Gcfc!f-23*A%GUrP z4H5>+44Sy|^Duz%+~}mbrTYBzXdnbjDK8+W5N^=e)iyoW+UDIyMS?rW%1>HM58;M? zz+iKyvBW4!(c=6Jd*S?Z?sJ8o%AVCm_QHcXd5h&hh{;(LgC`$fU9Qoa6lWKg_FnEx zC|r>ICKgin%AwUYUg6;6hl{oTJ3n`K!+GQn%A87?*D$a^LqC$=jvUZgU2c1}YyJjbY%c0c@z@gNTcM(MPSB~N?OA2-_C6nczn#2g_Bg7h zmlIPdo3adrHa0yMfs&_j5%dc5LvCJ#X3jt?yHyA=^#kEbu?_ZGH(_$FFA0?Mz{T*_ z)lSVO$xX(bTbp-WT|{g+mh7y`OA+=|dBUN_vV3F%_*~4~rE$)4n9kGwTbWMcO@>VQv~3wD z-zwVfJ?f1XZ(C68T6zY3I~U=c?0;>R>XuPyH6}{uxjAS0T^y>eM1m^GL9=(rYj-l) z?49IRbpF=R8bQ-pWxMyepL!4wk9uPkknzUAETm1e@TO(Xz*Gm&?}ICP4knVmI#fHH z%8hl7>m`SqPD|95%!zSYYk8wY8av>`5ZxT(d^RaC_8siksUtp|tZlb5Zwl!-gCtT{ zrp~!kg8{`H^|^g&k|39ovD_Rfl-NG#YY8|(e&m~+@v;5XNE(iVNfc6N4K>f6wVNoC zPbzt{K_wU@KySX#Q~aZgmwk~{tWlV+lHlXwL6$yNEiHykXasT{+oAvl&-FNh0G6hJ zuo4A+qPdHWb$t>|lEdr7E)Rh|u`pwJ?wg)5-ev}-( z1P`)`2K3=5>LUMLUt+17Ov!-$Vr8}K9GOf6H7#Yzw81|%Gi@{pIM_F>M`J`$xPszB zrLF`9p}f{Psifg8i%7$jh=7J`h-Oki__vmvy(T2Y0OzXeR7sTK&O}-D=gae>L{a~m zAW1)^J)7g+sTwGtQ>0DEHM>JbSRsJY_`B>IQpQ7AEBoK5oQk&q$jgdoK96v2mBulV zi%p)aD3`|l!{gN+V{1jT5s}~E(pMg`GB^a(Bts}p;*V+2*M${Xs{X*0!wDB$t?@?Qok^+EX6mEM zg^>ViK2aAUG(}8j_y`TI7~xU{KSv2?o&-OdFWDhUS!BE1MBO#H2~~ZCeuoQgJ*vGq zGh}fnM*5AInvw9MrASP-lrmk5TUtxf1N0j^8P-YH)zz^&?z$4R(Ch+>B?&h34nlM^ zbB$^VA1LCfxjtobHbUb4@(pmJPDs6YiKFLMhNQ-C@3@TB7FK?9Yn@PY6jO9LEEn;AJvQqryP96bdTg&IcsBwELkW@M7u0DSl2Q&*mJfGCjW}q`ICw880y_ab|7YNMYMJe>Zp2$L&LMM@s5g2IRO&`BDViHUH^xuTLd3;8YE;R?hQdc6ZDOpNgn9eN1 zO+9sM2v5#wgEU78MAEmOEAw(KfIAUz$(#voEtdqEnE5h?ifXiELrVpE>b&Z)q&XAn zZruR`ck+Szh6donj)x|1(KFrHq97Yd-fNNMKi*uoVVs)QQdJG7;2^V9pex!d!)M9D zj-OVJGcQ;srqbt$kk2B%uknU)l|Cqe3(iNYZ(n>Z+wE=Et;JN@f1^#p(*5?RVUe8E z*N+%fIT*3pPO#Gi65Mqc7Vd%a+O(`pYDxl1`e#fNPb$g$LwQ@HIvZ&n)q0I>A#jcO z%@TjjalsiNl(_T7^OjX>?@N8DiEznv8Mh0>4+ur0_|Uy5uvv$1xL+Mu$>A}j1s|90 zCt7B*E|OUgyrjV2BnP7Durza5Oh6$?B5t->`6ko3Y?{V{m6~O(VX_bUleI^VhesgX zr|suKanUa;I+|Xx=r?RjfkK6GxuX!B#rQ3N2zJL#1@BB9sd9TK4dUCN+u* zSgsVrFG3cXHQL{9pqx|ZNcNT(<*ozJN4B-pRA$U`6&)Dm-ob?nLF5tZzkFvOhh$?ZUN{$?N*N< z2;Qw>8%V1Q!XAGu8c?y3&o>Q?4dqyAH=43*>Lq(7xH*f^gihMZ*fG{D+u*{RWr#E} zCe|~6r#_7s^Rf@=g$Og_D1+_zwH1}rJx3Ajy2Cu962{S|OsZ-XgGdnNuUew~E@u^h zvm6vx%S84mD=H5wQ#Ky;pFBzhEbaZ&=EG}DKSQKK0saWqT=Tv112)^Hvd5%AGsS4~ zJ_T}KIvdvmNfe8Ia~;H`7*uuvuH(+VV9;dpCgF9uTsaX?T3I~9>`uOhs^#wHt>J$=>4W^Tz zRpb*@HpGQVH+n@k$k0+WfjQ-ec_Pqd6bc$n*)$4;{B^<+t%!oEhbtoXin#1$K)+-M zF_6kj!&)D@hU$Sn7-8P#0rM`8Uaxh-6%paew5=BnoB>KpCZ#zPjjx{esKVKW5-98d z#Bsw>&+K{QV_S4oM@uJaD0MMo0baUK^2@=BiP0fGJ6iGSLzqMR!INZq+8p$;W}1up zL>Avh)W{?Qgu+MASXXoec~AAnFp^W>uQ7r;OWL&F8T*s0cOUL1m3%gpGd7SYi;|)S z@ejg9i!x*;#zXVH!#h-kS#y5FRGl|HG!c5h+JX>b$RFFQ-GEJb2%{{1e4k&QRGuD4 zLZPrr&(u|G^@#qmx2H;4#=4pAWYi()PBaa|_~Rm9uD(1uc(0Y3yrUhoaiXK?6#7fl zNqlav%vIJo--cEHRkPVB7Y_>lUrK{seR)Bhp@q@a$m;D%`CfUU`BfD8Op#SNyX23b z*GC^*u~my!vS_p!T!qpsyuX=W?7>I4#lgI%$d&KbAEc zmHPVR^2iTsFzR3MzTqUrBL1xk7xZX?&W6Huw)*x046Uy}zMzvo;VQR&^fiCtk0u+8 ze8`Ixm;Pe);E+GwL%_|y^AoXwN+W{+uUXSuBt?>%u*mCc!KsQ3lHKO^PCV0n3jH`h z==8^uTld3j(;1u0FRSI=^4I$K75c znl2Sn)ipmW@rs&FD=>=_DqUyTk7p}KHy&MoU4Z%oYT>>EwOt?BL~W@sg?7O%4H)p& zL!ZmORVT(~A7Hq2{NKsIl5B`p^ZGnMR|%4t=SOTkxQ7Vu$g+m&)5zju)m=z^g-D2) z5Ik?Q-n}hQmHovLhqH}ek(#1-_<(9Y_uyWA-uy|K@5L3%M%A^+T&2coM3L+Vd-0Yp z-Dh&gDZ1%$dPSK#G*gd9e_`&NF+k+(%}Zdz9I-<#S2+~HbuO0$UNa1I(yZJA^SUOH zTKH)ee84-WuoA&7IxmU!>y|6#`PcaI>wq>B(h$bEXHmu`NfTt!y7;o+`M}$hqV6J? zc3D*gYe6I5<{Lu;8QCBVh$F|-|F12Fxhg#{hAT=SEgRHDo=u)w z!$0t5@~*2W2*GInLmX3GF$&k`;=@m@Dh$Vrk_a~}_TiPDHT0>76E`W`dy7a+Q(W3fnr2_F} zRooOA0Md`|nJ@x#f(W*sR6D#4ihd5)e%@I2Rp%^Gq<=#!!7ZMxmd{WOCzi8s57GBM zY}fdS1T8`r>{w6xqbci7t&7w|1cfnEd4UXhEC@xl4&IxbEqX(%g!pW+PfM&*W`VTD3S?c5k09Qx?sjGyGhCpHqJ!JcX7hu3!q%x#(IBoB@sm7L8t%-CM+U?TSTW$WgXF#+14jpTLJgr3s$VHt2 zGwLXPntWK7vlrm_=C`?5{Aw9$sCJCqu~+Q{yhWNaZQfz-5XLdygUw|VQcg~2#DjI* z9epOQzPE%TzRVPE7)(95wDhI%DZuvbU1>R_hO1Fd!mJ0-V4_GJnQ1gCUb(!tb-r>) zGyRHX@w0e5ryHgwsG7VeLOAe5NK;Nje=Zffmk5rYiN8QcgrRvKO~3P^V0FWxv01jL zZ&!5247^pMflcvr(`GR*rAjndmOcy^EgLuY$2q5baSZ{nvbnd{H~zP(FFYDOcF=18 zE=Y3b*Nmk?)SOic8d6_}#{EcrGg^@PQLKx-8>DInYZ-X)cEC0b&$i!`l90-bAzq>h zxh6^e-Q_Ma6Quz`%ynF?P5tw5Su&?lZz+M5iw>@S!oEloDvyFjYL-duS1Dz@r8C2sL44Ak?_S0Mm zQGuc2Qi1DI-O4jrhQUo9)mQhDPsQawRxi$|(Nr;; zRB}?Bu5?jbc;7Yz|M_e5_ZJ-}ivBrSNd%sD1|5LV@y%t&*$;Vl@G z%`WTLe4%Yaa&o^f(Se%QC8|h|B}-AyR!nT}^q9jdeaWm^#!!g>utk;?2mXbkD|m;V zOf)Q*cJt1Qy6p$M4{&Bg+)&%|;0bH%wFLz(q%YqUPvCb78{UWeM!aEy#zqOA-Jksd zA*JA(9MmR8zE-+D!I9#x^#|bw!ovTs`u3aWfBwT){L*o&{_QoZT04$ARt(^n@|g3g5u#d%AmQ7Rrk5#KZ`DHX7X)RaAWWPX z%{60293gX=ZAH%GZPxaY`aTl zuq;5*Sr#t0DIL3Y2pC4lkd&>XIOYzCp7hr$O&SN~KJ}`#6OE=jeT#AL3;B3=$jNr- z!1NK*G6_b1>LV?AH#%{INU@O8fdYrJ!8V-=r=7(-u##+0=EPB92D0-_PAX9>JtFYS z^8IV+^ah}}g!A(shPZo4oN&%Bjj2q&Aec^*ma?hhe+yyBC>ImCuWERtBmm3emD;*5 z8EqU_@==9Pm`u)}p_pntgx-1y0uY4L*Mt06f||u@YgPl}UVC(7<5V zGg))uzwI${d`s5g)r<+Y0lTs@vP%Bm_MD4E=S@wM1k6iflAE(<~oh! zybpAX)h*?AVnh6ape5YGW|qzTrN<}*wH2V-@F>qLwmU3{$P%oy?pG@tB_p=q#IPwS z^o~q=JjEP?VwJ+NG8~R)ozLsml`RJ<2Aa*MW@&(;w4fO%JIkXSDoI`p#{6&b*gtS15CMd55(3cbp%oK;CsWygX0v-x zHpw0kyha1a5X!y6@qiLQ@H_Fe1N{!P$DN)uXd6uiqZr@C0gekk^rXT;6(rDYHuD2j z0vU3_gTn_W_FO5o_2au-A%DOuA%ahj1aI+}+-7avyn9M-`X-&Yo?UuR_ z*@k}`NR&b|%y6+23yaVmXu&CPnMjbWDN8B7I?^nP>t&Hg)BU0(1m)+j{G#q*MY+9w zLyp}MQuc_-#2rzBepS5%VUS%(nZQ4m%1S--;*0meK}+p9oh%@4&-E2fYAgrBY7~)c zT0cTzPP2<%aT~*BtkQh1pTO|^>!@q<@JQS|!$Y8Qfwlqk%PwX@YSgGiT4|DETpF!x zc(Z9~;=gDrCfc7XHN0)i^g-dix1H!3I(5>*nu2dd*wL*NAXk;11;vnwwyzz2c9H%YXIX zzJKx+f8g=tj~Dzw4(y7*e)M^adcRMv1Jwfb<`ag%|43LOGjZ*m{INOO-s+K4x%iP(i#sk)(Tu*7MmG7LqM<2a+)=W(%s`oOttlC9XcSF3 z$%b`8E0&Q!apvKU(S4vo;&~jNSnj5UGB{%r6TgLf$<8|aYF@Mv5*^x!M!+%kKG!Gj ze%nmSv14$O+vL3>gns9}YH&!(l@(a+G9?0&etq++rMYI-8~i`&plL`8sVw*8p!;SS zltQkyIq^51K5M?nP87Ap8@yXBbIy?gpxU@v#J-vs7FKs)Mk)y2%Kj!}!r)og!v=w9 zy0=W#sCA@1kmFW5Q+&>2Q>wr$af^8>K^nYg&MU|w>U!A4Xi!2zNgqMWYqGTVJ^UxC zE~?ZE5-&KzW#`!sd;Ij4Cz0~D{Y_6IKSK$=cu(OF_If;to8PZ;bd7BDl_DOaVy57t zVA+xq8Xy}TcR(Hx(e^HFHQl-nf`+TxIMSvmx+7#bQ_&=~o=MewbZ48)pxaLq;J_0d z%i3?#Q#0g*jNNmK45P)8o^o?Z$p^ICrWG$jb`$BSL3Y)1 zCU6v)U2SP1^RluMlHCQ?Td=RJEnDC)gH`p4Ut|YkiMaD~RcvF81$I~}OGFcAgzwze zPjWR&Fk9>KarIca%>C+xQr!rxD>4g^C>#}d)B<8`q^zBvlqxy)#CEpq<`yy6gaZvx zUJ?Jeo(nzLI37_9JZv$9Q>58auHhId!KM(qR^;Ku=D+Ynt z&d2DT4vPkM<5VhWXU1DVK@3GNd$@3IMadN>SW8+g&dM1Qh_F%gI)R85PEp2LmJ53- z-+3Gn#P%lxd^Y)AojKyQuaQfqwWkS8(vwVnGEvGdNmWoB4BXo)j_rD4_S8}$2#yfM ztV5WqS%+W&ta*Z3*aC-W8X6XCm?}?93JbAJhtQ$JC1@Wq%z_iI;Kxvd*o}9>Pa^#U zT>uP!87|%I!LYjy_~x&#ZAK%X!)%74w5Q?}r{auHQVZefJ=eqvo7qe??)foC1ejc$ z)YZ{gHVzV6H^t}{%$X}(cB2~9bZ6vSOMbz{M`jR%PN-K>DLxEKl(`hy&AB~iox@U4 zvh}c70-M!!{LRJ9IqWs;DL2~fR@?8fm*S-c2nKtt;6T>w0ZFX6J6LTq1+{^g{~}3k z-@$9UL5I7pU&6a}(5MU=hpAfsRee(Wn^<*1DxgUy&2+=?v~BB6!mNLMzVJ+@IZEXU z9erOq@ed{mCsUXdT{v-!5lD2?yf|JBlfnVUe~U?hw*$SO4%P&jy4A2@QIs4Gcg8&5 zuqYTNsR-=+{gy?sO(jSF@~15dGCelzl7+g}EDCzk@-1mFM58S=69dg5B&=s*0J9Q3 zysJf}hjWTb{Znu3R{Ts4*N-vd>geI|S*nL84I9ougKuq%GH%@w4<(#5Q^N1EZ{K){ zXG%EfR%qc-B}T&M=sbl2qLlHl(E!I`Ee5}(F=qLtBCe0v8kV7m!<4%6SXu_D1b1oT zGh#)$zOQ66CVp!in!xcNn1f~8=qVj;_2PB~>;`0mD?JzuTB3i7A}ib1)_wa#3RcQ~kL)dbRkX7I#YN3iRKoKO3>!^l>(83fiuF zzF|(6Np~|leeu}FE#^G+S2XC`gwXWN(9~|m5=cOa7c>=mQH-Vlt~;Rk7FE%Twa}@q zyE{j*g(0FDr@_qN+R=;+*))wfDnHifB6o5MZ!;f^rnfSPolOP}=?-IFFJff~b}v=; zfo&-CmBkxGcGLSVmahL^F3rKqeXk4Y7Ml(P_vNGR|JOW6Ms32jbsFKO{r7(bc3(A5 zP&jt6>gO=;=c?UNxN$EuW?mZk%X=tUu%REueS)d%f;bXg8h6&?LymQnKx-lM+eh?U zn>a8UcJt}LHD~szOzP3oyG&F(;14%VREO}~GEo?xTm8j?2RJUG-0318-gS;0AZe8EE{u&a4S=dU&h@iDaTsz{}Gh=gE4{K0E75gn=~-VWbda>FLUU?c+xbtw4fX`{ z*VSealf|ZL_5HfpQJ2x0^Y#B(H1yDZDSqMEZ!5{-g%wDwPTOD)i{hZXK1klVC*vcg z4WHv$Q3>}k3%hq?&DBSR>wEz|G{jfSVf+Z{?g=lkvb7W1SnyfV$*=RN?98lYCQ=9t91*40wki`W(=~Iu zu%Ph!l?^n95O6YxZCELab9IwxPKpgUfB$DYlCz(xGbG*C#3(c1+h%nOf@Vlzy$ZA` z8mNiw6SV(w?|$ZI0Ywv+8!NuN8I3Vk#Ac2!59pb%{S0qp%_&?X{G!K|FDZt=yK~Y_ z9=(0=8pWdOJ}W^dj^fgIka8ud0abhYuRVQW9gbo^%DYZCEmd2sE`mb~=T)m{nBaNe z?0Z0@@jql`5Cnrv3?%Xu^#lywoT1L|L0#>^dG8{fo_<2 zUDX#V%(uJC8w!mBJ%MQA%t_d!5krN{keFw-iIyS|s3w|t2~?1xX}tD;xB}FG;;i%L zs(Lx>Sq^DXZ!~KSUQM|M96DVg0j@LL@8Yr&e!Lf=uuIdJD!-sj1quw`)bllT?fT(P z1q#k+ta1MNQ65eUQGv4_&=9AauzAp8K6`%F6MDnqhLp0sz}-<$aPg37$j2574D$3A zc(3)_i{ArBQ#T3QmuSCWEk~&XOYymrlepxZj72%2otlx%T{|jYyX{v2RuI$%uL~b; z2dPq2qqwcJbpQx2^Xq^_=6tsL{$~s9&wg9%{QEz1bDeh!`#WAefBx?%8=v#nm*irB zgroTBD$4ZxdlZwtTJ3(m`};=^G+6T80jMG)&!Z2_4Ky!kW8B_C0`Gh@DV6w#w!i-~ zf9~`ot;XrQ-@Ve^bx-gqBh`%Up3wXj%^#B=80_+YaKY{&w&VP>x7&i_o}HDcM@?pe zE%7AabsR3IQcK}%hcGi6v%{}XR}xB_dC>9Q%$sR?T{3c>J9jjpe@`o|B5(LaonjV@ z;WsjS97j;pg$569mq2hoUTIiA`77R`>Gt#%8)z&jG+d8)HySG6l4X8G*9Xp?oNCQa zAaL8WX1bmoGVd6at)Xd2Puy#hoI2KRlhUaZhEI`$-a*WsNtBL_zT47l<*%^OSESs1 z!6a%fT_0%8l?Q-fW^_<61N~GHTf~g|GEaoFaIQ z7@n^#tYOt}2WbwG5CNgE`9+x5r81g^Nl@yCn@evlS|7j#kOYQRB5V4GDcDI4vfWs9 z24jU^kO&Vm5GifI0=K?%nL^A#FA)4c-XYSuMtR8bbLq6n01V|RdXeFF@)qRN4` zSro7sKw;8>eCOP-90;y$p0NpZA3x6kfbg>6NNT>N=H8gpNQZ5d7AS>;v+1U-=2wHm zH5~$Kj;MtK#TtJYK8V1mOCMS}2igJ2`8^yt(=7rKO*s&l+_LCYp-0T6@N@#(_wI{_ zmKgjEMu0k76=(&EQE!Ll4qb>ydJRzf4wYQ{g@j&Ub0DlVcc-Q__wQ8&t0*Djbg~}T*cdfMbwa{G z#KG%R74m`yRT5ETX2=$$K~@O>G)rOtA#en!ZLeE_L$v?00p3)_DQ&`x5}eNDk^VE_ zkeyWvm=Zi`)`9;{58O;afHu3H8<&HYF1O2ORWUm7nk6hB1uzIhu;(bF%uvU(<_1uxmr{ z*Nbb4ry+77Pmk)-c&<;FrCg~1P{{fnBR3;Fw@h~mCmNkzruixsO7o%kg5R(H5fA|v zw1G#oZU-#S-g%{B(Xh1ZytwA8SJw{!{Ijd8i^B)Sqkh94_a6P0k{F|IOv#c7oBc@( zGJ>Y*8TfCH8zg{$9)K0zBuMWJ@xNX@e*iwec?VlOLnuhDKe%u%bCjEJ&R>6g>(7PC z@#YMMLf%7k46lTPeg5dHk!iQD`#=`SeLd-mN zM57#*KFQHp_!$QB$17;R^NR1vl4-Sy^nGK}qg~wSRIGF8-?keFpEXMO6P}NCsAvLL zo62Sc^cr@MH}GkHsFkl^URe~kIcF>Z8-%= z=NI=wWB80U!NY)Mo#9(NB^57Tg4AAI5P (b) ? (a) : (b)) +#define SQR(a) ((a) * (a)) + +#define FAILIF(b) {if (b) {fprintf(ERROR_OUTPUT, "FAILIF triggered on line %d, file %s. Memory allocated: %d\n", __LINE__, __FILE__, totalAllocatedMemory); exit(1);}} +#define FAILIFWR(b, s) {if (b) {fprintf(ERROR_OUTPUT, "FAILIF triggered on line %d, file %s. Memory allocated: %d\nReason: %s\n", __LINE__, __FILE__, totalAllocatedMemory, s); exit(1);}} + +#define ASSERT(b) {if (!(b)) {fprintf(ERROR_OUTPUT, "ASSERT failed on line %d, file %s.\n", __LINE__, __FILE__); exit(1);}} + +// Critical ASSERT -- it is off when not debugging +#ifdef DEBUG +#define CR_ASSERT(b) {if (!(b)) {fprintf(ERROR_OUTPUT, "ASSERT failed on line %d, file %s.\n", __LINE__, __FILE__); exit(1);}} +#define CR_ASSERTWR(b, reason) {if (!(b)) {fprintf(ERROR_OUTPUT, "ASSERT failed on line %d, file %s.\nReason: %s.\n", __LINE__, __FILE__, reason); exit(1);}} +#else +#define CR_ASSERT(b) +#define CR_ASSERTWR(b, reason) +#endif + +#ifdef DEBUG +#define DC {fprintf(DEBUG_OUTPUT, "Debug checkpoint. Line %d, file %s.\n", __LINE__, __FILE__);} +#define DPRINTF1(p1) {fprintf(DEBUG_OUTPUT, p1);} +#define DPRINTF(p1, p2) {fprintf(DEBUG_OUTPUT, p1, p2);} +#define DPRINTF3(p1, p2, p3) {fprintf(DEBUG_OUTPUT, p1, p2, p3);} +#define DPRINTF4(p1, p2, p3, p4) {fprintf(DEBUG_OUTPUT, p1, p2, p3, p4);} +#else +#define DC +#define DPRINTF1(p1) +#define DPRINTF(p1, p2) +#define DPRINTF3(p1, p2, p3) +#define DPRINTF4(p1, p2, p3, p4) +#endif + +#define TimeVarT double + +#ifdef DEBUG_TIMINGS +#define TIMEV_START(timeVar) { \ + if (timingOn) { \ + /*CR_ASSERTWR(timeVar >= 0, "timevar<0.");*/ \ + timeval _timev; \ + struct timezone _timez; \ + gettimeofday(&_timev, &_timez); \ + double timeInSecs = _timev.tv_sec + (double)_timev.tv_usec / 1000000.0; \ + timeVar -= timeInSecs; \ + /*int _b = (currentTime <= timeInSecs * 1.0000001);*/ \ + /*if (!_b) {*/ \ + /*printf("currentTime: %lf\n", currentTime);*/ \ + /*printf("timeInSecs: %lf\n", timeInSecs);*/ \ + /*printf("currentTime <= timeInSecs: %d\n", _b);*/ \ + /*}*/ \ + CR_ASSERTWR((currentTime <= timeInSecs * 1.0000001), "currentTime not increasing."); \ + CR_ASSERTWR(((currentTime = timeInSecs) >= 0), "timeInSecs < 0"); \ + /*CR_ASSERTWR(timeVar < 0, "timevar>=0");*/ \ + } \ +} + +#define TIMEV_END(timeVar) { \ + if (timingOn) { \ + /*CR_ASSERTWR(timeVar < 0, "timevar >=0");*/ \ + timeval _timev; \ + struct timezone _timez; \ + gettimeofday(&_timev, &_timez); \ + double timeInSecs = _timev.tv_sec + (double)_timev.tv_usec / 1000000.0; \ + timeVar += timeInSecs - timevSpeed; \ + if (timeVar < 0) { timeVar = 0;}; \ + /*int _b = (currentTime <= timeInSecs * 1.0000001);*/ \ + /*if (!_b) {*/ \ + /*printf("currentTime: %lf\n", currentTime);*/ \ + /*printf("timeInSecs: %lf\n", timeInSecs);*/ \ + /*printf("currentTime <= timeInSecs: %d\n", _b);*/ \ + /*}*/ \ + CR_ASSERTWR((currentTime <= timeInSecs * 1.0000001), "currentTime not increasing."); \ + /*CR_ASSERTWR(((currentTime = timeInSecs) >= 0), "timeInSecs < 0");*/ \ + /*CR_ASSERTWR(timeVar >= -0.0000001, "timevar <0")*/; \ + } \ +} + +#else +#define TIMEV_START(timeVar) +#define TIMEV_END(timeVar) +#endif + +#define MALLOC(amount) ((amount > 0) ? malloc(totalAllocatedMemory + 2 * amount - (totalAllocatedMemory = totalAllocatedMemory + amount)) : NULL) + +#define REALLOC(oldPointer, amount) ((oldPointer != NULL) ? \ + realloc(oldPointer, totalAllocatedMemory + amount + (amount - (amount * 2) / 3) - (totalAllocatedMemory = totalAllocatedMemory + (amount - (amount * 2) / 3))) : \ + MALLOC(amount)) + +#define FREE(pointer) {if (pointer != NULL) {free(pointer);} pointer = NULL; } + +#endif diff --git a/src/lsh/sources/BucketHashing.cpp b/src/lsh/sources/BucketHashing.cpp new file mode 100755 index 0000000..c696255 --- /dev/null +++ b/src/lsh/sources/BucketHashing.cpp @@ -0,0 +1,698 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include "headers.h" + + +// Creates a new bucket with specified fields. The new bucket contains +// only a single entry -- bucketEntry. bucketEntry->nextEntry is +// expected to be NULL. +inline PGBucketT newGBucket(PUHashStructureT uhash, Uns32T control1, /*PPointT point, */ Int32T pointIndex, PGBucketT nextGBucket){ + PGBucketT bucket; + if (uhash != NULL && uhash->unusedPGBuckets != NULL){ + bucket = uhash->unusedPGBuckets; + uhash->unusedPGBuckets = uhash->unusedPGBuckets->nextGBucketInChain; + } else { + FAILIF(NULL == (bucket = (PGBucketT)MALLOC(sizeof(GBucketT)))); + nAllocatedGBuckets++; + } + ASSERT(bucket != NULL); + bucket->controlValue1 = control1; + bucket->firstEntry.pointIndex = pointIndex; + bucket->firstEntry.nextEntry = NULL; + bucket->nextGBucketInChain = nextGBucket; + + nGBuckets++; + return bucket; +} + +// Adds the entry to the bucket . +inline void addPointToGBucket(PUHashStructureT uhash, PGBucketT bucket/*, PPointT point*/ , Int32T pointIndex){ + ASSERT(bucket != NULL); + ASSERT(uhash != NULL); + + // create a new bucket entry for the point + TIMEV_START(timeBucketCreation); + PBucketEntryT bucketEntry; + if (uhash->unusedPBucketEntrys != NULL){ + bucketEntry = uhash->unusedPBucketEntrys; + uhash->unusedPBucketEntrys = uhash->unusedPBucketEntrys->nextEntry; + }else{ + FAILIF(NULL == (bucketEntry = (PBucketEntryT)MALLOC(sizeof(BucketEntryT)))); + nAllocatedBEntries++; + } + ASSERT(bucketEntry != NULL); + bucketEntry->pointIndex = pointIndex; + TIMEV_END(timeBucketCreation); + + bucketEntry->nextEntry = bucket->firstEntry.nextEntry; + bucket->firstEntry.nextEntry = bucketEntry; +} + +// Creates a new UH structure (initializes the hash table and the hash +// functions used). If ==HT_PACKED or HT_HYBRID_CHAINS, then +// gives the sizes of all the static arrays that are +// used. Otherwise parameter is not used. +PUHashStructureT newUHashStructure(IntT typeHT, Int32T hashTableSize, IntT bucketVectorLength, BooleanT useExternalUHFs, Uns32T *(&mainHashA), Uns32T *(&controlHash1), PUHashStructureT modelHT){ + PUHashStructureT uhash; + FAILIF(NULL == (uhash = (PUHashStructureT)MALLOC(sizeof(UHashStructureT)))); + uhash->typeHT = typeHT; + uhash->hashTableSize = hashTableSize; + uhash->nHashedBuckets = 0; + uhash->nHashedPoints = 0; + uhash->unusedPGBuckets = NULL; + uhash->unusedPBucketEntrys = NULL; + + uhash->prime = UH_PRIME_DEFAULT; + uhash->hashedDataLength = bucketVectorLength; + + uhash->chainSizes = NULL; + uhash->bucketPoints.pointsArray = NULL; + uhash->hybridChainsStorage = NULL; + + Int32T totalN = 0; + Int32T indexInStorage = 0; + Int32T lastIndexInSt = 0; + switch (typeHT) { + case HT_LINKED_LIST: + FAILIF(NULL == (uhash->hashTable.llHashTable = (PGBucketT*)MALLOC(hashTableSize * sizeof(PGBucketT)))); + uhash->chainSizes = NULL; + for(Int32T i = 0; i < hashTableSize; i++){ + uhash->hashTable.llHashTable[i] = NULL; + } + break; + case HT_STATISTICS: + ASSERT(FALSE); // Not supported + FAILIF(NULL == (uhash->hashTable.linkHashTable = (LinkPackedGBucketT**)MALLOC(hashTableSize * sizeof(LinkPackedGBucketT*)))); + FAILIF(NULL == (uhash->chainSizes = (IntT*)(MALLOC(hashTableSize * sizeof(IntT))))); + FAILIF(NULL == (uhash->bucketPoints.pointsList = (PointsListEntryT*)MALLOC(hashTableSize * sizeof(PointsListEntryT)))); + for(Int32T i = 0; i < hashTableSize; i++){ + uhash->chainSizes[i] = CHAIN_INIT_SIZE; + FAILIF(NULL == (uhash->hashTable.linkHashTable[i] = (LinkPackedGBucketT*)MALLOC(uhash->chainSizes[i] * sizeof(LinkPackedGBucketT))) && (uhash->chainSizes[i] > 0)); + if (uhash->chainSizes[i] > 0) { + // the first bucket is empty + uhash->hashTable.linkHashTable[i][0].indexStart = INDEX_START_EMPTY; + } + } + break; + case HT_PACKED: + ASSERT(FALSE); // Not supported +// ASSERT(modelHT != NULL); +// ASSERT(modelHT->typeHT == HT_STATISTICS); +// ASSERT(modelHT->nHashedPoints == hashTableSize); // TODO +// FAILIF(NULL == (uhash->hashTable.packedHashTable = (PackedGBucketT**)MALLOC(hashTableSize * sizeof(PackedGBucketT*)))); +// FAILIF(NULL == (uhash->chainSizes = (IntT*)(MALLOC(hashTableSize * sizeof(IntT))))); +// FAILIF(NULL == (uhash->bucketPoints.pointsArray = (PPointT*)MALLOC(hashTableSize * sizeof(PPointT)))); +// totalN = 0; // total number of points hashed so far. +// for(Int32T i = 0; i < hashTableSize; i++){ + +// // TODO: NOT TESTED AT ALL + +// // count first size of the chain: +// IntT j; +// for(j = 0; j < modelHT->chainSizes[i] && modelHT->hashTable.linkHashTable[i][j].indexStart != INDEX_START_EMPTY; j++) +// ; +// uhash->chainSizes[i] = j; + +// if (j == 0){ +// uhash->hashTable.packedHashTable[i] = NULL; +// continue; +// } + +// // allocate memory for the chain +// FAILIF(NULL == (uhash->hashTable.packedHashTable[i] = (PackedGBucketT*)MALLOC(uhash->chainSizes[i] * sizeof(PackedGBucketT)))); + +// // copy each bucket in the chain: +// for(j = 0; j < uhash->chainSizes[i]; j++){ +// // "general" info for the bucket +// uhash->hashTable.packedHashTable[i][j].controlValue1 = modelHT->hashTable.linkHashTable[i][j].controlValue1; +// uhash->hashTable.packedHashTable[i][j].indexStart = totalN; + +// Int32T p = modelHT->hashTable.linkHashTable[i][j].indexStart; +// ASSERT(p != INDEX_START_EMPTY); +// IntT count = 0; +// while(p != INDEX_START_EMPTY){ +// uhash->bucketPoints.pointsArray[totalN] = modelHT->bucketPoints.pointsList[p].point; +// totalN++; +// count++; +// p = modelHT->bucketPoints.pointsList[p].nextPoint; +// } +// uhash->hashTable.packedHashTable[i][j].nPointsInBucket = count; +// } +// } + break; + case HT_HYBRID_CHAINS: + ASSERT(modelHT != NULL); + ASSERT(modelHT->typeHT == HT_LINKED_LIST); + FAILIF(NULL == (uhash->hashTable.hybridHashTable = (PHybridChainEntryT*)MALLOC(hashTableSize * sizeof(PHybridChainEntryT)))); + FAILIF(NULL == (uhash->hybridChainsStorage = (HybridChainEntryT*)MALLOC((modelHT->nHashedPoints + modelHT->nHashedBuckets) * sizeof(HybridChainEntryT)))); + + // the index of the first unoccupied entry in hybridChainsStorage>. + indexInStorage = 0; + + // the index of the last unoccupied entry in hybridChainsStorage> + // (the entries of hybridChainsStorage> are filled from start and from end: + // at the beginning we fill the normal buckets with their points; + // at the end we fill the "overflow" points of buckets (additional points of buckets that have + // more than MAX_NONOVERFLOW_POINTS_PER_BUCKET points). + lastIndexInSt = modelHT->nHashedPoints + modelHT->nHashedBuckets - 1; + + for(Int32T i = 0; i < hashTableSize; i++){ + PGBucketT bucket = modelHT->hashTable.llHashTable[i]; + if (bucket != NULL){ + uhash->hashTable.hybridHashTable[i] = uhash->hybridChainsStorage + indexInStorage; // the position where the bucket starts + }else{ + uhash->hashTable.hybridHashTable[i] = NULL; + } + while(bucket != NULL){ + // Compute number of points in the current bucket. + Int32T nPointsInBucket = 1; + PBucketEntryT bucketEntry = bucket->firstEntry.nextEntry; + while(bucketEntry != NULL){ + nPointsInBucket++; + bucketEntry = bucketEntry->nextEntry; + } + + + // Copy the points from the bucket to the new HT. + ASSERT(nPointsInBucket > 0); + + uhash->hybridChainsStorage[indexInStorage].controlValue1 = bucket->controlValue1; + indexInStorage++; + uhash->hybridChainsStorage[indexInStorage].point.isLastBucket = (bucket->nextGBucketInChain == NULL ? 1 : 0); + uhash->hybridChainsStorage[indexInStorage].point.bucketLength = (nPointsInBucket <= MAX_NONOVERFLOW_POINTS_PER_BUCKET ? + nPointsInBucket : + 0); // 0 means there are "overflow" points + uhash->hybridChainsStorage[indexInStorage].point.isLastPoint = (nPointsInBucket == 1 ? 1 : 0); + uhash->hybridChainsStorage[indexInStorage].point.pointIndex = bucket->firstEntry.pointIndex; + indexInStorage++; + + // Store all other points in the storage + Uns32T currentIndex = indexInStorage; // index where the "current" point will be stored. + Uns32T nOverflow = 0; + Uns32T overflowStart = lastIndexInSt; + if (nPointsInBucket <= MAX_NONOVERFLOW_POINTS_PER_BUCKET){ + indexInStorage = indexInStorage + nPointsInBucket - 1; + }else{ + // bucket too large. + // store the overflow points at the end of the array hybridChainsStorage>. + nOverflow = nPointsInBucket - MAX_NONOVERFLOW_POINTS_PER_BUCKET; + overflowStart = lastIndexInSt - nOverflow + 1; + lastIndexInSt = overflowStart - 1; + + // specify the offset of the start of overflow points in the + // fields of points 2, 3, ... of the space + // immediately after the bucket. + Uns32T value = overflowStart - (currentIndex - 1 + MAX_NONOVERFLOW_POINTS_PER_BUCKET); + for(IntT j = 0; j < N_FIELDS_PER_INDEX_OF_OVERFLOW; j++){ + uhash->hybridChainsStorage[currentIndex + j].point.bucketLength = value & ((1U << N_BITS_FOR_BUCKET_LENGTH) - 1); + value = value >> N_BITS_FOR_BUCKET_LENGTH; + } + + // update + indexInStorage = indexInStorage + MAX_NONOVERFLOW_POINTS_PER_BUCKET - 1; + ASSERT(indexInStorage <= lastIndexInSt + 1); + + //FAILIFWR(nPointsInBucket > MAX_NONOVERFLOW_POINTS_PER_BUCKET, "Too many points in a bucket -- feature not implemented yet. Try to lower N_BITS_PER_POINT_INDEX as much as possible.");// TODO: not implemented yet + } + + bucketEntry = bucket->firstEntry.nextEntry; + while(bucketEntry != NULL){ + uhash->hybridChainsStorage[currentIndex].point.pointIndex = bucketEntry->pointIndex; + uhash->hybridChainsStorage[currentIndex].point.isLastPoint = 0; + bucketEntry = bucketEntry->nextEntry; + + currentIndex++; + if (currentIndex == indexInStorage && nPointsInBucket > MAX_NONOVERFLOW_POINTS_PER_BUCKET){ + // finished the normal alloted space -> going to the space reserved at end of the table. + currentIndex = overflowStart; + } + } + + // set the field of the last point = 1. + uhash->hybridChainsStorage[currentIndex - 1].point.isLastPoint = 1; + + bucket = bucket->nextGBucketInChain; + //ASSERT((uhash->hashTable.hybridHashTable[i] + 1)->point.bucketLength > 0); + } + } + ASSERT(indexInStorage == lastIndexInSt + 1); + uhash->nHashedPoints = modelHT->nHashedPoints; + uhash->nHashedBuckets = modelHT->nHashedBuckets; + break; + default: + ASSERT(FALSE); + } + + // Initializing the main hash function. + if (!useExternalUHFs){ + FAILIF(NULL == (uhash->mainHashA = (Uns32T*)MALLOC(uhash->hashedDataLength * sizeof(Uns32T)))); + for(IntT i = 0; i < uhash->hashedDataLength; i++){ + uhash->mainHashA[i] = genRandomUns32(1, MAX_HASH_RND); + } + mainHashA = uhash->mainHashA; + } else { + uhash->mainHashA = mainHashA; + } + + // Initializing the control hash functions. + if (!useExternalUHFs){ + FAILIF(NULL == (uhash->controlHash1 = (Uns32T*)MALLOC(uhash->hashedDataLength * sizeof(Uns32T)))); + for(IntT i = 0; i < uhash->hashedDataLength; i++){ + uhash->controlHash1[i] = genRandomUns32(1, MAX_HASH_RND); + } + controlHash1 = uhash->controlHash1; + } else { + uhash->controlHash1 = controlHash1; + } + + return uhash; +} + +// Removes all the buckets/points from the hash table. Used only for +// HT_LINKED_LIST. +void clearUHashStructure(PUHashStructureT uhash){ + ASSERT(uhash != NULL); + switch (uhash->typeHT) { + case HT_LINKED_LIST: + for(Int32T i = 0; i < uhash->hashTableSize; i++){ + PGBucketT bucket = uhash->hashTable.llHashTable[i]; + while(bucket != NULL){ + PGBucketT tempBucket = bucket; + bucket = bucket->nextGBucketInChain; + tempBucket->nextGBucketInChain = uhash->unusedPGBuckets; + uhash->unusedPGBuckets = tempBucket; + + PBucketEntryT bucketEntry = tempBucket->firstEntry.nextEntry; + while(bucketEntry != NULL){ + PBucketEntryT tempEntry = bucketEntry; + bucketEntry = bucketEntry->nextEntry; + tempEntry->nextEntry = uhash->unusedPBucketEntrys; + uhash->unusedPBucketEntrys = tempEntry; + } + } + + uhash->hashTable.llHashTable[i] = NULL; + } + break; + case HT_STATISTICS: + ASSERT(FALSE); // Not supported. +// for(IntT i = 0; i < uhash->hashTableSize; i++){ +// uhash->hashTable.linkHashTable[i][0].indexStart = INDEX_START_EMPTY; +// } + break; + default: + ASSERT(FALSE); + } + uhash->nHashedPoints = 0; + uhash->nHashedBuckets = 0; +} + +// Reorders uhash (of type HT_STATISTICS) to optimize for cache +// behavior. If is NULL, it is allocated, but not freed at +// the end (so, the caller has to free ). +void optimizeUHashStructure(PUHashStructureT uhash, PointsListEntryT *(&auxPtsList)){ + ASSERT(FALSE); // HT_STATISTICS not supported + ASSERT(uhash->typeHT == HT_STATISTICS); + + if (auxPtsList == NULL){ + FAILIF(NULL == (auxPtsList = (PointsListEntryT*)MALLOC(uhash->hashTableSize * sizeof(PointsListEntryT)))); + } + + Int32T newP = 0; + for(Int32T i = 0; i < uhash->hashTableSize; i++){ + for(IntT j = 0; j < uhash->chainSizes[i] && uhash->hashTable.linkHashTable[i][j].indexStart != INDEX_START_EMPTY; j++){ + Int32T p = uhash->hashTable.linkHashTable[i][j].indexStart; + uhash->hashTable.linkHashTable[i][j].indexStart = newP; + while (p != INDEX_START_EMPTY) { + auxPtsList[newP].point = uhash->bucketPoints.pointsList[p].point; + auxPtsList[newP].nextPoint = newP + 1; + newP++; + p = uhash->bucketPoints.pointsList[p].nextPoint; + } + auxPtsList[newP - 1].nextPoint = INDEX_START_EMPTY; + } + } + PointsListEntryT *tempList = auxPtsList; + auxPtsList = uhash->bucketPoints.pointsList; + uhash->bucketPoints.pointsList = tempList; +} + +// Frees the structure. If ==FALSE, then +// the hash functions are not freed (because they might be reused, and +// therefore shared by several PUHashStructureT structures). +void freeUHashStructure(PUHashStructureT uhash, BooleanT freeHashFunctions){ + if (uhash == NULL){ + return; + } + + switch (uhash->typeHT) { + case HT_LINKED_LIST: + for(IntT i = 0; i < uhash->hashTableSize; i++){ + PGBucketT bucket = uhash->hashTable.llHashTable[i]; + while (bucket != NULL){ + PGBucketT tempBucket = bucket; + bucket = bucket->nextGBucketInChain; + PBucketEntryT bucketEntry = tempBucket->firstEntry.nextEntry; + while (bucketEntry != NULL){ + PBucketEntryT tempEntry = bucketEntry; + bucketEntry = bucketEntry->nextEntry; + free(tempEntry); + } + free(tempBucket); + } + } + free(uhash->hashTable.llHashTable); + if (uhash->unusedPGBuckets != NULL){ + PGBucketT bucket = uhash->unusedPGBuckets; + while (bucket != NULL){ + PGBucketT tempBucket = bucket; + bucket = bucket->nextGBucketInChain; + free(tempBucket); + } + } + if (uhash->unusedPBucketEntrys != NULL){ + PBucketEntryT bucketEntry = uhash->unusedPBucketEntrys; + while (bucketEntry != NULL){ + PBucketEntryT tempEntry = bucketEntry; + bucketEntry = bucketEntry->nextEntry; + free(tempEntry); + } + } + ASSERT(uhash->chainSizes == NULL); + break; + case HT_HYBRID_CHAINS: + free(uhash->hashTable.hybridHashTable); + free(uhash->hybridChainsStorage); + ASSERT(uhash->chainSizes == NULL); + break; + default: + ASSERT(FALSE); + } + + if (freeHashFunctions){ + free(uhash->mainHashA); + free(uhash->controlHash1); + } + free(uhash); +} + +// Computes (a.b)mod UH_PRIME_DEFAULT. b is coded as blocks +// of size totaling . is of length . +inline Uns32T computeBlockProductModDefaultPrime(Uns32T *a, Uns32T *(b[]), IntT nBPieces, IntT size){ + LongUns64T h = 0; + //IntT j = 0; + //IntT bPiece = 0; + IntT bPieceSize = size / nBPieces; + IntT i = 0; + for(IntT bPiece = 0; bPiece < nBPieces; bPiece++){ + for(IntT j = 0; j < bPieceSize; j++, i++){ + h = h + (LongUns64T)a[i] * (LongUns64T)b[bPiece][j]; + h = (h & TWO_TO_32_MINUS_1) + 5 * (h >> 32); + if (h >= UH_PRIME_DEFAULT) { + h = h - UH_PRIME_DEFAULT; + } + CR_ASSERT(h < UH_PRIME_DEFAULT); + } + } + return h; +} + +// Computes (a.b)mod UH_PRIME_DEFAULT. +inline Uns32T computeProductModDefaultPrime(Uns32T *a, Uns32T *b, IntT size){ + LongUns64T h = 0; + //IntT i = 0; + for(IntT i = 0; i < size; i++){ + h = h + (LongUns64T)a[i] * (LongUns64T)b[i]; + h = (h & TWO_TO_32_MINUS_1) + 5 * (h >> 32); + if (h >= UH_PRIME_DEFAULT) { + h = h - UH_PRIME_DEFAULT; + } + CR_ASSERT(h < UH_PRIME_DEFAULT); + } + return h; +} + +// Compute fuction ((rndVector . data)mod prime)mod hashTableSize +// Vectors and are assumed to have length . +inline Uns32T computeUHashFunction(Uns32T *rndVector, Uns32T *(data[]), IntT nDataPieces, IntT size, Uns32T prime, Int32T hashTableSize){ + ASSERT(prime == UH_PRIME_DEFAULT); + ASSERT(rndVector != NULL); + ASSERT(data != NULL); + + Uns32T h = computeBlockProductModDefaultPrime(rndVector, data, nDataPieces, size) % hashTableSize; + + ASSERT(h >= 0 && h < hashTableSize); + + return h; +} + +inline Uns32T combinePrecomputedHashes(Uns32T *firstBucketVector, Uns32T *secondBucketVector, IntT nBucketVectorPieces, IntT uhfIndex){ + // CR_ASSERT(bucketVector != NULL); +// if (nBucketVectorPieces == 1) { +// // using normal functions. +// CR_ASSERT(bucketVector[1] != NULL); +// return (bucketVector[1][uhfIndex] % UH_PRIME_DEFAULT); +// } else { +// CR_ASSERT(nBucketVectorPieces == 2); // each of the functions is a pair of 2 functions. +// //CR_ASSERT(bucketVector[2] != NULL); +// //CR_ASSERT(bucketVector[3] != NULL); +// LongUns64T r = (LongUns64T)(bucketVector[2][uhfIndex]) + (LongUns64T)(bucketVector[3][uhfIndex + UHF_NUMBER_OF_HASHES]); +// if (r >= UH_PRIME_DEFAULT) { +// r -= UH_PRIME_DEFAULT; +// } +// CR_ASSERT(r < UH_PRIME_DEFAULT); +// return (Uns32T)r; +// } + if (nBucketVectorPieces == 1) { + // using normal functions. + Uns32T h = firstBucketVector[uhfIndex]; + if (h > UH_PRIME_DEFAULT){ + h = h - UH_PRIME_DEFAULT; + } + return h; + } else { + CR_ASSERT(nBucketVectorPieces == 2); // each of the functions is a pair of 2 functions. + //CR_ASSERT(bucketVector[2] != NULL); + //CR_ASSERT(bucketVector[3] != NULL); + LongUns64T r = (LongUns64T)(firstBucketVector[uhfIndex]) + (LongUns64T)(secondBucketVector[uhfIndex + UHF_NUMBER_OF_HASHES]); + if (r >= UH_PRIME_DEFAULT) { + r -= UH_PRIME_DEFAULT; + } + CR_ASSERT(r < UH_PRIME_DEFAULT); + return (Uns32T)r; + } +} + +// Adds the bucket entry (a point ) to the bucket defined by +// bucketVector in the uh structure with number uhsNumber. If no such +// bucket exists, then it is first created. +void addBucketEntry(PUHashStructureT uhash, IntT nBucketVectorPieces, Uns32T firstBucketVector[], Uns32T secondBucketVector[]/*, PPointT point*/ , Int32T pointIndex){ + CR_ASSERT(uhash != NULL); + // CR_ASSERT(bucketVector != NULL); + + Uns32T hIndex; + Uns32T control1; + + if (!USE_PRECOMPUTED_HASHES){ + // if not using the same hash functions across multiple + // UHashStructureT, then we need to compute explicitly the hases. + Uns32T *tempVector[2]; + tempVector[0] = firstBucketVector; + tempVector[1] = secondBucketVector; + hIndex = computeUHashFunction(uhash->mainHashA, tempVector, nBucketVectorPieces, uhash->hashedDataLength, uhash->prime, uhash->hashTableSize); + control1 = computeBlockProductModDefaultPrime(uhash->controlHash1, tempVector, nBucketVectorPieces, uhash->hashedDataLength); + } else { + // if using the same hash functions across multiple + // UHashStructureT, then we can use the (possibly partially) + // precomputed hash values. + CR_ASSERT(uhash->prime == UH_PRIME_DEFAULT); + hIndex = combinePrecomputedHashes(firstBucketVector, secondBucketVector, nBucketVectorPieces, UHF_MAIN_INDEX) % uhash->hashTableSize; + control1 = combinePrecomputedHashes(firstBucketVector, secondBucketVector, nBucketVectorPieces, UHF_CONTROL1_INDEX); + } + + PGBucketT p; + BooleanT found; // only used in the following disabled code + Int32T j; + Int32T temp; + switch (uhash->typeHT) { + case HT_LINKED_LIST: + p = uhash->hashTable.llHashTable[hIndex]; + while(p != NULL && + (p->controlValue1 != control1)) { + p = p->nextGBucketInChain; + } + if (p == NULL) { + // new bucket to add to the hash table + uhash->nHashedBuckets++; + uhash->hashTable.llHashTable[hIndex] = newGBucket(uhash, + control1, + pointIndex, + uhash->hashTable.llHashTable[hIndex]); + } else { + // add this bucket entry to the existing bucket + addPointToGBucket(uhash, p, pointIndex); + } + break; + case HT_PACKED: +// // The bucket should already exist. +// IntT i; +// for(i = 0; i < uhash->chainSizes[hIndex] && uhash->hashTable.packedHashTable[hIndex][i].nPointsInBucket > 0; i++){ +// if (uhash->hashTable.packedHashTable[hIndex][i].controlValue1 == control1){ +// break; +// } +// } +// uhash->hashTable.packedHashTable[hIndex][i].nPointsInBucket++; + break; + case HT_STATISTICS: + ASSERT(FALSE); +// uhash->bucketPoints.pointsList[uhash->nHashedPoints].point = point; +// found = FALSE; +// for(j = 0; j < uhash->chainSizes[hIndex] && uhash->hashTable.linkHashTable[hIndex][j].indexStart != INDEX_START_EMPTY; j++){ +// if (uhash->hashTable.linkHashTable[hIndex][j].controlValue1 == control1){ +// found = TRUE; +// break; +// } +// } +// if (!found) { +// // new bucket +// if (j >= uhash->chainSizes[hIndex]) { +// // dont have enough space in pREALLOCated memory. +// if (uhash->chainSizes[hIndex] > 0) { +// uhash->chainSizes[hIndex] = CEIL(uhash->chainSizes[hIndex] * CHAIN_RESIZE_RATIO); +// }else{ +// uhash->chainSizes[hIndex] = 1; +// } +// uhash->hashTable.linkHashTable[hIndex] = (LinkPackedGBucketT*)REALLOC(uhash->hashTable.linkHashTable[hIndex], uhash->chainSizes[hIndex] * sizeof(LinkPackedGBucketT)); +// uhash->hashTable.linkHashTable[hIndex][j].controlValue1 = control1; +// } +// uhash->hashTable.linkHashTable[hIndex][j].controlValue1 = control1; +// uhash->hashTable.linkHashTable[hIndex][j].indexStart = INDEX_START_EMPTY; +// uhash->nHashedBuckets++; +// if (j + 1 < uhash->chainSizes[hIndex]){ +// uhash->hashTable.linkHashTable[hIndex][j + 1].indexStart = INDEX_START_EMPTY; +// } +// } + +// temp = uhash->hashTable.linkHashTable[hIndex][j].indexStart; +// uhash->hashTable.linkHashTable[hIndex][j].indexStart = uhash->nHashedPoints; +// uhash->bucketPoints.pointsList[uhash->nHashedPoints].nextPoint = temp; + + break; + default: + ASSERT(FALSE); + } + uhash->nHashedPoints++; +} + +// Returns the bucket defined by the vector in the UH +// structure number . +GeneralizedPGBucket getGBucket(PUHashStructureT uhash, IntT nBucketVectorPieces, Uns32T firstBucketVector[], Uns32T secondBucketVector[]){ + Uns32T hIndex; + Uns32T control1; + + //TIMEV_START(timeGBHash); + if (!USE_PRECOMPUTED_HASHES){ + // if not using the same hash functions across multiple + // UHashStructureT, then we need to compute explicitly the hases. + Uns32T *tempVector[2]; + tempVector[0] = firstBucketVector; + tempVector[1] = secondBucketVector; + hIndex = computeUHashFunction(uhash->mainHashA, tempVector, nBucketVectorPieces, uhash->hashedDataLength, uhash->prime, uhash->hashTableSize); + control1 = computeBlockProductModDefaultPrime(uhash->controlHash1, tempVector, nBucketVectorPieces, uhash->hashedDataLength); + } else { + // if using the same hash functions across multiple + // UHashStructureT, then we can use the (possibly partially) + // precomputed hash values. + CR_ASSERT(uhash->prime == UH_PRIME_DEFAULT); + hIndex = combinePrecomputedHashes(firstBucketVector, secondBucketVector, nBucketVectorPieces, UHF_MAIN_INDEX) % uhash->hashTableSize; + control1 = combinePrecomputedHashes(firstBucketVector, secondBucketVector, nBucketVectorPieces, UHF_CONTROL1_INDEX); + } + //TIMEV_END(timeGBHash); + + GeneralizedPGBucket result; + PGBucketT p; + PHybridChainEntryT indexHybrid = NULL; + //TIMEV_START(timeChainTraversal); + switch(uhash->typeHT) { + case HT_LINKED_LIST: + p = uhash->hashTable.llHashTable[hIndex]; + while(p != NULL && + (p->controlValue1 != control1)) { + p = p->nextGBucketInChain; + nBucketsInChains++; + } + result.llGBucket = p; + return result; + case HT_STATISTICS: + ASSERT(FALSE); // HT_STATISTICS not supported anymore + for(IntT j = 0; j < uhash->chainSizes[hIndex] && uhash->hashTable.linkHashTable[hIndex][j].indexStart != INDEX_START_EMPTY; j++){ + if (uhash->hashTable.linkHashTable[hIndex][j].controlValue1 == control1){ + result.linkGBucket = &(uhash->hashTable.linkHashTable[hIndex][j]); + return result; + } + } + result.linkGBucket = NULL; + return result; + case HT_PACKED: + ASSERT(FALSE); // HT_PACKED not supported anymore + for(IntT j = 0; j < uhash->chainSizes[hIndex]; j++){ + if (uhash->hashTable.packedHashTable[hIndex][j].controlValue1 == control1){ + result.packedGBucket = &(uhash->hashTable.packedHashTable[hIndex][j]); + return result; + } + } + result.packedGBucket = NULL; + return result; + case HT_HYBRID_CHAINS: + indexHybrid = uhash->hashTable.hybridHashTable[hIndex]; + while (indexHybrid != NULL){ + if (indexHybrid->controlValue1 == control1){ + result.hybridGBucket = indexHybrid + 1; + return result; + }else{ + indexHybrid = indexHybrid + 1; + if (indexHybrid->point.isLastBucket != 0){ + result.hybridGBucket = NULL; + return result; + } + indexHybrid = indexHybrid + indexHybrid->point.bucketLength; + } + } + result.hybridGBucket = NULL; + return result; + break; + default: + ASSERT(FALSE); + } + //TIMEV_END(timeChainTraversal); + +} + +void precomputeUHFsForULSH(PUHashStructureT uhash, Uns32T *uVector, IntT length, Uns32T *result){ + if (length == uhash->hashedDataLength){ + result[UHF_MAIN_INDEX] = computeProductModDefaultPrime(uhash->mainHashA, uVector, length); + result[UHF_CONTROL1_INDEX] = computeProductModDefaultPrime(uhash->controlHash1, uVector, length); + } else { + ASSERT(2 * length == uhash->hashedDataLength); // the length is 1/2 of the bucket length + result[UHF_MAIN_INDEX] = computeProductModDefaultPrime(uhash->mainHashA, uVector, length); + result[UHF_CONTROL1_INDEX] = computeProductModDefaultPrime(uhash->controlHash1, uVector, length); + result[UHF_MAIN_INDEX + UHF_NUMBER_OF_HASHES] = computeProductModDefaultPrime(uhash->mainHashA + length, uVector, length); + result[UHF_CONTROL1_INDEX + UHF_NUMBER_OF_HASHES] = computeProductModDefaultPrime(uhash->controlHash1 + length, uVector, length); + } +} + diff --git a/src/lsh/sources/BucketHashing.h b/src/lsh/sources/BucketHashing.h new file mode 100755 index 0000000..cf10ca7 --- /dev/null +++ b/src/lsh/sources/BucketHashing.h @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef BUCKETHASHING_INCLUDED +#define BUCKETHASHING_INCLUDED + +// An entry (point) in a bucket of points (a bucket is specified by a +// vector in integers of length k). There is link to the actual point +// stored in the entry, as well as link to the next entry in the +// bucket. +typedef struct _BucketEntryT { + //PPointT point; + Int32T pointIndex; + _BucketEntryT *nextEntry; +} BucketEntryT, *PBucketEntryT; + +// The type definition for a bucket. A bucket is a container for +// points that all have the same value for hash function g (function g +// is a vector of K LSH functions). +typedef struct _GBucketT { + // These controlValues are used instead of the full k-vector (value + // of the hash function g) describing the bucket. With a high + // probability all buckets will have different pairs of + // controlValues. + Uns32T controlValue1; + + // The bucket entries (stored in a linked list). + BucketEntryT firstEntry; + _GBucketT *nextGBucketInChain; +} GBucketT, *PGBucketT; + +typedef struct _LinkPackedGBucketT { + Uns32T controlValue1; + Int32T indexStart; +} LinkPackedGBucketT, *PLinkPackedGBucketT; + +typedef struct _PackedGBucketT { + Uns32T controlValue1; + Int32T indexStart; + Int32T nPointsInBucket; +} PackedGBucketT, *PPackedGBucketT; + +// Number of bits reserved for storing the #points in a bucket +#define N_BITS_FOR_BUCKET_LENGTH (32 - 2 - N_BITS_PER_POINT_INDEX) + +// 2^N_BITS_FOR_BUCKET_LENGTH - 1 +#define MAX_NONOVERFLOW_POINTS_PER_BUCKET ((1U << N_BITS_FOR_BUCKET_LENGTH) - 1) + +// how many fields of N_BITS_FOR_BUCKET_LENGTH bits are needed to store a 32-bit (unsigned) integer. +#define N_FIELDS_PER_INDEX_OF_OVERFLOW ((32 + N_BITS_FOR_BUCKET_LENGTH - 1) / N_BITS_FOR_BUCKET_LENGTH) + +typedef union _HybridChainEntryT { + Uns32T controlValue1; + struct _OverloadedPoint { + Uns32T isLastBucket : 1; + Uns32T bucketLength : N_BITS_FOR_BUCKET_LENGTH; + Uns32T isLastPoint : 1; + Uns32T pointIndex : N_BITS_PER_POINT_INDEX; + } point; +} HybridChainEntryT, *PHybridChainEntryT; + +typedef union _GeneralizedPGBucket { + PGBucketT llGBucket; + PLinkPackedGBucketT linkGBucket; + PPackedGBucketT packedGBucket; + PHybridChainEntryT hybridGBucket; +} GeneralizedPGBucket; + +typedef struct _PointsListEntryT { + PPointT point; + Int32T nextPoint; +} PointsListEntryT; + +// A big number (>> max # of points) +#define INDEX_START_EMPTY 1000000000U + +// 4294967291 = 2^32-5 +#define UH_PRIME_DEFAULT 4294967291U + +// 2^29 +#define MAX_HASH_RND 536870912U + +// 2^32-1 +#define TWO_TO_32_MINUS_1 4294967295U + +// Whether to use the same hash functions (for universal hashing) or +// not. If using the same hash functions, then we can precompute some +// of the hash values and reuse them. +#define USE_SAME_UHASH_FUNCTIONS TRUE + +#define USE_PRECOMPUTED_HASHES USE_SAME_UHASH_FUNCTIONS + +// Two hash functions used: main one and a control one. +#define UHF_NUMBER_OF_HASHES 2 + +#define UHF_MAIN_INDEX 0 + +#define UHF_CONTROL1_INDEX 1 + +// Number of precomputed Uns32T words needed to store precomputed +// hashes of a (part of a) bucket description (more precisely of a +// function). It is 2*2 because: need 2 words for each of 1) the main +// hash; 2) control value 1 hash function (2 words per hash function +// because a function can occupy two positions in the bucket +// vector). +#define N_PRECOMPUTED_HASHES_NEEDED (UHF_NUMBER_OF_HASHES * 2) + +// An universal hash table with collision solved by chaining. The +// chains and the buckets are stored using either singly linked lists +// or static arrays (depending on the value of the field ). +typedef struct _UHashStructureT { + // The type of the hash table (can take values HT_*). when + // =HT_LINKED_LIST, chains&buckets are linked lists. when + // =HT_PACKED, chains&buckets are static arrays. when + // =HT_STATISTICS, chains are static arrays and buckets only + // count # of elements. when =HT_HYBRID_CHAINS, a chain is + // a "hybrid" array that contains both the buckets and the points + // (the an element of the chain array is of type + // ). all chains are conglamerated in the same + // array . + IntT typeHT; + + // The array containing the hash slots of the universal hashing. + union _hashTableT { + PGBucketT *llHashTable; + PackedGBucketT **packedHashTable; + LinkPackedGBucketT **linkHashTable; + PHybridChainEntryT *hybridHashTable; + } hashTable; + + // The sizes of each of the chains of the hashtable (used only when + // typeHT=HT_PACKED or HT_STATISTICS. + IntT *chainSizes; + + union _bucketPoints{ + PPointT *pointsArray; + PointsListEntryT *pointsList; + } bucketPoints; + + HybridChainEntryT *hybridChainsStorage; + + // The size of hashTable. + Int32T hashTableSize; + + // Number of elements(buckets) stored in the hash table in total (that + // is the number of non-empty buckets). + Int32T nHashedBuckets; + + Int32T nHashedPoints; + + // Unused (but allocated) instances of the corresponding + // structs. May be reused if needed (instead of allocated new + // memory). + PGBucketT unusedPGBuckets; + PBucketEntryT unusedPBucketEntrys; + + Uns32T prime; // the prime used for the universal hash functions. + IntT hashedDataLength;// the number of IntT's in an element from U (U is the set of values to hash). + + // The hash functions used for the universal hashing. + + // The main hash function (that defines the index + // of the slot in the table). + // The type of the hash function is: h_{a}(k) = ((a\cdot k)mod p)mod hashTableSize. + Uns32T *mainHashA; + + // Control hash functions: used to compute/check the s + // of s. + // The type of the hash function is: h_{a}(k) = (a\cdot k)mod p + Uns32T *controlHash1; +} UHashStructureT, *PUHashStructureT; + +#define HT_LINKED_LIST 0 + +#define HT_PACKED 1 + +#define HT_STATISTICS 2 + +#define HT_HYBRID_CHAINS 3 + +#define CHAIN_INIT_SIZE 0 +#define CHAIN_RESIZE_RATIO 1.5 + + + +PUHashStructureT newUHashStructure(IntT typeHT, Int32T hashTableSize, IntT bucketVectorLength, BooleanT useExternalUHFs, Uns32T *(&mainHashA), Uns32T *(&controlHash1), PUHashStructureT modelHT); + +void clearUHashStructure(PUHashStructureT uhash); + +void optimizeUHashStructure(PUHashStructureT uhash, PointsListEntryT *(&auxPtsList)); + +void freeUHashStructure(PUHashStructureT uhash, BooleanT freeHashFunctions); + +void addBucketEntry(PUHashStructureT uhash, IntT nBucketVectorPieces, Uns32T firstBucketVector[], Uns32T secondBucketVector[], Int32T pointIndex); + +GeneralizedPGBucket getGBucket(PUHashStructureT uhash, IntT nBucketVectorPieces, Uns32T firstBucketVector[], Uns32T secondBucketVector[]); + +void precomputeUHFsForULSH(PUHashStructureT uhash, Uns32T *uVector, IntT length, Uns32T *result); + +#endif diff --git a/src/lsh/sources/Geometry.cpp b/src/lsh/sources/Geometry.cpp new file mode 100755 index 0000000..92b0756 --- /dev/null +++ b/src/lsh/sources/Geometry.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include "headers.h" + +// SqrDistanceT computeSqrDistance(PointT p1, PointT p2){ +// SqrDistanceT result = 0; +// IntT n = MIN(p1.dimension, p2.dimension); + +// for (IntT i = 0; i < n; i++) +// result += SQR(p1.coordinates[i] - p2.coordinates[i]); + +// return result; +// } + +// Returns the Euclidean distance from point to . +RealT distance(IntT dimension, PPointT p1, PPointT p2){ + RealT result = 0; + + for (IntT i = 0; i < dimension; i++){ + result += SQR(p1->coordinates[i] - p2->coordinates[i]); + } + + return SQRT(result); +} diff --git a/src/lsh/sources/Geometry.h b/src/lsh/sources/Geometry.h new file mode 100755 index 0000000..cb80330 --- /dev/null +++ b/src/lsh/sources/Geometry.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef GEOMETRY_INCLUDED +#define GEOMETRY_INCLUDED + +/* A set of properties of a point */ +typedef enum { + ENUM_PPROP_FILE, ENUM_PPROP_LINE, ENUM_PPROP_OFFSET, ENUM_PPROP_NODE_KIND, + ENUM_PPROP_NUM_NODE, ENUM_PPROP_NUM_DECL, ENUM_PPROP_NUM_STMT, ENUM_PPROP_NUM_EXPR, + // the following is mainly for bug finding: + ENUM_PPROP_TBID, ENUM_PPROP_TEID, + ENUM_PPROP_nVARs, + ENUM_PPROP_CONTEXT_KIND, ENUM_PPROP_NEIGHBOR_KIND, ENUM_PPROP_OIDs, + ENUM_PPROP_LAST_NOT_USED +} pprop_t; + +// A simple point in d-dimensional space. A point is defined by a +// vector of coordinates. +typedef struct _PointT { + //IntT dimension; + IntT index; // the index of this point in the dataset list of points + RealT *coordinates; + RealT sqrLength; // the square of the length of the vector + char *filename; + int prop[ENUM_PPROP_LAST_NOT_USED-1]; // doesn't contain ENUM_PPROP_FILE. + char *oids; +} PointT, *PPointT; + +RealT distance(IntT dimension, PPointT p1, PPointT p2); + +#endif diff --git a/src/lsh/sources/GlobalVars.cpp b/src/lsh/sources/GlobalVars.cpp new file mode 100755 index 0000000..9d1bec6 --- /dev/null +++ b/src/lsh/sources/GlobalVars.cpp @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#define GLOBALVARS_CPP +#include "headers.h" diff --git a/src/lsh/sources/GlobalVars.h b/src/lsh/sources/GlobalVars.h new file mode 100755 index 0000000..d81b490 --- /dev/null +++ b/src/lsh/sources/GlobalVars.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef GLOBALVARS_INCLUDED +#define GLOBALVARS_INCLUDED + +#ifndef GLOBALVARS_CPP +#define DECLARE_EXTERN extern +#define EXTERN_INIT(x) +#else +#define DECLARE_EXTERN +#define EXTERN_INIT(x) x +#endif + +DECLARE_EXTERN Uns32T availableTotalMemory EXTERN_INIT(= DEFAULT_MEMORY_MAX_AVAILABLE); + +DECLARE_EXTERN TimeVarT timeComputeULSH; +DECLARE_EXTERN TimeVarT timeGetBucket; +DECLARE_EXTERN TimeVarT timeCycleBucket; +DECLARE_EXTERN TimeVarT timeDistanceComputation; +DECLARE_EXTERN TimeVarT timeResultStoring; +DECLARE_EXTERN TimeVarT timePrecomputeHash; +DECLARE_EXTERN TimeVarT timeGBHash; +DECLARE_EXTERN TimeVarT timeChainTraversal; +DECLARE_EXTERN TimeVarT timeBucketCreation; +DECLARE_EXTERN TimeVarT timeBucketIntoUH; +DECLARE_EXTERN TimeVarT timeCycleProc; +DECLARE_EXTERN TimeVarT timeRNNQuery; +DECLARE_EXTERN TimeVarT timeCopyingULSHs; +DECLARE_EXTERN TimeVarT timeTotalBuckets; +DECLARE_EXTERN TimeVarT timeUnmarking; + +DECLARE_EXTERN BooleanT timingOn EXTERN_INIT(= TRUE); +DECLARE_EXTERN TimeVarT currentTime EXTERN_INIT(= 0); +DECLARE_EXTERN TimeVarT timevSpeed EXTERN_INIT(= 0); + +DECLARE_EXTERN IntT nOfDistComps EXTERN_INIT(= 0); +DECLARE_EXTERN IntT totalAllocatedMemory EXTERN_INIT(= 0); +DECLARE_EXTERN IntT nGBuckets EXTERN_INIT(= 0); +DECLARE_EXTERN IntT nBucketsInChains EXTERN_INIT(= 0); +//DECLARE_EXTERN IntT nPointsInBuckets EXTERN_INIT(= 0); // total # of points found in collinding buckets (including repetitions) +DECLARE_EXTERN IntT nAllocatedGBuckets EXTERN_INIT(= 0); +DECLARE_EXTERN IntT nAllocatedBEntries EXTERN_INIT(= 0); + +DECLARE_EXTERN BooleanT noExpensiveTiming EXTERN_INIT(= FALSE); + + +#endif diff --git a/src/lsh/sources/LSHMain.cpp b/src/lsh/sources/LSHMain.cpp new file mode 100755 index 0000000..1e6e9a4 --- /dev/null +++ b/src/lsh/sources/LSHMain.cpp @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +/* + The main entry file containing the main() function. The main() + function parses the command line parameters and depending on them + calls the correspondin functions. + */ + +#include +#include +#include +#include "headers.h" + +#define N_SAMPLE_QUERY_POINTS 100 + +// The data set containing all the points. +PPointT *dataSetPoints = NULL; +// Number of points in the data set. +IntT nPoints = 0; +// The dimension of the points. +IntT pointsDimension = 0; +// The value of parameter R (a near neighbor of a point is any +// point

from the data set that is the within distance +// ). +//RealT thresholdR = 1.0; + +// The succes probability of each point (each near neighbor is +// reported by the algorithm with probability ). +RealT successProbability = 0.9; + +// Same as , only an array of R's (for the case when +// multiple R's are specified). +RealT *listOfRadii = NULL; +IntT nRadii = 0; + +RealT *memRatiosForNNStructs = NULL; + +char sBuffer[600000]; + +/* + Prints the usage of the LSHMain. + */ +void usage(char *programName){ + printf("Usage: %s #pts_in_data_set #queries dimension successProbability radius data_set_file query_points_file max_available_memory [-c|-p params_file]\n", programName); +} + +inline PPointT readPoint(FILE *fileHandle){ + PPointT p; + RealT sqrLength = 0; + FAILIF(NULL == (p = (PPointT)MALLOC(sizeof(PointT)))); + FAILIF(NULL == (p->coordinates = (RealT*)MALLOC(pointsDimension * sizeof(RealT)))); + for(IntT d = 0; d < pointsDimension; d++){ + FSCANF_REAL(fileHandle, &(p->coordinates[d])); + sqrLength += SQR(p->coordinates[d]); + } + fscanf(fileHandle, "%[^\n]", sBuffer); + p->index = -1; + p->sqrLength = sqrLength; + return p; +} + +// Reads in the data set points from in the array +// . Each point get a unique number in the field +// to be easily indentifiable. +void readDataSetFromFile(char *filename){ + FILE *f = fopen(filename, "rt"); + FAILIF(f == NULL); + + //fscanf(f, "%d %d ", &nPoints, &pointsDimension); + //FSCANF_DOUBLE(f, &thresholdR); + //FSCANF_DOUBLE(f, &successProbability); + //fscanf(f, "\n"); + FAILIF(NULL == (dataSetPoints = (PPointT*)MALLOC(nPoints * sizeof(PPointT)))); + for(IntT i = 0; i < nPoints; i++){ + dataSetPoints[i] = readPoint(f); + dataSetPoints[i]->index = i; + } +} + +// Tranforming from +// to +// . +void transformMemRatios(){ + RealT sum = 0; + for(IntT i = nRadii - 1; i >= 0; i--){ + sum += memRatiosForNNStructs[i]; + memRatiosForNNStructs[i] = memRatiosForNNStructs[i] / sum; + //DPRINTF("%0.6lf\n", memRatiosForNNStructs[i]); + } + ASSERT(sum <= 1.000001); +} + + +int compareInt32T(const void *a, const void *b){ + Int32T *x = (Int32T*)a; + Int32T *y = (Int32T*)b; + return (*x > *y) - (*x < *y); +} + +/* + The main entry to LSH package. Depending on the command line + parameters, the function computes the R-NN data structure optimal + parameters and/or construct the R-NN data structure and runs the + queries on the data structure. + */ +int main(int nargs, char **args){ + if(nargs < 9){ + usage(args[0]); + exit(1); + } + + //initializeLSHGlobal(); + + // Parse part of the command-line parameters. + nPoints = atoi(args[1]); + IntT nQueries = atoi(args[2]); + pointsDimension = atoi(args[3]); + successProbability = atof(args[4]); + char* endPtr[1]; + RealT thresholdR = strtod(args[5], endPtr); + if (thresholdR == 0 || endPtr[1] == args[5]){ + // The value for R is not specified, instead there is a file + // specifying multiple R's. + thresholdR = 0; + + // Read in the file + FILE *radiiFile = fopen(args[5], "rt"); + FAILIF(radiiFile == NULL); + fscanf(radiiFile, "%d\n", &nRadii); + ASSERT(nRadii > 0); + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + FAILIF(NULL == (memRatiosForNNStructs = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + for(IntT i = 0; i < nRadii; i++){ + FSCANF_REAL(radiiFile, &listOfRadii[i]); + ASSERT(listOfRadii[i] > 0); + FSCANF_REAL(radiiFile, &memRatiosForNNStructs[i]); + ASSERT(memRatiosForNNStructs[i] > 0); + } + }else{ + nRadii = 1; + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + FAILIF(NULL == (memRatiosForNNStructs = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + listOfRadii[0] = thresholdR; + memRatiosForNNStructs[0] = 1; + } + DPRINTF("No. radii: %d\n", nRadii); + //thresholdR = atof(args[5]); + availableTotalMemory = atol(args[8]); + + if (nPoints > MAX_N_POINTS) { + printf("Error: the structure supports at most %d points (%d were specified).\n", MAX_N_POINTS, nPoints); + fprintf(ERROR_OUTPUT, "Error: the structure supports at most %d points (%d were specified).\n", MAX_N_POINTS, nPoints); + exit(1); + } + + readDataSetFromFile(args[6]); + DPRINTF("Allocated memory (after reading data set): %d\n", totalAllocatedMemory); + + Int32T nSampleQueries = N_SAMPLE_QUERY_POINTS; + PPointT sampleQueries[nSampleQueries]; + Int32T sampleQBoundaryIndeces[nSampleQueries]; + if ((nargs < 9) || (strcmp("-c", args[9]) == 0)){ + // In this cases, we need to generate a sample query set for + // computing the optimal parameters. + + // Generate a sample query set. + FILE *queryFile = fopen(args[7], "rt"); + if (strcmp(args[7], ".") == 0 || queryFile == NULL || nQueries <= 0){ + // Choose several data set points for the sample query points. + for(IntT i = 0; i < nSampleQueries; i++){ + sampleQueries[i] = dataSetPoints[genRandomInt(0, nPoints - 1)]; + } + }else{ + // Choose several actual query points for the sample query points. + nSampleQueries = MIN(nSampleQueries, nQueries); + Int32T sampleIndeces[nSampleQueries]; + for(IntT i = 0; i < nSampleQueries; i++){ + sampleIndeces[i] = genRandomInt(0, nQueries - 1); + } + qsort(sampleIndeces, nSampleQueries, sizeof(*sampleQueries), compareInt32T); + //printIntVector("sampleIndeces: ", nSampleQueries, sampleIndeces); + Int32T j = 0; + for(Int32T i = 0; i < nQueries; i++){ + if (i == sampleIndeces[j]){ + sampleQueries[j] = readPoint(queryFile); + j++; + while (i == sampleIndeces[j]){ + sampleQueries[j] = sampleQueries[j - 1]; + j++; + } + }else{ + fscanf(queryFile, "%[^\n]", sBuffer); + fscanf(queryFile, "\n"); + } + } + nSampleQueries = j; + fclose(queryFile); + } + + // Compute the array sampleQBoundaryIndeces that specifies how to + // segregate the sample query points according to their distance + // to NN. + sortQueryPointsByRadii(pointsDimension, + nSampleQueries, + sampleQueries, + nPoints, + dataSetPoints, + nRadii, + listOfRadii, + sampleQBoundaryIndeces); + } + + RNNParametersT *algParameters = NULL; + PRNearNeighborStructT *nnStructs = NULL; + if (nargs > 9) { + // Additional command-line parameter is specified. + if (strcmp("-c", args[9]) == 0) { + // Only compute the R-NN DS parameters and output them to stdout. + + printf("%d\n", nRadii); + transformMemRatios(); + for(IntT i = 0; i < nRadii; i++){ + // which sample queries to use + Int32T segregatedQStart = (i == 0) ? 0 : sampleQBoundaryIndeces[i - 1]; + Int32T segregatedQNumber = nSampleQueries - segregatedQStart; + if (segregatedQNumber == 0) { + // XXX: not the right answer + segregatedQNumber = nSampleQueries; + segregatedQStart = 0; + } + ASSERT(segregatedQStart < nSampleQueries); + ASSERT(segregatedQStart >= 0); + ASSERT(segregatedQStart + segregatedQNumber <= nSampleQueries); + ASSERT(segregatedQNumber >= 0); + RNNParametersT optParameters = computeOptimalParameters(listOfRadii[i], + successProbability, + nPoints, + pointsDimension, + dataSetPoints, + segregatedQNumber, + sampleQueries + segregatedQStart, + (Uns32T)((availableTotalMemory - totalAllocatedMemory) * memRatiosForNNStructs[i])); + printRNNParameters(stdout, optParameters); + } + exit(0); + } else if (strcmp("-p", args[9]) == 0) { + // Read the R-NN DS parameters from the given file and run the + // queries on the constructed data structure. + if (nargs < 10){ + usage(args[0]); + exit(1); + } + FILE *pFile = fopen(args[10], "rt"); + FAILIFWR(pFile == NULL, "Could not open the params file."); + fscanf(pFile, "%d\n", &nRadii); + DPRINTF1("Using the following R-NN DS parameters:\n"); + DPRINTF("N radii = %d\n", nRadii); + FAILIF(NULL == (nnStructs = (PRNearNeighborStructT*)MALLOC(nRadii * sizeof(PRNearNeighborStructT)))); + FAILIF(NULL == (algParameters = (RNNParametersT*)MALLOC(nRadii * sizeof(RNNParametersT)))); + for(IntT i = 0; i < nRadii; i++){ + algParameters[i] = readRNNParameters(pFile); + printRNNParameters(stderr, algParameters[i]); + nnStructs[i] = initLSH_WithDataSet(algParameters[i], nPoints, dataSetPoints); + } + + pointsDimension = algParameters[0].dimension; + FREE(listOfRadii); + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + for(IntT i = 0; i < nRadii; i++){ + listOfRadii[i] = algParameters[i].parameterR; + } + } else{ + // Wrong option. + usage(args[0]); + exit(1); + } + } else { + FAILIF(NULL == (nnStructs = (PRNearNeighborStructT*)MALLOC(nRadii * sizeof(PRNearNeighborStructT)))); + // Determine the R-NN DS parameters, construct the DS and run the queries. + transformMemRatios(); + for(IntT i = 0; i < nRadii; i++){ + // XXX: segregate the sample queries... + nnStructs[i] = initSelfTunedRNearNeighborWithDataSet(listOfRadii[i], + successProbability, + nPoints, + pointsDimension, + dataSetPoints, + nSampleQueries, + sampleQueries, + (Uns32T)((availableTotalMemory - totalAllocatedMemory) * memRatiosForNNStructs[i])); + } + } + + DPRINTF1("X\n"); + + IntT resultSize = nPoints; + PPointT *result = (PPointT*)MALLOC(resultSize * sizeof(*result)); + PPointT queryPoint; + FAILIF(NULL == (queryPoint = (PPointT)MALLOC(sizeof(PointT)))); + FAILIF(NULL == (queryPoint->coordinates = (RealT*)MALLOC(pointsDimension * sizeof(RealT)))); + + FILE *queryFile = fopen(args[7], "rt"); + FAILIF(queryFile == NULL); + TimeVarT meanQueryTime = 0; + for(IntT i = 0; i < nQueries; i++){ + + RealT sqrLength = 0; + // read in the query point. + for(IntT d = 0; d < pointsDimension; d++){ + FSCANF_REAL(queryFile, &(queryPoint->coordinates[d])); + sqrLength += SQR(queryPoint->coordinates[d]); + } + queryPoint->sqrLength = sqrLength; + //printRealVector("Query: ", pointsDimension, queryPoint->coordinates); + + // get the near neighbors. + IntT nNNs = 0; + for(IntT r = 0; r < nRadii; r++){ + nNNs = getRNearNeighbors(nnStructs[r], queryPoint, result, resultSize); + printf("Total time for R-NN query at radius %0.6lf (radius no. %d):\t%0.6lf\n", (double)(listOfRadii[r]), r, timeRNNQuery); + meanQueryTime += timeRNNQuery; + + if (nNNs > 0){ + printf("Query point %d: found %d NNs at distance %0.6lf (radius no. %d). NNs are:\n", i, nNNs, (double)(listOfRadii[r]), r); + for(IntT j = 0; j < nNNs; j++){ + ASSERT(result[j] != NULL); + printf("%09d\tdist:%0.6lf\n", result[j]->index, distance(pointsDimension, queryPoint, result[j])); + CR_ASSERT(distance(pointsDimension, queryPoint, result[j]) <= listOfRadii[r]); + //DPRINTF("Distance: %lf\n", distance(pointsDimension, queryPoint, result[j])); + //printRealVector("NN: ", pointsDimension, result[j]->coordinates); + } + break; + } + } + if (nNNs == 0){ + printf("Query point %d: no NNs found.\n", i); + } + } + if (nQueries > 0){ + meanQueryTime = meanQueryTime / nQueries; + printf("Mean query time: %0.6lf\n", (double)meanQueryTime); + } + + //freePRNearNeighborStruct(nnStruct); + + return 0; +} diff --git a/src/lsh/sources/LocalitySensitiveHashing.cpp b/src/lsh/sources/LocalitySensitiveHashing.cpp new file mode 100755 index 0000000..e190f1a --- /dev/null +++ b/src/lsh/sources/LocalitySensitiveHashing.cpp @@ -0,0 +1,762 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +/* + The main functionality of the LSH scheme is in this file (all except + the hashing of the buckets). This file includes all the functions + for processing a PRNearNeighborStructT data structure, which is the + main R-NN data structure based on LSH scheme. The particular + functions are: initializing a DS, adding new points to the DS, and + responding to queries on the DS. + */ + +#include "headers.h" +#include + +void printRNNParameters(FILE *output, RNNParametersT parameters){ + ASSERT(output != NULL); + fprintf(output, "R\n"); + fprintf(output, "%0.9lf\n", parameters.parameterR); + fprintf(output, "Success probability\n"); + fprintf(output, "%0.9lf\n", parameters.successProbability); + fprintf(output, "Dimension\n"); + fprintf(output, "%d\n", parameters.dimension); + fprintf(output, "R^2\n"); + fprintf(output, "%0.9lf\n", parameters.parameterR2); + fprintf(output, "Use functions\n"); + fprintf(output, "%d\n", parameters.useUfunctions); + fprintf(output, "k\n"); + fprintf(output, "%d\n", parameters.parameterK); + fprintf(output, "m [# independent tuples of LSH functions]\n"); + fprintf(output, "%d\n", parameters.parameterM); + fprintf(output, "L\n"); + fprintf(output, "%d\n", parameters.parameterL); + fprintf(output, "W\n"); + fprintf(output, "%0.9lf\n", parameters.parameterW); + fprintf(output, "T\n"); + fprintf(output, "%d\n", parameters.parameterT); + fprintf(output, "typeHT\n"); + fprintf(output, "%d\n", parameters.typeHT); +} + +RNNParametersT readRNNParameters(FILE *input){ + ASSERT(input != NULL); + RNNParametersT parameters; + char s[1000];// TODO: possible buffer overflow + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + FSCANF_REAL(input, ¶meters.parameterR); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + FSCANF_REAL(input, ¶meters.successProbability); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + fscanf(input, "%d", ¶meters.dimension); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + FSCANF_REAL(input, ¶meters.parameterR2); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + fscanf(input, "%d", ¶meters.useUfunctions); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + fscanf(input, "%d", ¶meters.parameterK); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + fscanf(input, "%d", ¶meters.parameterM); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + fscanf(input, "%d", ¶meters.parameterL); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + FSCANF_REAL(input, ¶meters.parameterW); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + fscanf(input, "%d", ¶meters.parameterT); + + fscanf(input, "\n");fscanf(input, "%[^\n]\n", s); + fscanf(input, "%d", ¶meters.typeHT); + + return parameters; +} + +// Creates the LSH hash functions for the R-near neighbor structure +// . The functions fills in the corresponding field of +// . +void initHashFunctions(PRNearNeighborStructT nnStruct){ + ASSERT(nnStruct != NULL); + LSHFunctionT **lshFunctions; + // allocate memory for the functions + FAILIF(NULL == (lshFunctions = (LSHFunctionT**)MALLOC(nnStruct->nHFTuples * sizeof(LSHFunctionT*)))); + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + FAILIF(NULL == (lshFunctions[i] = (LSHFunctionT*)MALLOC(nnStruct->hfTuplesLength * sizeof(LSHFunctionT)))); + for(IntT j = 0; j < nnStruct->hfTuplesLength; j++){ + FAILIF(NULL == (lshFunctions[i][j].a = (RealT*)MALLOC(nnStruct->dimension * sizeof(RealT)))); + } + } + + // initialize the LSH functions + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + for(IntT j = 0; j < nnStruct->hfTuplesLength; j++){ + // vector a + for(IntT d = 0; d < nnStruct->dimension; d++){ + lshFunctions[i][j].a[d] = genGaussianRandom(); + } + // b + lshFunctions[i][j].b = genUniformRandom(0, nnStruct->parameterW); + } + } + + nnStruct->lshFunctions = lshFunctions; +} + +// Initializes the fields of a R-near neighbors data structure except +// the hash tables for storing the buckets. +PRNearNeighborStructT initializePRNearNeighborFields(RNNParametersT algParameters, Int32T nPointsEstimate){ + PRNearNeighborStructT nnStruct; + FAILIF(NULL == (nnStruct = (PRNearNeighborStructT)MALLOC(sizeof(RNearNeighborStructT)))); + nnStruct->parameterR = algParameters.parameterR; + nnStruct->parameterR2 = algParameters.parameterR2; + nnStruct->useUfunctions = algParameters.useUfunctions; + nnStruct->parameterK = algParameters.parameterK; + if (!algParameters.useUfunctions) { + // Use normal functions. + nnStruct->parameterL = algParameters.parameterL; + nnStruct->nHFTuples = algParameters.parameterL; + nnStruct->hfTuplesLength = algParameters.parameterK; + }else{ + // Use hash functions; a function is a pair of 2 functions. + nnStruct->parameterL = algParameters.parameterL; + nnStruct->nHFTuples = algParameters.parameterM; + nnStruct->hfTuplesLength = algParameters.parameterK / 2; + } + nnStruct->parameterT = algParameters.parameterT; + nnStruct->dimension = algParameters.dimension; + nnStruct->parameterW = algParameters.parameterW; + + nnStruct->nPoints = 0; + nnStruct->pointsArraySize = nPointsEstimate; + + FAILIF(NULL == (nnStruct->points = (PPointT*)MALLOC(nnStruct->pointsArraySize * sizeof(PPointT)))); + + // create the hash functions + initHashFunctions(nnStruct); + + // init fields that are used only in operations ("temporary" variables for operations). + + // init the vector and the vector + // + FAILIF(NULL == (nnStruct->pointULSHVectors = (Uns32T**)MALLOC(nnStruct->nHFTuples * sizeof(Uns32T*)))); + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + FAILIF(NULL == (nnStruct->pointULSHVectors[i] = (Uns32T*)MALLOC(nnStruct->hfTuplesLength * sizeof(Uns32T)))); + } + FAILIF(NULL == (nnStruct->precomputedHashesOfULSHs = (Uns32T**)MALLOC(nnStruct->nHFTuples * sizeof(Uns32T*)))); + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + FAILIF(NULL == (nnStruct->precomputedHashesOfULSHs[i] = (Uns32T*)MALLOC(N_PRECOMPUTED_HASHES_NEEDED * sizeof(Uns32T)))); + } + // init the vector + FAILIF(NULL == (nnStruct->reducedPoint = (RealT*)MALLOC(nnStruct->dimension * sizeof(RealT)))); + // init the vector + nnStruct->sizeMarkedPoints = nPointsEstimate; + FAILIF(NULL == (nnStruct->markedPoints = (BooleanT*)MALLOC(nnStruct->sizeMarkedPoints * sizeof(BooleanT)))); + for(IntT i = 0; i < nnStruct->sizeMarkedPoints; i++){ + nnStruct->markedPoints[i] = FALSE; + } + // init the vector + FAILIF(NULL == (nnStruct->markedPointsIndeces = (Int32T*)MALLOC(nnStruct->sizeMarkedPoints * sizeof(Int32T)))); + + nnStruct->reportingResult = TRUE; + + return nnStruct; +} + +// Constructs a new empty R-near-neighbor data structure. +PRNearNeighborStructT initLSH(RNNParametersT algParameters, Int32T nPointsEstimate){ + ASSERT(algParameters.typeHT == HT_LINKED_LIST || algParameters.typeHT == HT_STATISTICS); + PRNearNeighborStructT nnStruct = initializePRNearNeighborFields(algParameters, nPointsEstimate); + + // initialize second level hashing (bucket hashing) + FAILIF(NULL == (nnStruct->hashedBuckets = (PUHashStructureT*)MALLOC(nnStruct->parameterL * sizeof(PUHashStructureT)))); + Uns32T *mainHashA = NULL, *controlHash1 = NULL; + BooleanT uhashesComputedAlready = FALSE; + for(IntT i = 0; i < nnStruct->parameterL; i++){ + nnStruct->hashedBuckets[i] = newUHashStructure(algParameters.typeHT, nPointsEstimate, nnStruct->parameterK, uhashesComputedAlready, mainHashA, controlHash1, NULL); + uhashesComputedAlready = TRUE; + } + + return nnStruct; +} + +void preparePointAdding(PRNearNeighborStructT nnStruct, PUHashStructureT uhash, PPointT point); + + +// Construct PRNearNeighborStructT given the data set (all +// the points will be contained in the resulting DS). +// Currenly only type HT_HYBRID_CHAINS is supported for this +// operation. +PRNearNeighborStructT initLSH_WithDataSet(RNNParametersT algParameters, Int32T nPoints, PPointT *dataSet){ + ASSERT(algParameters.typeHT == HT_HYBRID_CHAINS); + ASSERT(dataSet != NULL); + ASSERT(USE_SAME_UHASH_FUNCTIONS); + + PRNearNeighborStructT nnStruct = initializePRNearNeighborFields(algParameters, nPoints); + + // Set the fields and . + nnStruct->nPoints = nPoints; + for(Int32T i = 0; i < nPoints; i++){ + nnStruct->points[i] = dataSet[i]; + } + + // initialize second level hashing (bucket hashing) + FAILIF(NULL == (nnStruct->hashedBuckets = (PUHashStructureT*)MALLOC(nnStruct->parameterL * sizeof(PUHashStructureT)))); + Uns32T *mainHashA = NULL, *controlHash1 = NULL; + PUHashStructureT modelHT = newUHashStructure(HT_LINKED_LIST, nPoints, nnStruct->parameterK, FALSE, mainHashA, controlHash1, NULL); + + Uns32T **(precomputedHashesOfULSHs[nnStruct->nHFTuples]); + for(IntT l = 0; l < nnStruct->nHFTuples; l++){ + FAILIF(NULL == (precomputedHashesOfULSHs[l] = (Uns32T**)MALLOC(nPoints * sizeof(Uns32T*)))); + for(IntT i = 0; i < nPoints; i++){ + FAILIF(NULL == (precomputedHashesOfULSHs[l][i] = (Uns32T*)MALLOC(N_PRECOMPUTED_HASHES_NEEDED * sizeof(Uns32T)))); + } + } + + for(IntT i = 0; i < nPoints; i++){ + preparePointAdding(nnStruct, modelHT, dataSet[i]); + for(IntT l = 0; l < nnStruct->nHFTuples; l++){ + for(IntT h = 0; h < N_PRECOMPUTED_HASHES_NEEDED; h++){ + precomputedHashesOfULSHs[l][i][h] = nnStruct->precomputedHashesOfULSHs[l][h]; + } + } + } + + //DPRINTF("Allocated memory(modelHT and precomputedHashesOfULSHs just a.): %d\n", totalAllocatedMemory); + + // Initialize the counters for defining the pair of functions used for functions. + IntT firstUComp = 0; + IntT secondUComp = 1; + for(IntT i = 0; i < nnStruct->parameterL; i++){ + // build the model HT. + for(IntT p = 0; p < nPoints; p++){ + // Add point to modelHT. + if (!nnStruct->useUfunctions) { + // Use usual functions (truly independent; s are precisly + // s). + addBucketEntry(modelHT, 1, precomputedHashesOfULSHs[i][p], NULL, p); + } else { + // Use functions (s are pairs of functions). + addBucketEntry(modelHT, 2, precomputedHashesOfULSHs[firstUComp][p], precomputedHashesOfULSHs[secondUComp][p], p); + } + } + + //ASSERT(nAllocatedGBuckets <= nPoints); + //ASSERT(nAllocatedBEntries <= nPoints); + + // compute what is the next pair of functions. + secondUComp++; + if (secondUComp == nnStruct->nHFTuples) { + firstUComp++; + secondUComp = firstUComp + 1; + } + + // copy the model HT into the actual (packed) HT. copy the uhash function too. + nnStruct->hashedBuckets[i] = newUHashStructure(algParameters.typeHT, nPoints, nnStruct->parameterK, TRUE, mainHashA, controlHash1, modelHT); + + // clear the model HT for the next iteration. + clearUHashStructure(modelHT); + } + + freeUHashStructure(modelHT, FALSE); // do not free the uhash functions since they are used by nnStruct->hashedBuckets[i] + + // freeing precomputedHashesOfULSHs + for(IntT l = 0; l < nnStruct->nHFTuples; l++){ + for(IntT i = 0; i < nPoints; i++){ + FREE(precomputedHashesOfULSHs[l][i]); + } + FREE(precomputedHashesOfULSHs[l]); + } + + return nnStruct; +} + + + +// // Packed version (static). +// PRNearNeighborStructT buildPackedLSH(RealT R, BooleanT useUfunctions, IntT k, IntT LorM, RealT successProbability, IntT dim, IntT T, Int32T nPoints, PPointT *points){ +// ASSERT(points != NULL); +// PRNearNeighborStructT nnStruct = initializePRNearNeighborFields(R, useUfunctions, k, LorM, successProbability, dim, T, nPoints); + +// // initialize second level hashing (bucket hashing) +// FAILIF(NULL == (nnStruct->hashedBuckets = (PUHashStructureT*)MALLOC(nnStruct->parameterL * sizeof(PUHashStructureT)))); +// Uns32T *mainHashA = NULL, *controlHash1 = NULL; +// PUHashStructureT modelHT = newUHashStructure(HT_STATISTICS, nPoints, nnStruct->parameterK, FALSE, mainHashA, controlHash1, NULL); +// for(IntT i = 0; i < nnStruct->parameterL; i++){ +// // build the model HT. +// for(IntT p = 0; p < nPoints; p++){ +// // addBucketEntry(modelHT, ); +// } + + + +// // copy the model HT into the actual (packed) HT. +// nnStruct->hashedBuckets[i] = newUHashStructure(HT_PACKED, nPointsEstimate, nnStruct->parameterK, TRUE, mainHashA, controlHash1, modelHT); + +// // clear the model HT for the next iteration. +// clearUHashStructure(modelHT); +// } + +// return nnStruct; +// } + + +// Optimizes the nnStruct (non-agressively, i.e., without changing the +// parameters). +void optimizeLSH(PRNearNeighborStructT nnStruct){ + ASSERT(nnStruct != NULL); + + PointsListEntryT *auxList = NULL; + for(IntT i = 0; i < nnStruct->parameterL; i++){ + optimizeUHashStructure(nnStruct->hashedBuckets[i], auxList); + } + FREE(auxList); +} + +// Frees completely all the memory occupied by the +// structure. +void freePRNearNeighborStruct(PRNearNeighborStructT nnStruct){ + if (nnStruct == NULL){ + return; + } + + if (nnStruct->points != NULL) { + free(nnStruct->points); + } + + if (nnStruct->lshFunctions != NULL) { + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + free(nnStruct->lshFunctions[i]); + } + free(nnStruct->lshFunctions); + } + + if (nnStruct->precomputedHashesOfULSHs != NULL) { + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + free(nnStruct->precomputedHashesOfULSHs[i]); + } + free(nnStruct->precomputedHashesOfULSHs); + } + + freeUHashStructure(nnStruct->hashedBuckets[0], TRUE); + for(IntT i = 1; i < nnStruct->parameterL; i++){ + freeUHashStructure(nnStruct->hashedBuckets[i], FALSE); + } + free(nnStruct->hashedBuckets); + + if (nnStruct->pointULSHVectors != NULL){ + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + free(nnStruct->pointULSHVectors[i]); + } + free(nnStruct->pointULSHVectors); + } + + if (nnStruct->reducedPoint != NULL){ + free(nnStruct->reducedPoint); + } + + if (nnStruct->markedPoints != NULL){ + free(nnStruct->markedPoints); + } + + if (nnStruct->markedPointsIndeces != NULL){ + free(nnStruct->markedPointsIndeces); + } +} + +// If == FALSe, no points are reported back in a +// function. In particular any point that is found in the bucket +// is considered to be outside the R-ball of the query point. If +// == TRUE, then the structure behaves normally. +void setResultReporting(PRNearNeighborStructT nnStruct, BooleanT reportingResult){ + ASSERT(nnStruct != NULL); + nnStruct->reportingResult = reportingResult; +} + +// Compute the value of a hash function u=lshFunctions[gNumber] (a +// vector of LSH functions) in the point . The +// result is stored in the vector . must be +// already allocated (and have space for Uns32T-words). +inline void computeULSH(PRNearNeighborStructT nnStruct, IntT gNumber, RealT *point, Uns32T *vectorValue){ + CR_ASSERT(nnStruct != NULL); + CR_ASSERT(point != NULL); + CR_ASSERT(vectorValue != NULL); + + for(IntT i = 0; i < nnStruct->hfTuplesLength; i++){ + RealT value = 0; + for(IntT d = 0; d < nnStruct->dimension; d++){ + value += point[d] * nnStruct->lshFunctions[gNumber][i].a[d]; + } + + vectorValue[i] = (Uns32T)(FLOOR_INT32((value + nnStruct->lshFunctions[gNumber][i].b) / nnStruct->parameterW) /*- MIN_INT32T*/); + } +} + +inline void preparePointAdding(PRNearNeighborStructT nnStruct, PUHashStructureT uhash, PPointT point){ + ASSERT(nnStruct != NULL); + ASSERT(uhash != NULL); + ASSERT(point != NULL); + + // Pi: if 'R' is small, just leave the point as it is to avoid division by 0 problem. + if ( nnStruct->parameterR < Pi_EPSILON /* better be the machine epsilon */ ) { + for(IntT d = 0; d < nnStruct->dimension; d++){ + nnStruct->reducedPoint[d] = point->coordinates[d]; + } + } else { + for(IntT d = 0; d < nnStruct->dimension; d++){ + nnStruct->reducedPoint[d] = point->coordinates[d] / nnStruct->parameterR; // Pi: ==>inf if 'R'==0; ==> all vectors are hashed into a same bucket ==> O(dn^2), instead of O(dn^\rho\logn), query time! + } + } + + // Compute all ULSH functions. + TIMEV_START(timeComputeULSH); + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + computeULSH(nnStruct, i, nnStruct->reducedPoint, nnStruct->pointULSHVectors[i]); + } + + // Compute data for . + if (USE_SAME_UHASH_FUNCTIONS) { + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + precomputeUHFsForULSH(uhash, nnStruct->pointULSHVectors[i], nnStruct->hfTuplesLength, nnStruct->precomputedHashesOfULSHs[i]); + } + } + + TIMEV_END(timeComputeULSH); +} + +inline void batchAddRequest(PRNearNeighborStructT nnStruct, IntT i, IntT &firstIndex, IntT &secondIndex, PPointT point){ +// Uns32T *(gVector[4]); +// if (!nnStruct->useUfunctions) { +// // Use usual functions (truly independent). +// gVector[0] = nnStruct->pointULSHVectors[i]; +// gVector[1] = nnStruct->precomputedHashesOfULSHs[i]; +// addBucketEntry(nnStruct->hashedBuckets[firstIndex], gVector, 1, point); +// } else { +// // Use functions (s are pairs of functions). +// gVector[0] = nnStruct->pointULSHVectors[firstIndex]; +// gVector[1] = nnStruct->pointULSHVectors[secondIndex]; +// gVector[2] = nnStruct->precomputedHashesOfULSHs[firstIndex]; +// gVector[3] = nnStruct->precomputedHashesOfULSHs[secondIndex]; + +// // compute what is the next pair of functions. +// secondIndex++; +// if (secondIndex == nnStruct->nHFTuples) { +// firstIndex++; +// secondIndex = firstIndex + 1; +// } + +// addBucketEntry(nnStruct->hashedBuckets[i], gVector, 2, point); +// } +} + +// Adds a new point to the LSH data structure, that is for each +// i=0..parameterL-1, the point is added to the bucket defined by +// function g_i=lshFunctions[i]. +void addNewPointToPRNearNeighborStruct(PRNearNeighborStructT nnStruct, PPointT point){ + ASSERT(nnStruct != NULL); + ASSERT(point != NULL); + ASSERT(nnStruct->reducedPoint != NULL); + ASSERT(!nnStruct->useUfunctions || nnStruct->pointULSHVectors != NULL); + ASSERT(nnStruct->hashedBuckets[0]->typeHT == HT_LINKED_LIST || nnStruct->hashedBuckets[0]->typeHT == HT_STATISTICS); + + nnStruct->points[nnStruct->nPoints] = point; + nnStruct->nPoints++; + + preparePointAdding(nnStruct, nnStruct->hashedBuckets[0], point); + + // Initialize the counters for defining the pair of functions used for functions. + IntT firstUComp = 0; + IntT secondUComp = 1; + + TIMEV_START(timeBucketIntoUH); + for(IntT i = 0; i < nnStruct->parameterL; i++){ + if (!nnStruct->useUfunctions) { + // Use usual functions (truly independent; s are precisly + // s). + addBucketEntry(nnStruct->hashedBuckets[i], 1, nnStruct->precomputedHashesOfULSHs[i], NULL, nnStruct->nPoints - 1); + } else { + // Use functions (s are pairs of functions). + addBucketEntry(nnStruct->hashedBuckets[i], 2, nnStruct->precomputedHashesOfULSHs[firstUComp], nnStruct->precomputedHashesOfULSHs[secondUComp], nnStruct->nPoints - 1); + + // compute what is the next pair of functions. + secondUComp++; + if (secondUComp == nnStruct->nHFTuples) { + firstUComp++; + secondUComp = firstUComp + 1; + } + } + //batchAddRequest(nnStruct, i, firstUComp, secondUComp, point); + } + TIMEV_END(timeBucketIntoUH); + + // Check whether the vectors & is still big enough. + if (nnStruct->nPoints > nnStruct->sizeMarkedPoints) { + nnStruct->sizeMarkedPoints = 2 * nnStruct->nPoints; + FAILIF(NULL == (nnStruct->markedPoints = (BooleanT*)REALLOC(nnStruct->markedPoints, nnStruct->sizeMarkedPoints * sizeof(BooleanT)))); + for(IntT i = 0; i < nnStruct->sizeMarkedPoints; i++){ + nnStruct->markedPoints[i] = FALSE; + } + FAILIF(NULL == (nnStruct->markedPointsIndeces = (Int32T*)REALLOC(nnStruct->markedPointsIndeces, nnStruct->sizeMarkedPoints * sizeof(Int32T)))); + } +} + +// Returns TRUE iff |p1-p2|_2^2 <= threshold +inline BooleanT isDistanceSqrLeq(IntT dimension, PPointT p1, PPointT p2, RealT threshold){ + RealT result = 0; + nOfDistComps++; + + //TIMEV_START(timeDistanceComputation); + for (IntT i = 0; i < dimension; i++){ + RealT temp = p1->coordinates[i] - p2->coordinates[i]; + result += SQR(temp); + if (result > threshold){ + TIMEV_END(timeDistanceComputation); + return 0; + } + } + //TIMEV_END(timeDistanceComputation); + + //return result <= threshold; + return 1; +} + +// // Returns TRUE iff |p1-p2|_2^2 <= threshold +// inline BooleanT isDistanceSqrLeq(IntT dimension, PPointT p1, PPointT p2, RealT threshold){ +// RealT result = 0; +// nOfDistComps++; + +// //TIMEV_START(timeDistanceComputation); +// for (IntT i = 0; i < dimension; i++){ +// result += p1->coordinates[i] * p2->coordinates[i]; +// } +// //TIMEV_END(timeDistanceComputation); + +// return p1->sqrLength + p2->sqrLength - 2 * result <= threshold; +// } + +// Returns the list of near neighbors of the point (with a +// certain success probability). Near neighbor is defined as being a +// point within distance . Each near neighbor from the +// data set is returned is returned with a certain probability, +// dependent on , , and . The +// returned points are kept in the array . If result is not +// allocated, it will be allocated to at least some minimum size +// (RESULT_INIT_SIZE). If number of returned points is bigger than the +// size of , then the is resized (to up to twice the +// number of returned points). The return value is the number of +// points found. +Int32T getNearNeighborsFromPRNearNeighborStruct(PRNearNeighborStructT nnStruct, PPointT query, PPointT *(&result), Int32T &resultSize){ + ASSERT(nnStruct != NULL); + ASSERT(query != NULL); + ASSERT(nnStruct->reducedPoint != NULL); + ASSERT(!nnStruct->useUfunctions || nnStruct->pointULSHVectors != NULL); + + PPointT point = query; + + if (result == NULL){ + resultSize = RESULT_INIT_SIZE; + FAILIF(NULL == (result = (PPointT*)MALLOC(resultSize * sizeof(PPointT)))); + } + + preparePointAdding(nnStruct, nnStruct->hashedBuckets[0], point); + + Uns32T precomputedHashesOfULSHs[nnStruct->nHFTuples][N_PRECOMPUTED_HASHES_NEEDED]; + for(IntT i = 0; i < nnStruct->nHFTuples; i++){ + for(IntT j = 0; j < N_PRECOMPUTED_HASHES_NEEDED; j++){ + precomputedHashesOfULSHs[i][j] = nnStruct->precomputedHashesOfULSHs[i][j]; + } + } + TIMEV_START(timeTotalBuckets); + + BooleanT oldTimingOn = timingOn; + if (noExpensiveTiming) { + timingOn = FALSE; + } + + // Initialize the counters for defining the pair of functions used for functions. + IntT firstUComp = 0; + IntT secondUComp = 1; + + Int32T nNeighbors = 0;// the number of near neighbors found so far. + Int32T nMarkedPoints = 0;// the number of marked points + for(IntT i = 0; i < nnStruct->parameterL; i++){ + TIMEV_START(timeGetBucket); + GeneralizedPGBucket gbucket; + if (!nnStruct->useUfunctions) { + // Use usual functions (truly independent; s are precisly + // s). + gbucket = getGBucket(nnStruct->hashedBuckets[i], 1, precomputedHashesOfULSHs[i], NULL); + } else { + // Use functions (s are pairs of functions). + gbucket = getGBucket(nnStruct->hashedBuckets[i], 2, precomputedHashesOfULSHs[firstUComp], precomputedHashesOfULSHs[secondUComp]); + + // compute what is the next pair of functions. + secondUComp++; + if (secondUComp == nnStruct->nHFTuples) { + firstUComp++; + secondUComp = firstUComp + 1; + } + } + TIMEV_END(timeGetBucket); + + PGBucketT bucket; + + TIMEV_START(timeCycleBucket); + switch (nnStruct->hashedBuckets[i]->typeHT){ + case HT_LINKED_LIST: + bucket = gbucket.llGBucket; + if (bucket != NULL){ + // circle through the bucket and add to the points that are near. + PBucketEntryT bucketEntry = &(bucket->firstEntry); + //TIMEV_START(timeCycleProc); + while (bucketEntry != NULL){ + //TIMEV_END(timeCycleProc); + //ASSERT(bucketEntry->point != NULL); + //TIMEV_START(timeDistanceComputation); + Int32T candidatePIndex = bucketEntry->pointIndex; + PPointT candidatePoint = nnStruct->points[candidatePIndex]; + if (isDistanceSqrLeq(nnStruct->dimension, point, candidatePoint, nnStruct->parameterR2) && nnStruct->reportingResult){ + //TIMEV_END(timeDistanceComputation); + if (nnStruct->markedPoints[candidatePIndex] == FALSE) { + //TIMEV_START(timeResultStoring); + // a new R-NN point was found (not yet in ). + if (nNeighbors >= resultSize){ + // run out of space => resize the array. + resultSize = 2 * resultSize; + result = (PPointT*)REALLOC(result, resultSize * sizeof(PPointT)); + } + result[nNeighbors] = candidatePoint; + nnStruct->markedPointsIndeces[nNeighbors] = candidatePIndex; + nnStruct->markedPoints[candidatePIndex] = TRUE; // do not include more points with the same index + nNeighbors++; + //TIMEV_END(timeResultStoring); + } + }else{ + //TIMEV_END(timeDistanceComputation); + } + //TIMEV_START(timeCycleProc); + bucketEntry = bucketEntry->nextEntry; + } + //TIMEV_END(timeCycleProc); + } + break; + case HT_STATISTICS: + ASSERT(FALSE); // HT_STATISTICS not supported anymore + +// if (gbucket.linkGBucket != NULL && gbucket.linkGBucket->indexStart != INDEX_START_EMPTY){ +// Int32T position; +// PointsListEntryT *pointsList = nnStruct->hashedBuckets[i]->bucketPoints.pointsList; +// position = gbucket.linkGBucket->indexStart; +// // circle through the bucket and add to the points that are near. +// while (position != INDEX_START_EMPTY){ +// PPointT candidatePoint = pointsList[position].point; +// if (isDistanceSqrLeq(nnStruct->dimension, point, candidatePoint, nnStruct->parameterR2) && nnStruct->reportingResult){ +// if (nnStruct->nearPoints[candidatePoint->index] == FALSE) { +// // a new R-NN point was found (not yet in ). +// if (nNeighbors >= resultSize){ +// // run out of space => resize the array. +// resultSize = 2 * resultSize; +// result = (PPointT*)REALLOC(result, resultSize * sizeof(PPointT)); +// } +// result[nNeighbors] = candidatePoint; +// nNeighbors++; +// nnStruct->nearPoints[candidatePoint->index] = TRUE; // do not include more points with the same index +// } +// } +// // Int32T oldP = position; +// position = pointsList[position].nextPoint; +// // ASSERT(position == INDEX_START_EMPTY || position == oldP + 1); +// } +// } + break; + case HT_HYBRID_CHAINS: + if (gbucket.hybridGBucket != NULL){ + PHybridChainEntryT hybridPoint = gbucket.hybridGBucket; + Uns32T offset = 0; + if (hybridPoint->point.bucketLength == 0){ + // there are overflow points in this bucket. + offset = 0; + for(IntT j = 0; j < N_FIELDS_PER_INDEX_OF_OVERFLOW; j++){ + offset += ((Uns32T)((hybridPoint + 1 + j)->point.bucketLength) << (j * N_BITS_FOR_BUCKET_LENGTH)); + } + } + Uns32T index = 0; + BooleanT done = FALSE; + while(!done){ + if (index == MAX_NONOVERFLOW_POINTS_PER_BUCKET){ + //CR_ASSERT(hybridPoint->point.bucketLength == 0); + index = index + offset; + } + Int32T candidatePIndex = (hybridPoint + index)->point.pointIndex; + CR_ASSERT(candidatePIndex >= 0 && candidatePIndex < nnStruct->nPoints); + done = (hybridPoint + index)->point.isLastPoint == 1 ? TRUE : FALSE; + index++; + if (nnStruct->markedPoints[candidatePIndex] == FALSE){ + // mark the point first. + nnStruct->markedPointsIndeces[nMarkedPoints] = candidatePIndex; + nnStruct->markedPoints[candidatePIndex] = TRUE; // do not include more points with the same index + nMarkedPoints++; + + PPointT candidatePoint = nnStruct->points[candidatePIndex]; + if (isDistanceSqrLeq(nnStruct->dimension, point, candidatePoint, nnStruct->parameterR2) && nnStruct->reportingResult){ + //if (nnStruct->markedPoints[candidatePIndex] == FALSE) { + // a new R-NN point was found (not yet in ). + //TIMEV_START(timeResultStoring); + if (nNeighbors >= resultSize){ + // run out of space => resize the array. + resultSize = 2 * resultSize; + result = (PPointT*)REALLOC(result, resultSize * sizeof(PPointT)); + } + result[nNeighbors] = candidatePoint; + nNeighbors++; + //TIMEV_END(timeResultStoring); + //nnStruct->markedPointsIndeces[nMarkedPoints] = candidatePIndex; + //nnStruct->markedPoints[candidatePIndex] = TRUE; // do not include more points with the same index + //nMarkedPoints++; + //} + } + }else{ + // the point was already marked (& examined) + } + } + } + break; + default: + ASSERT(FALSE); + } + TIMEV_END(timeCycleBucket); + + } + + timingOn = oldTimingOn; + TIMEV_END(timeTotalBuckets); + + // we need to clear the array nnStruct->nearPoints for the next query. + for(Int32T i = 0; i < nMarkedPoints; i++){ + ASSERT(nnStruct->markedPoints[nnStruct->markedPointsIndeces[i]] == TRUE); + nnStruct->markedPoints[nnStruct->markedPointsIndeces[i]] = FALSE; + } + + return nNeighbors; +} diff --git a/src/lsh/sources/LocalitySensitiveHashing.h b/src/lsh/sources/LocalitySensitiveHashing.h new file mode 100755 index 0000000..f1e5f41 --- /dev/null +++ b/src/lsh/sources/LocalitySensitiveHashing.h @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef LOCALITYSENSITIVEHASHING_INCLUDED +#define LOCALITYSENSITIVEHASHING_INCLUDED + +// The default value for algorithm parameter W. +#define PARAMETER_W_DEFAULT 4.0 + +// The probability p(1) -- a function of W. +// #define PROBABILITY_P1 0.8005 + +// The size of the initial result array. +#define RESULT_INIT_SIZE 8 + +// A function drawn from the locality-sensitive family of hash functions. +typedef struct _LSHFunctionT { + RealT *a; + RealT b; +} LSHFunctionT, *PLSHFunctionT; + +typedef struct _RNNParametersT { + RealT parameterR; // parameter R of the algorithm. + RealT successProbability; // the success probability 1-\delta + IntT dimension; // dimension of points. + RealT parameterR2; // = parameterR^2 + + // Whether to use hash functions instead of usual + // functions. When this flag is set to TRUE, functions are + // generated (which are roughly k/2-tuples of LSH), and a + // function is a pair of 2 different functions. + BooleanT useUfunctions; + + IntT parameterK; // parameter K of the algorithm. + + // parameter M (# of independent tuples of LSH functions) + // if useUfunctions==TRUE, parameterL = parameterM * (parameterM - 1) / 2 + // if useUfunctions==FALSE, parameterL = parameterM + IntT parameterM; + + IntT parameterL; // parameter L of the algorithm. + RealT parameterW; // parameter W of the algorithm. + IntT parameterT; // parameter T of the algorithm. + + // The type of the hash table used for storing the buckets (of the + // same function). + IntT typeHT; +} RNNParametersT, *PRNNParametersT; + +typedef struct _RNearNeighborStructT { + IntT dimension; // dimension of points. + IntT parameterK; // parameter K of the algorithm. + IntT parameterL; // parameter L of the algorithm. + RealT parameterW; // parameter W of the algorithm. + IntT parameterT; // parameter T of the algorithm. + RealT parameterR; // parameter R of the algorithm. + RealT parameterR2; // = parameterR^2 + + // Whether to use hash functions instead of usual + // functions. When this flag is set to TRUE, functions are + // generated (which are roughly k/2-tuples of LSH), and a + // function is a pair of 2 different functions. + BooleanT useUfunctions; + + // the number of tuples of hash functions used (= # of rows of + // ). When useUfunctions == FALSE, this field is equal + // to parameterL, otherwise, to , the number of hash + // functions (in this case, parameterL = m*(m-1)/2 = nHFTuples*(nHFTuples-1)/2 + IntT nHFTuples; + // How many LSH functions each of the tuple has (it is when + // useUfunctions == FALSE, and when useUfunctions == TRUE). + IntT hfTuplesLength; + + // number of points in the data set + Int32T nPoints; + + // The array of pointers to the points that are contained in the + // structure. Some types of this structure (of UHashStructureT, + // actually) use indeces in this array to refer to points (as + // opposed to using pointers). + PPointT *points; + + // The size of the array + Int32T pointsArraySize; + + // If == FALSE, no points are reported back in a + // function. In particular any point that is found in the + // bucket is considered to be outside the R-ball of the query point + // (the distance is still computed). If == TRUE, + // then the structure behaves normally. + BooleanT reportingResult; + + // This table stores the LSH functions. There are rows + // of LSH functions. + LSHFunctionT **lshFunctions; + + // Precomputed hashes of each of the of functions + // (to be used by the bucket hashing module). + Uns32T **precomputedHashesOfULSHs; + + // The set of non-empty buckets (which are hashed using + // PUHashStructureT). + PUHashStructureT *hashedBuckets; + + // *** + // The following vectors are used only for temporary operations + // within this R-NN structure during a query operation. + // *** + + // This vector is used to store the values of hash functions + // (-tuple of LSH fuctions). One function is a concatenation + // of of LSH functions. + Uns32T **pointULSHVectors; + + // A vector of length to store the reduced point (point + // with coordinates divided by ). + RealT *reducedPoint; + + // This vector is used for storing marked points in a query + // operation (for computing distances to a point at most once). If + // markedPoints[i]=TRUE then point was examined already. + BooleanT *markedPoints; + // This vector stored the indeces in the vector of all + // TRUE entries. + Int32T *markedPointsIndeces; + // the size of and of + IntT sizeMarkedPoints; +} RNearNeighborStructT, *PRNearNeighborStructT; + +void printRNNParameters(FILE *output, RNNParametersT parameters); + +RNNParametersT readRNNParameters(FILE *input); + +PRNearNeighborStructT initLSH(RNNParametersT algParameters, Int32T nPointsEstimate); + +PRNearNeighborStructT initLSH_WithDataSet(RNNParametersT algParameters, Int32T nPoints, PPointT *dataSet); + +//void optimizeLSH(PRNearNeighborStructT nnStruct); + +void freePRNearNeighborStruct(PRNearNeighborStructT nnStruct); + +void setResultReporting(PRNearNeighborStructT nnStruct, BooleanT reportingStopped); + +void addNewPointToPRNearNeighborStruct(PRNearNeighborStructT nnStruct, PPointT point); + +Int32T getNearNeighborsFromPRNearNeighborStruct(PRNearNeighborStructT nnStruct, PPointT query, PPointT *(&result), IntT &resultSize); + +#endif diff --git a/src/lsh/sources/Makefile b/src/lsh/sources/Makefile new file mode 100755 index 0000000..932d8e3 --- /dev/null +++ b/src/lsh/sources/Makefile @@ -0,0 +1,49 @@ +# DEFINE_FLOAT should be set by a configure script (using testFloat.cpp) +# if this doesn't work, try REAL_DOUBLE instead +DEFINE_FLOAT = REAL_FLOAT + +CPP_OPTS = -O3 -D$(DEFINE_FLOAT) # -Wunused-variable +LINK_OPTS = -O3 -lm +OUT_DIR = ../bin + +LSH_SRC = BucketHashing.cpp \ + Geometry.cpp \ + LocalitySensitiveHashing.cpp \ + Random.cpp \ + Util.cpp \ + GlobalVars.cpp \ + SelfTuning.cpp \ + NearNeighbors.cpp + +TEST_SRC = LSHMain.cpp \ + exactNNs.cpp \ + genDS.cpp \ + compareOutputs.cpp \ + genPlantedDS.cpp \ + enumBuckets.cpp \ + exploreBuckets.cpp + +LSH_OBJS = $(LSH_SRC:.cpp=.o) +TEST_BUILDS = $(addprefix $(OUT_DIR)/,$(TEST_SRC:.cpp=)) + +.PHONY: all clean clean_all + +all: $(TEST_BUILDS) + +$(OUT_DIR)/%: %.o $(LSH_OBJS) + g++ -o $@ $(LINK_OPTS) $^ + +clean: # remove intermediate files + -rm -f *~ *.o .depend + +clean_all: clean # also remove exe files + -rm -f $(TEST_BUILDS) + +%.o: %.cpp + g++ -c $(CPP_OPTS) $< + +.depend: $(LSH_SRC) $(TEST_SRC) + g++ -MM $^ > $@ + +include .depend + diff --git a/src/lsh/sources/NearNeighbors.cpp b/src/lsh/sources/NearNeighbors.cpp new file mode 100755 index 0000000..e3a8d7f --- /dev/null +++ b/src/lsh/sources/NearNeighbors.cpp @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +/* + An interface to the R-NN data structure. Here are two basic functions + -- creating the R-NN data structure (with or without data set points + given), and a function for replying to a NN query. The parameters of + R-NN data structure are computed automatically. + */ + +#include "headers.h" + +RealT lshPrecomp, uhashOver, distComp; + +/* + Checks whether the data types used are of the correct size. An + ASSERT will fail if some data type does not have the correct size. + */ +void checkDataTypes(){ + ASSERT(sizeof(IntT) == sizeof(int)); + ASSERT(sizeof(IntT) >= 2); // others are not quite supported (because of printf's). + ASSERT(sizeof(Int32T) >= 4); + ASSERT(sizeof(Uns32T) >= 4); + ASSERT(sizeof(LongUns64T) >= 8); +} + +/* + Initializes some of the global mechanisms (timing), and makes + necessary sanity checks (e.g., data types sizes). + */ +void initializeLSHGlobal(){ + checkDataTypes(); + + // Initialize global variables + timingOn = TRUE; + + // Initialize timings. + tuneTimeFunctions(); + + // Initialize the random number generator. + initRandom(); +} + +// Creates a new R-near neighbor data structure, +// PRNearNeighborStructT, from the parameters and +// . is the estimate of the +// number of points that will be in the data structure (a better +// estimate will better optimize the amount of memory used and the +// performance). +PRNearNeighborStructT initRNearNeighbor(RealT thresholdR, RealT successProbability, Int32T nPointsEstimate){ + ASSERT(FALSE); // Not implemented yet +} + +/* + Creates a new R-near neighbor data structure (PRNearNeighborStructT) + from the parameters and for the + data set . is the number of points in the data + set and is the dimension of the points. + + The set is a set with query sample points + (R-NN DS's parameters are optimized for query points from the set + ). could be a sample of points from the + actual query set or from the data set. When computing the estimated + number of collisions of a sample query point with the data set + points, if there is a point in the data set with the same pointer + with (that is when is a data set point), then the + corresponding point () is not considered in the data set (for the + purpose of computing the respective #collisions estimation). +*/ +PRNearNeighborStructT initSelfTunedRNearNeighborWithDataSet(RealT thresholdR, + RealT successProbability, + Int32T nPoints, + IntT dimension, + PPointT *dataSet, + IntT nSampleQueries, + PPointT *sampleQueries, + Int32T memoryUpperBound){ + initializeLSHGlobal(); + + PRNearNeighborStructT nnStruct = NULL; + + RNNParametersT optParameters = computeOptimalParameters(thresholdR, successProbability, nPoints, dimension, dataSet, nSampleQueries, sampleQueries, memoryUpperBound); + + if (!optParameters.useUfunctions) { + DPRINTF("Used L=%d\n", optParameters.parameterL); + }else{ + DPRINTF("Used m = %d\n", optParameters.parameterM); + DPRINTF("Used L = %d\n", optParameters.parameterL); + } + + TimeVarT timeInit = 0; + TIMEV_START(timeInit); + + // Init the R-NN data structure. + if (optParameters.typeHT != HT_HYBRID_CHAINS){ + nnStruct = initLSH(optParameters, nPoints); + }else{ + printRNNParameters(DEBUG_OUTPUT, optParameters); + nnStruct = initLSH_WithDataSet(optParameters, nPoints, dataSet); + } + + TIMEV_END(timeInit); + printf("Time for initializing: %0.6lf\n", timeInit); + DPRINTF("Allocated memory: %d\n", totalAllocatedMemory); + + TimeVarT timeAdding = 0; + if (optParameters.typeHT != HT_HYBRID_CHAINS){ + // Add the points to the LSH buckets. + TIMEV_START(timeAdding); + for(IntT i = 0; i < nPoints; i++){ + addNewPointToPRNearNeighborStruct(nnStruct, dataSet[i]); + } + TIMEV_END(timeAdding); + printf("Time for adding points: %0.6lf\n", timeAdding); + DPRINTF("Allocated memory: %d\n", totalAllocatedMemory); + } + + DPRINTF("Time for creating buckets: %0.6lf\n", timeBucketCreation); + DPRINTF("Time for putting buckets into UH: %0.6lf\n", timeBucketIntoUH); + DPRINTF("Time for computing GLSH: %0.6lf\n", timeComputeULSH); + DPRINTF("NGBuckets: %d\n", nGBuckets); + + return nnStruct; +} + +Int32T getRNearNeighbors(PRNearNeighborStructT nnStruct, PPointT queryPoint, PPointT *(&result), Int32T &resultSize){ + DPRINTF("Estimated ULSH comp: %0.6lf\n", lshPrecomp * nnStruct->nHFTuples * nnStruct->hfTuplesLength); + DPRINTF("Estimated UH overhead: %0.6lf\n", uhashOver * nnStruct->nHFTuples); +// RealT estNColls = estimateNCollisions(nnStruct->nPoints, +// nnStruct->dimension, +// nnStruct->points, +// queryPoint, +// nnStruct->parameterK, +// nnStruct->parameterL, +// nnStruct->parameterR); +// DPRINTF("Estimated #collisions (query specific): %0.6lf\n", (double)estNColls); +// estNColls = (double)estimateNDistinctCollisions(nnStruct->nPoints, +// nnStruct->dimension, +// nnStruct->points, +// queryPoint, +// nnStruct->useUfunctions, +// nnStruct->hfTuplesLength, +// nnStruct->nHFTuples, +// nnStruct->parameterR); +// DPRINTF("Estimated #distinct collisions (query specific): %0.6lf\n", estNColls); +// DPRINTF("Estimated Dist comp time (query specific): %0.6lf\n", distComp * estNColls); + + // reset all the timers + timeRNNQuery = 0; + timeComputeULSH = 0; + timeGetBucket = 0; + timeCycleBucket = 0; + timeDistanceComputation = 0; + timeResultStoring = 0; + timeCycleProc = 0; + timePrecomputeHash = 0; + timeGBHash = 0; + timeChainTraversal = 0; + nOfDistComps = 0; + timeTotalBuckets = 0; + + TIMEV_START(timeRNNQuery); + noExpensiveTiming = !DEBUG_PROFILE_TIMING; + Int32T nNearNeighbors = getNearNeighborsFromPRNearNeighborStruct(nnStruct, queryPoint, result, resultSize); + TIMEV_END(timeRNNQuery); + + DPRINTF("Time to compute LSH: %0.6lf\n", timeComputeULSH); + DPRINTF("Time to get bucket: %0.6lf\n", timeGetBucket); + DPRINTF("Time to cycle through buckets: %0.6lf\n", timeCycleBucket); + DPRINTF("Time to for processing buckets (UH+examining points): %0.6lf\n", timeTotalBuckets); + //DPRINTF("Time to copy ULSHs: %0.6lf\n", timeCopyingULSHs); + //DPRINTF("Time to unmark points: %0.6lf\n", timeUnmarking); + DPRINTF("Time for distance comps: %0.6lf\n", timeDistanceComputation); + DPRINTF("Time to store result: %0.6lf\n", timeResultStoring); + //printf("Time for cycle processing: %0.6lf\n", timeCycleProc); + //printf("Time for precomputing hashes: %0.6lf\n", timePrecomputeHash); + //printf("Time for GB hash: %0.6lf\n", timeGBHash); + //printf("Time for traversal of chains: %0.6lf\n", timeChainTraversal); + DPRINTF("Number of dist comps: %d\n", nOfDistComps); + DPRINTF("Number buckets in chains: %d\n", nBucketsInChains); + DPRINTF("Number buckets in chains / L: %0.3lf\n", (double)nBucketsInChains / nnStruct->nHFTuples); + DPRINTF("Cumulative time for R-NN query: %0.6lf\n", timeRNNQuery); + + return nNearNeighbors; +} diff --git a/src/lsh/sources/NearNeighbors.h b/src/lsh/sources/NearNeighbors.h new file mode 100755 index 0000000..6d9c7d0 --- /dev/null +++ b/src/lsh/sources/NearNeighbors.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef NEARNEIGHBORS_INCLUDED +#define NEARNEIGHBORS_INCLUDED + +void initializeLSHGlobal(); + +PRNearNeighborStructT initSelfTunedRNearNeighborWithDataSet(RealT thresholdR, RealT successProbability, Int32T nPoints, IntT dimension, PPointT *dataSet, IntT nSampleQueries, PPointT *sampleQueries, Int32T memoryUpperBound); + +Int32T getRNearNeighbors(PRNearNeighborStructT nnStruct, PPointT queryPoint, PPointT *(&result), Int32T &resultSize); + +#endif diff --git a/src/lsh/sources/Random.cpp b/src/lsh/sources/Random.cpp new file mode 100755 index 0000000..6617c7f --- /dev/null +++ b/src/lsh/sources/Random.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include "headers.h" + +// The state vector for generation of random numbers. +char rngState[256]; + +// Initialize the random number generator. +void initRandom(){ + FAILIF(NULL == initstate(2, rngState, 256)); +} + +// Generate a random integer in the range [rangeStart, +// rangeEnd]. Inputs must satisfy: rangeStart <= rangeEnd. +IntT genRandomInt(IntT rangeStart, IntT rangeEnd){ + ASSERT(rangeStart <= rangeEnd); + IntT r; + r = rangeStart + (IntT)((rangeEnd - rangeStart + 1.0) * random() / (RAND_MAX + 1.0)); + ASSERT(r >= rangeStart && r <= rangeEnd); + return r; +} + +// Generate a random 32-bits unsigned (Uns32T) in the range +// [rangeStart, rangeEnd]. Inputs must satisfy: rangeStart <= +// rangeEnd. +Uns32T genRandomUns32(Uns32T rangeStart, Uns32T rangeEnd){ + ASSERT(rangeStart <= rangeEnd); + Uns32T r; + if (RAND_MAX >= rangeEnd - rangeStart) { + r = rangeStart + (Uns32T)((rangeEnd - rangeStart + 1.0) * random() / (RAND_MAX + 1.0)); + } else { + r = rangeStart + (Uns32T)((rangeEnd - rangeStart + 1.0) * ((LongUns64T)random() * ((LongUns64T)RAND_MAX + 1) + (LongUns64T)random()) / ((LongUns64T)RAND_MAX * ((LongUns64T)RAND_MAX + 1) + (LongUns64T)RAND_MAX + 1.0)); + } + ASSERT(r >= rangeStart && r <= rangeEnd); + return r; +} + +// Generate a random real distributed uniformly in [rangeStart, +// rangeEnd]. Input must satisfy: rangeStart <= rangeEnd. The +// granularity of generated random reals is given by RAND_MAX. +RealT genUniformRandom(RealT rangeStart, RealT rangeEnd){ + ASSERT(rangeStart <= rangeEnd); + RealT r; + r = rangeStart + ((rangeEnd - rangeStart) * (RealT)random() / (RealT)RAND_MAX); + ASSERT(r >= rangeStart && r <= rangeEnd); + return r; +} + +// Generate a random real from normal distribution N(0,1). +RealT genGaussianRandom(){ + // Use Box-Muller transform to generate a point from normal + // distribution. + RealT x1, x2; + do{ + x1 = genUniformRandom(0.0, 1.0); + } while (x1 == 0); // cannot take log of 0. + x2 = genUniformRandom(0.0, 1.0); + RealT z; + z = SQRT(-2.0 * LOG(x1)) * COS(2.0 * M_PI * x2); + return z; +} diff --git a/src/lsh/sources/Random.h b/src/lsh/sources/Random.h new file mode 100755 index 0000000..a147e20 --- /dev/null +++ b/src/lsh/sources/Random.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef RANDOM_INCLUDED +#define RANDOM_INCLUDED + +void initRandom(); + +IntT genRandomInt(IntT rangeStart, IntT rangeEnd); + +Uns32T genRandomUns32(Uns32T rangeStart, Uns32T rangeEnd); + +RealT genUniformRandom(RealT rangeStart, RealT rangeEnd); + +RealT genGaussianRandom(); + +#endif diff --git a/src/lsh/sources/SelfTuning.cpp b/src/lsh/sources/SelfTuning.cpp new file mode 100755 index 0000000..34eec90 --- /dev/null +++ b/src/lsh/sources/SelfTuning.cpp @@ -0,0 +1,574 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +/* + The self-tuning module. This file contains all the functions for + estimating the running times and for computing the optimal (at least + in estimation) parameters for the R-NN data structure (within the + memory limits). + */ + +#include "headers.h" + +// Computes how much time it takes to run timing functions (functions +// that compute timings) -- we need to substract this value when we +// compute the length of an actual interval of time. +void tuneTimeFunctions(){ + timevSpeed = 0; + // Compute the time needed for a calls to TIMEV_START and TIMEV_END + IntT nIterations = 100000; + TimeVarT timeVar = 0; + for(IntT i = 0; i < nIterations; i++){ + TIMEV_START(timeVar); + TIMEV_END(timeVar); + } + timevSpeed = timeVar / nIterations; + DPRINTF("Tuning: timevSpeed = %0.9lf\n", timevSpeed); +} + +// Encapsulates a PPoint and a RealT in a single struct. +typedef struct _PPointAndRealTStructT { + PPointT ppoint; + RealT real; +} PPointAndRealTStructT; + +// Compares according to the field "real" of the struct. +int comparePPointAndRealTStructT(const void *a, const void *b){ + PPointAndRealTStructT *x = (PPointAndRealTStructT*)a; + PPointAndRealTStructT *y = (PPointAndRealTStructT*)b; + return (x->real > y->real) - (x->real < y->real); +} + + +/* + * Given a set of queries, the data set, and a set of (sorted) radii, + * this function will compute the , i.e., the indeces + * at which the query points go from one "radius class" to another + * "radius class". + * + * More formally, the query set is sorted according to the distance to + * NN. Then, function fills in such that if + * , then all query points with index =A) have their NN at distance >radii[i]. + * + * must be preallocated for at least + * elements. + */ +void sortQueryPointsByRadii(IntT dimension, + Int32T nQueries, + PPointT *queries, + Int32T nPoints, + PPointT *dataSet, + IntT nRadii, + RealT *radii, + Int32T *boundaryIndeces){ + ASSERT(queries != NULL); + ASSERT(dataSet != NULL); + ASSERT(radii != NULL); + ASSERT(boundaryIndeces != NULL); + + + PPointAndRealTStructT *distToNN = NULL; + FAILIF(NULL == (distToNN = (PPointAndRealTStructT*)MALLOC(nQueries * sizeof(*distToNN)))); + for(IntT i = 0; i < nQueries; i++){ + distToNN[i].ppoint = queries[i]; + distToNN[i].real = distance(dimension, queries[i], dataSet[0]); + for(IntT p = 0; p < nPoints; p++){ + RealT dist = distance(dimension, queries[i], dataSet[p]); + if (dist < distToNN[i].real){ + distToNN[i].real = dist; + } + } + } + + qsort(distToNN, nQueries, sizeof(*distToNN), comparePPointAndRealTStructT); + + IntT radiusIndex = 0; + for(IntT i = 0; i < nQueries; i++) { + //DPRINTF("%0.6lf\n", distToNN[i].real); + queries[i] = distToNN[i].ppoint; // copy the sorted queries array back to + while ((distToNN[i].real > radii[radiusIndex]) && (radiusIndex < nRadii)) { + boundaryIndeces[radiusIndex] = i; + radiusIndex++; + } + } + + FREE(distToNN); +} + +// Determines the run-time coefficients of the different parts of the +// query algorithm. Values that are computed and returned are +// , , . is the time for +// pre-computing one function from the LSH family. is the +// time for getting a bucket from a hash table (of buckets). +// is the time to compute one distance between two points. These times +// are computed by constructing a R-NN DS on a sample data set and +// running a sample query set on it. +void determineRTCoefficients(RealT thresholdR, + RealT successProbability, + BooleanT useUfunctions, + IntT typeHT, + IntT dimension, + Int32T nPoints, + PPointT *realData, + RealT &lshPrecomp, + RealT &uhashOver, + RealT &distComp){ + + // use a subset of the original data set. + // there is not much theory behind the formula below. + IntT n = nPoints / 50; + if (n < 100) { + n = nPoints; + } + if (n > 10000) { + n = 10000; + } + + // Initialize the data set to use. + PPointT *dataSet; + FAILIF(NULL == (dataSet = (PPointT*)MALLOC(n * sizeof(PPointT)))); + for(IntT i = 0; i < n; i++){ + dataSet[i] = realData[genRandomInt(0, nPoints - 1)]; + } + + IntT hashTableSize = n; + RNNParametersT algParameters; + // Pi: use a larger 'R' for parameter tuning, but the same 'R^2' for NNs: + if ( thresholdR < Pi_EPSILON ) + algParameters.parameterR = Pi_PSEUDO_R; + else + algParameters.parameterR = thresholdR; + algParameters.successProbability = successProbability; + algParameters.dimension = dimension; + algParameters.parameterR2 = SQR(thresholdR); + algParameters.useUfunctions = useUfunctions; + algParameters.parameterK = 16; + algParameters.parameterW = PARAMETER_W_DEFAULT; + algParameters.parameterT = n; + algParameters.typeHT = typeHT; + + if (algParameters.useUfunctions){ + algParameters.parameterM = computeMForULSH(algParameters.parameterK, algParameters.successProbability); + algParameters.parameterL = algParameters.parameterM * (algParameters.parameterM - 1) / 2; + }else{ + algParameters.parameterM = computeLfromKP(algParameters.parameterK, algParameters.successProbability); + algParameters.parameterL = algParameters.parameterM; + } + +// FAILIF(NULL == (dataSet = (PPointT*)MALLOC(n * sizeof(PPointT)))); +// for(IntT i = 0; i < n; i++){ +// FAILIF(NULL == (dataSet[i] = (PPointT)MALLOC(sizeof(PointT)))); +// FAILIF(NULL == (dataSet[i]->coordinates = (RealT*)MALLOC(dimension * sizeof(RealT)))); + +// dataSet[i]->index = i; +// sqrLength = 0; +// for(IntT d = 0; d < dimension; d++){ +// if (i == 0) { +// dataSet[i]->coordinates[d] = genUniformRandom(-100, 100); +// }else{ +// dataSet[i]->coordinates[d] = dataSet[0]->coordinates[d]; +// } +// sqrLength += SQR(dataSet[i]->coordinates[d]); +// } +// dataSet[i]->sqrLength = sqrLength; +// } + + // switch on timing + BooleanT tempTimingOn = timingOn; + timingOn = TRUE; + + // initialize result arrays + PPointT *result = NULL; + IntT resultSize = 0; + IntT nNNs; + IntT nSucReps; + + do{ + // create the test structure + PRNearNeighborStructT nnStruct; + switch(algParameters.typeHT){ + case HT_LINKED_LIST: + nnStruct = initLSH(algParameters, n); + // add points to the test structure + for(IntT i = 0; i < n; i++){ + addNewPointToPRNearNeighborStruct(nnStruct, realData[i]); + } + break; + case HT_HYBRID_CHAINS: + nnStruct = initLSH_WithDataSet(algParameters, n, dataSet); + break; + default: + ASSERT(FALSE); + } + + // query point + PPointT queryPoint; +// FAILIF(NULL == (queryPoint = (PPointT)MALLOC(sizeof(PointT)))); +// FAILIF(NULL == (queryPoint->coordinates = (RealT*)MALLOC(dimension * sizeof(RealT)))); +// RealT sqrLength = 0; +// for(IntT i = 0; i < dimension; i++){ +// queryPoint->coordinates[i] = dataSet[0]->coordinates[i]; +// //queryPoint->coordinates[i] = 0.1; +// sqrLength += SQR(queryPoint->coordinates[i]); +// } + //queryPoint->coordinates[0] = dataPoint->coordinates[0] + 0.0001; + //queryPoint->sqrLength = sqrLength; + + // reset the R parameter so that there are no NN neighbors. + setResultReporting(nnStruct, FALSE); + //DPRINTF1("X\n"); + + lshPrecomp = 0; + uhashOver = 0; + distComp = 0; + IntT nReps = 20; + nSucReps = 0; + for(IntT rep = 0; rep < nReps; rep++){ + queryPoint = realData[genRandomInt(0, nPoints - 1)]; + timeComputeULSH = 0; + timeGetBucket = 0; + timeCycleBucket = 0; + nOfDistComps = 0; + nNNs = getNearNeighborsFromPRNearNeighborStruct(nnStruct, queryPoint, result, resultSize); + //DPRINTF("Time to compute LSH: %0.6lf\n", timeComputeULSH); + //DPRINTF("Time to get bucket: %0.6lf\n", timeGetBucket); + //DPRINTF("Time to cycle through buckets: %0.9lf\n", timeCycleBucket); + //DPRINTF("N of dist comp: %d\n", nOfDistComps); + + ASSERT(nNNs == 0); + if (nOfDistComps >= MIN(n / 10, 100)){ + nSucReps++; + lshPrecomp += timeComputeULSH / algParameters.parameterK / algParameters.parameterM; + uhashOver += timeGetBucket / algParameters.parameterL; + distComp += timeCycleBucket / nOfDistComps; + } + } + + if (nSucReps >= 5){ + lshPrecomp /= nSucReps; + uhashOver /= nSucReps; + distComp /= nSucReps; + DPRINTF1("RT coeffs computed.\n"); + }else{ + // Pi: + if ( algParameters.parameterR < Pi_EPSILON ) { + if ( nSucReps==0 ) { + // Pi: try to reset 'R' to a normal value to continue. It's + // already faster than R==0.0, but Pi_EPSILON may be still + // too small to make it fast. So, maybe better to just stop + // tuning because it's no good to compute all those things + // when R==0 anyway...but #collision increases exponentially + // when #points increases... + + // algParameters.parameterR = Pi_EPSILON; + freePRNearNeighborStruct(nnStruct); + break; // simply stop tuning. + } else { + lshPrecomp /= nSucReps; + uhashOver /= nSucReps; + distComp /= nSucReps; + freePRNearNeighborStruct(nnStruct); + break; // simply stop tuning. + } + } else + algParameters.parameterR *= 2; // double the radius and repeat + DPRINTF1("Could not determine the RT coeffs. Repeating.\n"); + } + + freePRNearNeighborStruct(nnStruct); + + }while(nSucReps < 5); + + FREE(dataSet); + FREE(result); + + timingOn = tempTimingOn; +} + +/* + The function

from the paper (probability of collision of 2 + points for 1 LSH function). + */ +RealT computeFunctionP(RealT w, RealT c){ + // Pi: + if ( c < Pi_EPSILON ) // c is close to zero; x->inf; then: + // assume erfc(inf)->0, the second part->0 + return 1.; // c->0 means the points should always conflict. + else { + RealT x = w / c; + return 1 - ERFC(x / M_SQRT2) - M_2_SQRTPI / M_SQRT2 / x * (1 - EXP(-SQR(x) / 2)); + } +} + +// Computes the parameter of the algorithm, given the parameter +// and the desired success probability +// . Functions are considered all independent +// (original scheme). +IntT computeLfromKP(IntT k, RealT successProbability){ + return CEIL(LOG(1 - successProbability) / LOG(1 - POW(computeFunctionP(PARAMETER_W_DEFAULT, 1), k))); +} + +// Computes the parameter of the algorithm, given the parameter +// and the desired success probability . Only +// meaningful when functions are interdependent (pairs of +// functions , where the functions are each k/2-tuples of +// independent LSH functions). +IntT computeMForULSH(IntT k, RealT successProbability){ + ASSERT((k & 1) == 0); // k should be even in order to use ULSH. + RealT mu = 1 - POW(computeFunctionP(PARAMETER_W_DEFAULT, 1), k / 2); + RealT P = successProbability; + RealT d = (1-mu)/(1-P)*1/LOG(1/mu) * POW(mu, -1/(1-mu)); + RealT y = LOG(d); + IntT m = CEIL(1 - y/LOG(mu) - 1/(1-mu)); + while (POW(mu, m-1) * (1 + m * (1-mu)) > 1 - P){ + m++; + } + return m; +} +RealT estimateNCollisions(IntT nPoints, IntT dim, PPointT *dataSet, PPointT query, IntT k, IntT L, RealT R){ + RealT sumCollisions = 0; + for(IntT i = 0; i < nPoints; i++){ + if (query != dataSet[i]) { + RealT dist = distance(dim, query, dataSet[i]); + // Pi: + if ( R < Pi_EPSILON ) + sumCollisions += POW(computeFunctionP(PARAMETER_W_DEFAULT, dist), k); + else + sumCollisions += POW(computeFunctionP(PARAMETER_W_DEFAULT, dist / R), k); + } + } + return L * sumCollisions; +} + +RealT estimateNCollisionsFromDSPoint(IntT nPoints, IntT dim, PPointT *dataSet, IntT queryIndex, IntT k, IntT L, RealT R){ + RealT sumCollisions = 0; + for(IntT i = 0; i < nPoints; i++){ + if (queryIndex != i) { + RealT dist = distance(dim, dataSet[queryIndex], dataSet[i]); + // Pi: + if ( R < Pi_EPSILON ) + sumCollisions += POW(computeFunctionP(PARAMETER_W_DEFAULT, dist), k); + else + sumCollisions += POW(computeFunctionP(PARAMETER_W_DEFAULT, dist / R), k); + } + } + return L * sumCollisions; +} + +RealT estimateNDistinctCollisions(IntT nPoints, IntT dim, PPointT *dataSet, PPointT query, BooleanT useUfunctions, IntT k, IntT LorM, RealT R){ + RealT sumCollisions = 0; + for(IntT i = 0; i < nPoints; i++){ + if (query != dataSet[i]) { + RealT dist = distance(dim, query, dataSet[i]); + if (!useUfunctions){ + // Pi: + if ( R < Pi_EPSILON ) + sumCollisions += 1-POW(1-POW(computeFunctionP(PARAMETER_W_DEFAULT, dist), k), LorM); + else + sumCollisions += 1-POW(1-POW(computeFunctionP(PARAMETER_W_DEFAULT, dist / R), k), LorM); + }else{ + RealT mu; + RealT x; + // Pi: + if ( R < Pi_EPSILON ) + mu = 1 - POW(computeFunctionP(PARAMETER_W_DEFAULT, dist), k / 2); + else + mu = 1 - POW(computeFunctionP(PARAMETER_W_DEFAULT, dist / R), k / 2); + x = POW(mu, LorM - 1); + sumCollisions += 1 - mu * x - LorM * (1 - mu) * x; + } + } + } + return sumCollisions; +} + +RealT estimateNDistinctCollisionsFromDSPoint(IntT nPoints, IntT dim, PPointT *dataSet, IntT queryIndex, BooleanT useUfunctions, IntT k, IntT LorM, RealT R){ + RealT sumCollisions = 0; + for(IntT i = 0; i < nPoints; i++){ + if (queryIndex != i) { + RealT dist = distance(dim, dataSet[queryIndex], dataSet[i]); + if (!useUfunctions){ + // Pi: + if ( R < Pi_EPSILON ) + sumCollisions += 1-POW(1-POW(computeFunctionP(PARAMETER_W_DEFAULT, dist), k), LorM); + else + sumCollisions += 1-POW(1-POW(computeFunctionP(PARAMETER_W_DEFAULT, dist / R), k), LorM); + }else{ + RealT mu; + RealT x; + // Pi: + if ( R < Pi_EPSILON ) + mu = 1 - POW(computeFunctionP(PARAMETER_W_DEFAULT, dist), k / 2); + else + mu = 1 - POW(computeFunctionP(PARAMETER_W_DEFAULT, dist / R), k / 2); + x = POW(mu, LorM - 1); + sumCollisions += 1 - mu * x - LorM * (1 - mu) * x; + } + } + } + return sumCollisions; +} + +/* + Given the actual data set , estimates the values for + algorithm parameters that would give the optimal running time of a + query. + + The set is a set with query sample points + (R-NN DS's parameters are optimized for query points from the set + ). could be a sample of points from the + actual query set or from the data set. When computing the estimated + number of collisions of a sample query point with the data set + points, if there is a point in the data set with the same pointer + with (that is when is a data set point), then the + corresponding point () is not considered in the data set (for the + purpose of computing the respective #collisions estimation). + + The return value is the estimate of the optimal parameters. +*/ +RNNParametersT computeOptimalParameters(RealT R, + RealT successProbability, + IntT nPoints, + IntT dimension, + PPointT *dataSet, + IntT nSampleQueries, + PPointT *sampleQueries, + Int32T memoryUpperBound){ + ASSERT(nSampleQueries > 0); + + initializeLSHGlobal(); + + RNNParametersT optParameters; + optParameters.successProbability = successProbability; + optParameters.dimension = dimension; + // Pi: + if ( R < Pi_EPSILON ) + optParameters.parameterR = Pi_PSEUDO_R; + else + optParameters.parameterR = R; + optParameters.parameterR2 = SQR(R); + optParameters.useUfunctions = TRUE; // TODO: could optimize here: + // maybe sometimes, the old way + // was better. + optParameters.parameterW = PARAMETER_W_DEFAULT; + optParameters.parameterT = nPoints; + optParameters.typeHT = HT_HYBRID_CHAINS; + + // Compute the run-time parameters (timings of different parts of the algorithm). + IntT nReps = 10; // # number of repetions + RealT lshPrecomp = 0, uhashOver = 0, distComp = 0; + for(IntT i = 0; i < nReps; i++){ + RealT lP, uO, dC; + determineRTCoefficients(optParameters.parameterR, + optParameters.successProbability, + optParameters.useUfunctions, + optParameters.typeHT, + optParameters.dimension, + nPoints, + dataSet, + lP, + uO, + dC); + lshPrecomp += lP; + uhashOver += uO; + distComp += dC; + DPRINTF4("Coefs: lP = %0.9lf\tuO = %0.9lf\tdC = %0.9lf\n", lP, uO, dC); + } + lshPrecomp /= nReps; + uhashOver /= nReps; + distComp /= nReps; + DPRINTF("Coefs (final): lshPrecomp = %0.9lf\n", lshPrecomp); + DPRINTF("Coefs (final): uhashOver = %0.9lf\n", uhashOver); + DPRINTF("Coefs (final): distComp = %0.9lf\n", distComp); + DPRINTF("Remaining memory: %d\n", memoryUpperBound); + + // Try all possible s and choose the one for which the time + // estimate of a query is minimal. + IntT k; + RealT timeLSH, timeUH, timeCycling; + //IntT queryIndex = genRandomInt(0, nPoints); + //PPointT query = dataSet[queryIndex]; // query points = a random points from the data set. + IntT bestK = 0; + RealT bestTime = 0; + for(k = 2; ; k += 2){ + + DPRINTF("ST. k = %d\n", k); + IntT m = computeMForULSH(k, successProbability); + IntT L = m * (m-1) / 2; + //DPRINTF("Available memory: %ld\n", getAvailableMemory()); + if (L * nPoints > memoryUpperBound / 12){ + break; + } + timeLSH = m * k * lshPrecomp; + timeUH = L * uhashOver; + //RealT nCollisions = estimateNCollisionsFromDSPoint(nPoints, dimension, dataSet, queryIndex, k, L, R); + + // Compute the mean number of collisions for the points from the sample query set. + RealT nCollisions = 0; + for(IntT i = 0; i < nSampleQueries; i++){ + nCollisions += estimateNDistinctCollisions(nPoints, dimension, dataSet, sampleQueries[i], TRUE, k, m, R); // Pi: should use "optParameters.parameterR" instead of 'R' here? Maybe not, because we should measure the #collisions for real 'R' (the formula for computing collision probability may not be appropriate any more, though. Anyway, it's an approximation.) + } + nCollisions /= nSampleQueries; + + timeCycling = nCollisions * distComp; + DPRINTF3("ST.m=%d L=%d \n", m, L); + DPRINTF("ST.Estimated # distinct collisions = %0.6lf\n", (double)nCollisions); + DPRINTF("ST.TimeLSH = %0.6lf\n", timeLSH); + DPRINTF("ST.TimeUH = %0.6lf\n", timeUH); + DPRINTF("ST.TimeCycling = %0.6lf\n", timeCycling); + DPRINTF("ST.Sum = %0.6lf\n", timeLSH + timeUH + timeCycling); + if (bestK == 0 || (timeLSH + timeUH + timeCycling) < bestTime) { + bestK = k; + bestTime = timeLSH + timeUH + timeCycling; + } + ASSERT(k < 100); // otherwise, k reached 100 -- which, from + // experience, should never happen for reasonable + // data set & available memory amount. + } + + + DPRINTF("STO.Optimal k = %d\n", bestK); + IntT m = computeMForULSH(bestK, successProbability); + IntT L = m * (m-1) / 2; + timeLSH = m * bestK * lshPrecomp; + timeUH = L * uhashOver; + + // Compute the mean number of collisions for the points from the sample query set. + RealT nCollisions = 0; + for(IntT i = 0; i < nSampleQueries; i++){ + nCollisions += estimateNDistinctCollisions(nPoints, dimension, dataSet, sampleQueries[i], TRUE, k, m, R); + } + nCollisions /= nSampleQueries; + + // timeCycling = estimateNCollisionsFromDSPoint(nPoints, dimension, dataSet, queryIndex, bestK, L, R) * distComp; + timeCycling = nCollisions * distComp; + DPRINTF("STO.TimeLSH = %0.6lf\n", timeLSH); + DPRINTF("STO.TimeUH = %0.6lf\n", timeUH); + DPRINTF("STO.TimeCycling = %0.6lf\n", timeCycling); + DPRINTF("STO.Sum = %0.6lf\n", timeLSH + timeUH + timeCycling); + + optParameters.parameterK = bestK; + optParameters.parameterM = m; + optParameters.parameterL = L; + + // Pi: restore the original 'R' + if ( R < Pi_EPSILON ) + optParameters.parameterR = R; + return optParameters; +} diff --git a/src/lsh/sources/SelfTuning.h b/src/lsh/sources/SelfTuning.h new file mode 100755 index 0000000..6df7611 --- /dev/null +++ b/src/lsh/sources/SelfTuning.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef SELFTUNING_INCLUDED +#define SELFTUNING_INCLUDED + +void tuneTimeFunctions(); + +void sortQueryPointsByRadii(IntT dimension, + Int32T nQueries, + PPointT *queries, + Int32T nPoints, + PPointT *dataSet, + IntT nRadii, + RealT *radii, + Int32T *boundaryIndeces); + +void determineRTCoefficients(RealT thresholdR, RealT successProbability, BooleanT useUfunctions, IntT typeHT, IntT dimension, Int32T nPoints, PPointT *realData, RealT &lshPrecomp, RealT &uhashOver, RealT &distComp); + +RealT estimateNCollisions(IntT nPoints, IntT dim, PPointT *dataSet, PPointT query, IntT k, IntT L, RealT R); + +RealT estimateNDistinctCollisions(IntT nPoints, IntT dim, PPointT *dataSet, PPointT query, BooleanT useUfunctions, IntT hfTuplesLength, IntT nHFTuples, RealT R); + +RealT computeFunctionP(RealT w, RealT c); + +IntT computeLfromKP(IntT k, RealT successProbability); + +IntT computeMForULSH(IntT k, RealT successProbability); + +RNNParametersT computeOptimalParameters(RealT R, RealT successProbability, IntT nPoints, IntT dimension, PPointT *dataSet, IntT nSampleQueries, PPointT *sampleQueries, Int32T memoryUpperBound); + + +#endif diff --git a/src/lsh/sources/Util.cpp b/src/lsh/sources/Util.cpp new file mode 100755 index 0000000..033c571 --- /dev/null +++ b/src/lsh/sources/Util.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include "headers.h" + +// Verifies whether vector v1 and v2 are equal (component-wise). The +// size of the vectors is given by the parameter size. +BooleanT vectorsEqual(IntT size, IntT *v1, IntT *v2){ + for(IntT i = 0; i < size; i++){ + if (v1[i] != v2[i]){ + return FALSE; + } + } + return TRUE; +} + +// Copies the vector to the vector . The size of the +// vectors is given by . +void copyVector(IntT size, IntT *from, IntT *to){ + for(IntT i = 0; i < size; i++){ + to[i] = from[i]; + } +} + +// Creates a new vector of size and copies the vector to +// the new vector. +IntT *copyOfVector(IntT size, IntT *from){ + IntT *newVector; + FAILIF(NULL == (newVector = (IntT*)MALLOC(size * sizeof(IntT)))); + for(IntT i = 0; i < size; i++){ + newVector[i] = from[i]; + } + return newVector; +} + +// Prints the vector of size . The string appears +// in front. +void printRealVector(char *s, IntT size, RealT *v){ + ASSERT(v != NULL); + + printf("%s", s); + for(IntT i = 0; i < size; i++){ + if (i > 0){ + printf(" "); + } + printf("%0.2lf", (double)v[i]); + } + + printf("\n"); +} + +// Prints the vector of size . The string appears +// in front. +void printIntVector(char *s, IntT size, IntT *v){ + ASSERT(v != NULL); + + printf("%s", s); + for(IntT i = 0; i < size; i++){ + if (i > 0){ + printf(" "); + } + printf("%d", v[i]); + } + + printf("\n"); +} + +// Returns the amount of available memory. +Uns32T getAvailableMemory(){ + // TODO + //printf("mem=%lu\n", MEMORY_MAX_AVAILABLE - totalAllocatedMemory); + FAILIFWR(availableTotalMemory < totalAllocatedMemory, "Not enough memory.\n"); + return availableTotalMemory - totalAllocatedMemory; +} diff --git a/src/lsh/sources/Util.h b/src/lsh/sources/Util.h new file mode 100755 index 0000000..bebc998 --- /dev/null +++ b/src/lsh/sources/Util.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#ifndef UTIL_INCLUDED +#define UTIL_INCLUDED + +#include "BasicDefinitions.h" + +BooleanT vectorsEqual(IntT size, IntT *v1, IntT *v2); + +void copyVector(IntT size, IntT *from, IntT *to); + +IntT *copyOfVector(IntT size, IntT *from); + +void printRealVector(char *s, IntT size, RealT *v); + +void printIntVector(char *s, IntT size, IntT *v); + +Uns32T getAvailableMemory(); + +#endif diff --git a/src/lsh/sources/compareOutputs.cpp b/src/lsh/sources/compareOutputs.cpp new file mode 100755 index 0000000..83792bf --- /dev/null +++ b/src/lsh/sources/compareOutputs.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include +#include + +#include "headers.h" + +void usage(char *programName){ + printf("Usage: %s correct_output lsh_output\n", programName); +} + +IntT main(IntT nargs, char **args){ + if (nargs < 2){ + usage(args[0]); + exit(1); + } + + FILE *fCorrect = fopen(args[1], "rt"); + FAILIF(fCorrect == NULL); + FILE *fLSH = fopen(args[2], "rt"); + FAILIF(fLSH == NULL); + + IntT nTotalCorrect = 0; + IntT nTotalLSH = 0; + IntT overallOK = 1; + + IntT nPoints; + fscanf(fCorrect, "nPoints = %d\n", &nPoints); + IntT *pointsNN = (int*)calloc(nPoints, sizeof(int)); + IntT nCorrect = 0; + char s[1000]; + char c; + IntT x; + IntT nQuery = 0; + c = ' '; + while(!feof(fCorrect)){ + // find the start of the query (letter Q). + while (!feof(fCorrect) && (c != 'Q')) { + fscanf(fCorrect, "%c", &c); + if (c != '0') { + fscanf(fCorrect, "%[^\n]", s); + fscanf(fCorrect, "\n"); + } + }; + + //printf("%c\n", c); + + if (!feof(fCorrect)){ + do { + fscanf(fCorrect, "%c", &c); + if (c != '0') { + fscanf(fCorrect, "%[^\n]", s); + fscanf(fCorrect, "\n"); + }else{ + fscanf(fCorrect, "%d\n", &x); + pointsNN[x] = 3 * nQuery + 1; + nCorrect++; + nTotalCorrect++; + } + } while (!feof(fCorrect) && (c == '0')); + } + + IntT ok = 1; + IntT nLSH = 0; + char g = ' '; + while(!feof(fLSH) && (g != 'Q')){ + fscanf(fLSH, "%c", &g); + if (g != '0') { + fscanf(fLSH, "%[^\n]", s); + fscanf(fLSH, "\n"); + } + } + + if (!feof(fLSH)){ + do { + fscanf(fLSH, "%c", &g); + if (g != '0') { + fscanf(fLSH, "%[^\n]", s); + fscanf(fLSH, "\n"); + }else{ + fscanf(fLSH, "%d\n", &x); + if (pointsNN[x] >= 3 * nQuery + 2 ){ + // LSH reported a poIntT more than once. + ok = 0; + printf("Error: LSH reported a poIntT (%d) more than once.\n", x); + } + if (pointsNN[x] <= 3 * nQuery){ + // LSH reported a poIntT that is not reported by exact NN. + ok = 0; + printf("Error: LSH reported a poIntT (%d) that was not reported by NN.\n", x); + } + pointsNN[x] = 3 * nQuery + 2; + nLSH++; + nTotalLSH++; + } + } while (!feof(fLSH) && (g == '0')); + } + + printf("OK = %d. NN_LSH/NN_Correct = %d/%d=%0.3lf\n", ok, nLSH, nCorrect, (nCorrect > 0)?((double)nLSH/(double)nCorrect):1); + if (ok == 0){ + overallOK = 0; + } + nCorrect = 0; + nQuery++; + } + + printf("\nOverall: OK = %d. NN_LSH/NN_Correct = %d/%d=%0.3lf\n", overallOK, nTotalLSH, nTotalCorrect, (nTotalCorrect > 0)?((double)nTotalLSH/(double)nTotalCorrect):1); + + fclose(fCorrect); + fclose(fLSH); +} diff --git a/src/lsh/sources/enumBuckets.cpp b/src/lsh/sources/enumBuckets.cpp new file mode 100755 index 0000000..62de45a --- /dev/null +++ b/src/lsh/sources/enumBuckets.cpp @@ -0,0 +1,733 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + * Modified by: Stephane Glondu (stephane.glondu@dptinfo.ens-cachan.fr) + */ + +/* + The main entry file containing the main() function. The main() + function parses the command line parameters and depending on them + calls the corresponding functions. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "headers.h" + +#define N_SAMPLE_QUERY_POINTS 100 + +// The data set containing all the points. +PPointT *dataSetPoints = NULL; +// Number of points in the data set. +IntT nPoints = 0; +// The dimension of the points. +IntT pointsDimension = 0; +// The value of parameter R (a near neighbor of a point is any +// point

from the data set that is the within distance +// ). +//RealT thresholdR = 1.0; + +// The succes probability of each point (each near neighbor is +// reported by the algorithm with probability ). +RealT successProbability = 0.9; + +// Same as , only an array of R's (for the case when +// multiple R's are specified). +RealT *listOfRadii = NULL; +IntT nRadii = 0; + +RealT *memRatiosForNNStructs = NULL; + +char sBuffer[600000]; +regex_t preg[ENUM_PPROP_LAST_NOT_USED]; + +// Will merge prefetch consecutive vectors from input file +int prefetch = 1; + +// The second heuristic for detecting bugs. +// return true when there are inconsistent changes. +bool inconsistentIDchanges(char *v1, char *v2) +{ + regex_t id; + regmatch_t pmatch[1]; + //char *id1, *id2; + int nChanged=0, nUnchanged=0; + + if ( v1==v2 ) + return false; + else if ( v1==NULL || v2==NULL ) + return true; + + FAILIF(0 != regcomp(&id, "([^,}[:blank:]]+)", REG_EXTENDED)); + + int a, b, c, d; + while ( regexec(&id, v1, 1, pmatch, 0) == 0 && + (a=pmatch[0].rm_so) != -1 ) { + char t1 = v1[(b=pmatch[0].rm_eo)]; + v1[b] = '\0'; + if ( regexec(&id, v2, 1, pmatch, 0) == 0 && + (c=pmatch[0].rm_so) != -1 ) { + char t2 = v2[(d=pmatch[0].rm_eo)]; + v2[d] = '\0'; + if ( strcmp(v1, v2)==0 ) + nUnchanged++; + else + nChanged++; + v2[d] = t2; + v1[b] = t1; + v2 += d; + v1 += b; + } else { + nChanged++; + v1[b] = t1; + v1 += b; + break; + } + } + while ( regexec(&id, v1, 1, pmatch, 0) == 0 && + (a=pmatch[0].rm_so) != -1 ) { + nChanged++; + v1 += pmatch[0].rm_eo; + } + + while ( regexec(&id, v2, 1, pmatch, 0) == 0 && + (c=pmatch[0].rm_so) != -1 ) { + nChanged++; + v2 += pmatch[0].rm_eo; + } + + // need quantified condition here. + + return false; +} + +/* + Prints the usage of the LSHMain. + */ +void usage(int code, char *programName) { + printf("Usage: %s [options: see source code] data_set_file [params_file]\n", programName); + exit(code); +} + +inline PPointT readPoint2(char *line, char *comment){ + + PPointT p; + RealT sqrLength = 0; + FAILIF(NULL == (p = (PPointT)MALLOC(sizeof(PointT)))); + FAILIF(NULL == (p->coordinates = (RealT*)MALLOC(pointsDimension * sizeof(RealT)))); + IntT d; + char *t; + + if (comment != NULL) { + int a, b; + regmatch_t pmatch[2]; + + if (regexec(&preg[ENUM_PPROP_FILE], comment, 2, pmatch, 0) == 0 && + (a = pmatch[1].rm_so) != -1) { + b = pmatch[1].rm_eo; + FAILIF(NULL == (p->filename = (char*)MALLOC(b-a+1))); + memmove(p->filename, comment + a, b-a); + p->filename[b-a] = '\0'; + } + + for (int i = 1; i < ENUM_PPROP_LAST_NOT_USED; i++) { + if (regexec(&preg[i], comment, 2, pmatch, 0) == 0 && + (a = pmatch[1].rm_so) != -1) { + b = pmatch[1].rm_eo; + char t = comment[b]; + comment[b] = '\0'; + p->prop[i-1] = atoi(comment + a); + comment[b] = t; + if ( i==ENUM_PPROP_OIDs ) { + // memeory bottleneck now +// int c = pmatch[0].rm_so, d = pmatch[0].rm_eo; +// FAILIF(NULL == (p->oids = (char*)MALLOC(d-c+1))); +// memmove(p->oids, comment + c, d-c); +// p->oids[d-c] = '\0'; + p->oids = NULL; + } + } else { + p->prop[i-1] = 0; + if ( i==ENUM_PPROP_OIDs ) + p->oids = NULL; + } + } + + p->prop[ENUM_PPROP_OFFSET-1] = p->prop[ENUM_PPROP_OFFSET-1] - p->prop[ENUM_PPROP_LINE-1] +1; // the line range. + } + + for (d = 0, t = line; *t != '\0' && d < pointsDimension; d++) { + while ( !isdigit(*t) && *t != '\0' && *t != '.') t++; + p->coordinates[d] = strtof(t, &t); // TOFIX: certain versions of gcc have bugs for strtof. + sqrLength += SQR(p->coordinates[d]); + } + + p->index = -1; + p->sqrLength = sqrLength; + return p; +} + +// Linked list structure for PPointT +typedef struct TPPointTList_s TPPointTList; +struct TPPointTList_s { + PPointT hd; + TPPointTList *tl; +}; + +// Reads in the data set points from in the array +// . Each point get a unique number in the field +// to be easily identifiable. +void readDataSetFromFile2(char *filename) +{ + int prefetchSize = 0; + nPoints = 0; + TPPointTList *prefetchStart; // beginning of prefetch queue + TPPointTList *prefetchEnd; // new cell at the end of prefetch queue + TPPointTList *pointsStart; // beginnning of point list + FAILIF(NULL == (prefetchEnd = (TPPointTList*)MALLOC(sizeof(TPPointTList)))); + prefetchEnd->tl = NULL; + pointsStart = prefetchStart = prefetchEnd; + + FILE *f = fopen(filename, "rt"); + FAILIF(f == NULL); + + char *line = NULL, *comment = NULL; + size_t bufferLength = 0; + ssize_t lineLength; + + while ((lineLength = getline(&line, &bufferLength, f)) > 0) { + if (line[0] == ';' ) { + // such a line is no use and could cause problems for the code + // which auto-detects the number of points now. -Pi 10/30/05. TODO + fprintf(stderr, "Warning: no fully-supported format around line %d\n", nPoints*2); + continue; + } else if (line[0] == '#') { + // the line is a comment + if (comment != NULL) free(comment); + comment = line; + if (comment[lineLength-1] == '\n') comment[lineLength-1] = '\0'; + line = NULL; bufferLength = 0; + } else { + // the line is a point + if (pointsDimension == 0) { + // compute the dimension + char *p = line; + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + while (*p != '\0') { + while (*p != ' ' && *p!='\t' && *p!='\r' && *p!='\n' && *p != '\0') p++; + pointsDimension++; + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + } + } + + // add the new point to the prefetch queue + prefetchEnd->hd = readPoint2(line, comment); + + if (prefetchSize < prefetch) prefetchSize++; + if (prefetchSize == prefetch) { + nPoints++; + // sum up list prefetch into the prefetchStart->hd. + // Performance problem here - Pi + for (TPPointTList *i = prefetchStart->tl; i != NULL; i = i->tl) { + for (int j = 0; j < pointsDimension; j++) { + prefetchStart->hd->coordinates[j] += i->hd->coordinates[j]; + } + prefetchStart->hd->prop[ENUM_PPROP_FILE-1] += i->hd->prop[ENUM_PPROP_FILE-1]; + } + // allocate the next cell and move the prefetch window + FAILIF(NULL == (prefetchEnd->tl = (TPPointTList*)MALLOC(sizeof(TPPointTList)))); + prefetchEnd = prefetchEnd->tl; + prefetchEnd->tl = NULL; + prefetchStart = prefetchStart->tl; + } else { + ASSERT(prefetchSize < prefetch); + // allocate the next cell + FAILIF(NULL == (prefetchEnd->tl = (TPPointTList*)MALLOC(sizeof(TPPointTList)))); + prefetchEnd = prefetchEnd->tl; + prefetchEnd->tl = NULL; + } + } // end of new point handling + } // end of file + fclose(f); + + // put the points in the array and free the point list + FAILIF(NULL == (dataSetPoints = (PPointT*)MALLOC(nPoints * sizeof(PPointT)))); + for(IntT i = 0; i < nPoints; i++) { + ASSERT(pointsStart != NULL); + dataSetPoints[i] = pointsStart->hd; + dataSetPoints[i]->index = i; + TPPointTList *cur = pointsStart->tl; + free(pointsStart); + pointsStart = cur; + } // nPoints == the number of total points - prefetch + 1 + if (comment != NULL) free(comment); + + // free the remaining prefetch queue + while (pointsStart != NULL) { + TPPointTList *cur = pointsStart->tl; + free(pointsStart); + pointsStart = cur; + } +} + + +// Tranforming from +// to +// . +void transformMemRatios(){ + RealT sum = 0; + for(IntT i = nRadii - 1; i >= 0; i--){ + sum += memRatiosForNNStructs[i]; + memRatiosForNNStructs[i] = memRatiosForNNStructs[i] / sum; + //DPRINTF("%0.6lf\n", memRatiosForNNStructs[i]); + } + ASSERT(sum <= 1.000001); +} + + +int compareInt32T(const void *a, const void *b){ + Int32T *x = (Int32T*)a; + Int32T *y = (Int32T*)b; + return (*x > *y) - (*x < *y); +} + + +#define ENUM_BUCKETS + +#define CHECK_INT(v) { \ + if (v <= 0) { \ + fprintf(stderr, "Incorrect integer value for variable %s\n", #v); \ + usage(1, argv[0]); \ + }} +#define CHECK_FLOAT(v) { \ + if (v < 1e-3) { \ + fprintf(stderr, "Incorrect float value for variable %s\n", #v); \ + usage(1, argv[0]); \ + }} + + +RNNParametersT *algParameters = NULL; +PRNearNeighborStructT *nnStructs = NULL; + +bool readParamsFile(char *paramsFile) +{ + FILE *pFile = fopen(paramsFile, "rt"); + if (pFile == NULL) { + fprintf(stderr, + "Warning: could not open %s, will try to compute parameters " + "and write them to that file\n", paramsFile); + return true; + } else { + fscanf(pFile, "%d\n", &nRadii); + fprintf(stderr, "Using the following R-NN DS parameters (from %s):\n", paramsFile); + fprintf(stderr, "N radii = %d, nPoints = %d\n", nRadii, nPoints); + FAILIF(NULL == (nnStructs = (PRNearNeighborStructT*)MALLOC(nRadii * sizeof(PRNearNeighborStructT)))); + FAILIF(NULL == (algParameters = (RNNParametersT*)MALLOC(nRadii * sizeof(RNNParametersT)))); + for(IntT i = 0; i < nRadii; i++){ + algParameters[i] = readRNNParameters(pFile); + printRNNParameters(stderr, algParameters[i]); + nnStructs[i] = initLSH_WithDataSet(algParameters[i], nPoints, dataSetPoints); + } + + pointsDimension = algParameters[0].dimension; + if (listOfRadii != NULL) FREE(listOfRadii); + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + for(IntT i = 0; i < nRadii; i++){ + listOfRadii[i] = algParameters[i].parameterR; + } + return false; + } +} + + +#define pointIsNotFiltered(p) ( \ + (*(p))->prop[ENUM_PPROP_NUM_NODE-1] >= minNumNodes && \ + (*(p))->prop[ENUM_PPROP_nVARs-1] >= min_nVars && \ + (*(p))->prop[ENUM_PPROP_OFFSET-1] >= min_lines ) + + +int comparePoints(const void *p1, const void *p2) +{ + PPointT a = *(PPointT*)p1; + PPointT b = *(PPointT*)p2; + int c = strcmp(a->filename, b->filename); + if (c) + return c; + else if ( a->prop[ENUM_PPROP_nVARs-1] != b->prop[ENUM_PPROP_nVARs-1] ) + return a->prop[ENUM_PPROP_nVARs-1] - b->prop[ENUM_PPROP_nVARs-1]; + else + return a->prop[ENUM_PPROP_LINE-1] - b->prop[ENUM_PPROP_LINE-1]; +} + + +/* + The main entry to LSH package. Depending on the command line + parameters, the function computes the R-NN data structure optimal + parameters and/or construct the R-NN data structure and runs the + queries on the data structure. + */ +int main(int argc, char *argv[]){ + + FAILIF(0 != regcomp(&preg[ENUM_PPROP_FILE], "FILE:([^,]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_LINE], "LINE:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_OFFSET], "OFFSET:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_NODE_KIND], "NODE_KIND:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_NUM_NODE], "NUM_NODE:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_NUM_DECL], "NUM_DECL:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_NUM_STMT], "NUM_STMT:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_NUM_EXPR], "NUM_EXPR:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_TBID], "TBID:([-]?[0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_TEID], "TEID:([-]?[0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_nVARs], "VARs:\\{[^}]*\\}([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_CONTEXT_KIND], "CONTEXT_KIND:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_NEIGHBOR_KIND], "NEIGHBOR_KIND:([0-9]+)", REG_EXTENDED)); + FAILIF(0 != regcomp(&preg[ENUM_PPROP_OIDs], "OIDs:\\{[^}]*\\}([0-9]+)", REG_EXTENDED)); // TODO, pair-wise comparision of Vars. + + //initializeLSHGlobal(); + availableTotalMemory = (unsigned int)8e8; // 800MB by default + + // Parse part of the command-line parameters. + bool computeParameters = false; + char *paramsFile = NULL; + + // Parameters for filtering: + bool no_filtering = false, bug_detecting = true; + int upperBound = 0, lowerBound = 2; + int minNumNodes = 0, min_nVars = 0; + int max_num_diff_vars = 16; + float max_num_diff_nVars_diff = 0.5, max_nVars_diff = 0.35; + bool interfiles = false; + int min_lines = 0; + + for (int opt; (opt = getopt(argc, argv, "ABl:v:V:e:E:a:m:N:d:p:P:R:M:cFf:b:t:")) != -1; ) { + // Needed: -p -f -R + switch (opt) { + case 'A': + fprintf(stderr, "Warning: output all clones. Takes more time...\n"); + no_filtering = true; break; + case 'B': + fprintf(stderr, "Warning: no filtering for bugs now.\n"); + bug_detecting = false; break; + case 'l': min_lines = atoi(optarg); break; + case 'v': min_nVars = atoi(optarg); break; + case 'V': max_num_diff_vars = atoi(optarg); break; + case 'e': max_num_diff_nVars_diff = atof(optarg); break; + case 'E': max_nVars_diff = atof(optarg); break; + case 'm': minNumNodes = atoi(optarg); break; + case 'b': lowerBound = atoi(optarg); break; + case 't': upperBound = atoi(optarg); break; + case 'N': nPoints = atoi(optarg); break; + case 'd': pointsDimension = atoi(optarg); break; + case 'p': paramsFile = optarg; break; + case 'P': successProbability = atof(optarg); break; + case 'M': availableTotalMemory = atol(optarg); break; + case 'a': prefetch = atoi(optarg); break; + case 'c': + fprintf(stderr, "Warning: will compute parameters\n"); + computeParameters = true; + break; + case 'F': + fprintf(stderr, "Warning: inter-file clone detection. Takes more time...\n"); + interfiles = true; break; + case 'R': + nRadii = 1; + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + FAILIF(NULL == (memRatiosForNNStructs = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + listOfRadii[0] = strtod(optarg, NULL); + memRatiosForNNStructs[0] = 1; + break; + case 'f': + readDataSetFromFile2(optarg); + DPRINTF("Allocated memory (after reading data set): %d\n", totalAllocatedMemory); + break; + default: + fprintf(stderr, "Unknown option: -%c\n", opt); + usage(1, argv[0]); + } + } + + if (optind < argc) { + fprintf(stderr, "There are unprocessed parameters left\n"); + usage(1, argv[0]); + } + + CHECK_INT(availableTotalMemory); + CHECK_INT(nPoints); + CHECK_INT(pointsDimension); + CHECK_INT(nRadii); + + if (nPoints > MAX_N_POINTS) { + printf("Error: the structure supports at most %d points (%d were specified).\n", MAX_N_POINTS, nPoints); + fprintf(ERROR_OUTPUT, "Error: the structure supports at most %d points (%d were specified).\n", MAX_N_POINTS, nPoints); + exit(1); + } + + if (computeParameters == false) + computeParameters = readParamsFile(paramsFile); + + if (computeParameters) { + Int32T nSampleQueries = N_SAMPLE_QUERY_POINTS; + PPointT sampleQueries[nSampleQueries]; + Int32T sampleQBoundaryIndeces[nSampleQueries]; + + // Choose several data set points for the sample query points. + for(IntT i = 0; i < nSampleQueries; i++){ + sampleQueries[i] = dataSetPoints[genRandomInt(0, nPoints - 1)]; + } + + // Compute the array sampleQBoundaryIndeces that specifies how to + // segregate the sample query points according to their distance + // to NN. + sortQueryPointsByRadii(pointsDimension, + nSampleQueries, + sampleQueries, + nPoints, + dataSetPoints, + nRadii, + listOfRadii, + sampleQBoundaryIndeces); + + + // Compute the R-NN DS parameters + // if a parameter file is given, output them to that file, and continue + // otherwise, output them to stdout, and exit + + FILE *fd; + if (paramsFile == NULL) { + fd = stdout; + } else { + fd = fopen(paramsFile, "wt"); + if (fd == NULL) { + fprintf(stderr, "Unable to write to parameter file %s\n", paramsFile); + exit(1); + } + } + + fprintf(fd, "%d\n", nRadii); + transformMemRatios(); + for(IntT i = 0; i < nRadii; i++) { + // which sample queries to use + Int32T segregatedQStart = (i == 0) ? 0 : sampleQBoundaryIndeces[i - 1]; + Int32T segregatedQNumber = nSampleQueries - segregatedQStart; + if (segregatedQNumber == 0) { + // XXX: not the right answer + segregatedQNumber = nSampleQueries; + segregatedQStart = 0; + } + ASSERT(segregatedQStart < nSampleQueries); + ASSERT(segregatedQStart >= 0); + ASSERT(segregatedQStart + segregatedQNumber <= nSampleQueries); + ASSERT(segregatedQNumber >= 0); + RNNParametersT optParameters = computeOptimalParameters(listOfRadii[i], + successProbability, + nPoints, + pointsDimension, + dataSetPoints, + segregatedQNumber, + sampleQueries + segregatedQStart, + (Uns32T)((availableTotalMemory - totalAllocatedMemory) * memRatiosForNNStructs[i])); + printRNNParameters(fd, optParameters); + } + if (fd == stdout) { + exit(0); + } else { + fclose(fd); + ASSERT(readParamsFile(paramsFile) == false); + } + } + + // output vector clusters according to the filtering parameters. + printf("========================= Structure built =========================\n"); + printf("nPoints = %d, Dimension = %d\n", nPoints, pointsDimension); + printf("no_filtering (0/1) = %d, inter-file (0/1) = %d, prefetch = %d\n", no_filtering, prefetch, interfiles); + printf("*** Filtering Parameters for individual vectors ***\n"); + printf("minNumNodes = %d, min_nVars = %d, min_lines = %d\n", minNumNodes, min_nVars, min_lines); + printf("*** Filtering Parameters for clusters ***\n"); + printf("lowerBound = %d, upperBound = %d\n", lowerBound, upperBound); + printf("Max num of different nVars = %d, Max diff among different nVars = %g, \nMax diff among the num of different nVars = %g\n", max_num_diff_vars, max_nVars_diff, max_num_diff_nVars_diff); + + + IntT resultSize = nPoints; + PPointT *result = (PPointT*)MALLOC(resultSize * sizeof(*result)); + PPointT queryPoint; + FAILIF(NULL == (queryPoint = (PPointT)MALLOC(sizeof(PointT)))); + FAILIF(NULL == (queryPoint->coordinates = (RealT*)MALLOC(pointsDimension * sizeof(RealT)))); + + TimeVarT meanQueryTime = 0; + int nQueries = 0; + bool seen[nPoints]; + int nBuckets = 0, nBucketedPoints = 0; + + memset(seen, 0, nPoints * sizeof(bool)); + for(IntT i = 0; i < nPoints; nQueries++, i++) { + + // find the next unseen point + while (i < nPoints && seen[i]) i++; + if (i >= nPoints) break; + queryPoint = dataSetPoints[i]; + + // get the near neighbors. + IntT nNNs = 0; + for(IntT r = 0; r < nRadii; r++) { // nRadii is always 1 so far. + nNNs = getRNearNeighbors(nnStructs[r], queryPoint, result, resultSize); + //printf("Total time for R-NN query at radius %0.6lf (radius no. %d):\t%0.6lf\n", (double)(listOfRadii[r]), r, timeRNNQuery); + meanQueryTime += timeRNNQuery; + + //printf("\nQuery point %d: found %d NNs at distance %0.6lf (radius no. %d). NNs are:\n", + // i, nNNs, (double)(listOfRadii[r]), r); + + // sort by filename, then number of variables, then line number + qsort(result, nNNs, sizeof(*result), comparePoints); + + // The result array may contain the queryPoint, so do not output it in the following. + + PPointT *cur = result, *end = result + nNNs; + + if ( ! no_filtering ) { // Filter out certain vectors and clusters. + while (cur < end) { // Shall we discard the rest results + // and start over for a new point? Not + // now for the sake of + // performance...TODO + ASSERT(*cur != NULL); + + // Look for the first un-filtered point for the next bucket. + while ( cur < end ) { + if ( pointIsNotFiltered(cur) ) { + break; + } + seen[(*cur)->index] = true; + cur++; + } + if ( cur >= end ) + break; + + bool worthy = false; + int sizeBucket = 1; // 1 means the first un-filtered point + PPointT *begin = cur; + seen[(*begin)->index] = true; + cur++; + while ( cur < end && + // look for the next point outside the current file + // if interfiles is false; that point is the end of + // current bucket (assume vectors in a bucket are + // sorted by their filenames already). + ( interfiles || strcmp((*begin)->filename, (*cur)->filename)==0 ) ) { + if ( pointIsNotFiltered(cur) ) { + // prepare for filtering + sizeBucket++; + + // the first heuristics for bugs AFTER filtering: + worthy = worthy || (*begin)->prop[ENUM_PPROP_nVARs-1] != (*cur)->prop[ENUM_PPROP_nVARs-1]; + + // the second heuristics for bugs AFTER filtering: + worthy = worthy || inconsistentIDchanges((*begin)->oids, (*cur)->oids); // TODO + } + seen[(*cur)->index] = true; + cur++; + } + + // output the bucket if: + // - there are >= 2 different points + // - there are <= upperBound (default 0) && >= lowerBound (default 2) points + // - there are >= 2 different numbers of variables + // and update nBuckets and nBucketedPoints consequently + if (sizeBucket >= lowerBound && (upperBound < lowerBound || sizeBucket <= upperBound) && ( bug_detecting ? worthy : true ) ) { + nBuckets++; + printf("\n"); + for (PPointT *p = begin; p < cur; p++) { + ASSERT(*p != NULL); + if ( pointIsNotFiltered(p) ) { + nBucketedPoints++; + + // compute the distance to the query point (maybe useless) + RealT distance = 0.; + for (int i = 0; i < pointsDimension; i++) { + RealT t = (*p)->coordinates[i] - queryPoint->coordinates[i]; + // L1 distance +// distance += (t >= 0) ? t : -t; + // Pi--L2 distance, LSH uses L2 by default, we should output L2 distance here. + distance += t*t; + } + // L1 distance +// printf("%09d\tdist:%0.1lf", (*p)->index, distance); + // L2 distance + printf("%09d\tdist:%0.1lf", (*p)->index, sqrt(distance)); + printf("\tFILE %s LINE:%d:%d NODE_KIND:%d nVARs:%d NUM_NODE:%d TBID:%d TEID:%d\n", + (*p)->filename, (*p)->prop[ENUM_PPROP_LINE-1], (*p)->prop[ENUM_PPROP_OFFSET-1], + (*p)->prop[ENUM_PPROP_NODE_KIND-1], (*p)->prop[ENUM_PPROP_nVARs-1], + (*p)->prop[ENUM_PPROP_NUM_NODE-1], (*p)->prop[ENUM_PPROP_TBID-1], (*p)->prop[ENUM_PPROP_TEID-1]); + //CR_ASSERT(distance(pointsDimension, queryPoint, *p) <= listOfRadii[r]); + //DPRINTF("Distance: %lf\n", distance(pointsDimension, queryPoint, result[j])); + //printRealVector("NN: ", pointsDimension, result[j]->coordinates); + } + } + } // end of enumeration of a bucket + } // end of !no_filtering + } + else { + if ( nNNs>=lowerBound ) { // filter out non-clones anyway + nBuckets++; + printf("\n"); + for (PPointT *p = cur; p < end; p++) { + ASSERT(*p != NULL); + nBucketedPoints++; + seen[(*p)->index] = true; + + // compute the distance to the query point (maybe useless) + RealT distance = 0.; + for (int i = 0; i < pointsDimension; i++) { + RealT t = (*p)->coordinates[i] - queryPoint->coordinates[i]; + // L1 distance +// distance += (t >= 0) ? t : -t; + // Pi--L2 distance, LSH uses L2 by default, we should output L2 distance here. + distance += t*t; + } + + // L1 distance +// printf("%09d\tdist:%0.1lf", (*p)->index, distance); + // L2 distance + printf("%09d\tdist:%0.1lf", (*p)->index, sqrt(distance)); + printf("\tFILE %s LINE:%d:%d NODE_KIND:%d nVARs:%d NUM_NODE:%d TBID:%d TEID:%d\n", + (*p)->filename, (*p)->prop[ENUM_PPROP_LINE-1], (*p)->prop[ENUM_PPROP_OFFSET-1], + (*p)->prop[ENUM_PPROP_NODE_KIND-1], (*p)->prop[ENUM_PPROP_nVARs-1], + (*p)->prop[ENUM_PPROP_NUM_NODE-1], (*p)->prop[ENUM_PPROP_TBID-1], (*p)->prop[ENUM_PPROP_TEID-1]); + //CR_ASSERT(distance(pointsDimension, queryPoint, *p) <= listOfRadii[r]); + //DPRINTF("Distance: %lf\n", distance(pointsDimension, queryPoint, result[j])); + //printRealVector("NN: ", pointsDimension, result[j]->coordinates); + } // end of enumeration of a bucket + } // end of nNNs>=lowerBound + } // end of no_filtering and exploration of NNs + } // for (...nRadii...) + } + + // Simple statistics and finish + if (nQueries > 0) { + meanQueryTime = meanQueryTime / nQueries; + printf("\n%d queries, Mean query time: %0.6lf\n", nQueries, (double)meanQueryTime); + printf("%d buckets, %d points (out of %d, %.2f %%) in them\n", + nBuckets, nBucketedPoints, nPoints, 100*(float)nBucketedPoints/(float)nPoints); + } else { + printf("No query\n"); + } + + //freePRNearNeighborStruct(nnStruct); + + return 0; +} diff --git a/src/lsh/sources/exactNNs.cpp b/src/lsh/sources/exactNNs.cpp new file mode 100755 index 0000000..a85ed49 --- /dev/null +++ b/src/lsh/sources/exactNNs.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include +#include "headers.h" + +#define SQR(a) ((a) * (a)) + +RealT **points; +int nPoints; +RealT *query; +int nQueries; +int dimension; +RealT R; +RealT p; + +RealT *listOfRadii = NULL; +IntT nRadii = 0; + +// nearNeighbors[i] is the list of the near neighbors. +int *nearNeighbors; + +void usage(char *programName){ + printf("Usage: %s #pts_in_data_set #queries dimension successProbability radius data_set_file queries_file\n", programName); +} + +RealT norm(int dimension, RealT *p1){ + RealT result = 0; + + for (int i = 0; i < dimension; i++){ + result += SQR(p1[i]); + } + + return sqrt(result); +} + +// Reads in the data set (points and the initial parameters for ) +// in the from the file . +void readPoints(char *filename){ + FILE *f = fopen(filename, "rt"); + //fscanf(f, "%d %d %lf %lf\n", &nPoints, &dimension, &R, &p); + points = (RealT**)malloc(nPoints * sizeof(RealT)); + for(int i = 0; i < nPoints; i++){ + points[i] = (RealT*)malloc(dimension * sizeof(RealT)); + for(int d = 0; d < dimension; d++){ + FSCANF_REAL(f, &(points[i][d])); + } + //printf("norm (%d): %lf\n", i, norm(dimension, points[i])); + } +} + +// Prints the vector of size . The string appears +// in front. +void printRealVector1(char *s, int size, RealT *v){ + printf("%s", s); + for(int i = 0; i < size; i++){ + if (i > 0){ + printf(" "); + } + printf("%lf", v[i]); + } + + printf("\n"); +} + +// Returns the Euclidean distance from point to . +RealT dist(RealT *p1, RealT *p2){ + RealT result = 0; + + for (int i = 0; i < dimension; i++){ + result += SQR(p1[i] - p2[i]); + } + + return SQRT(result); +} + +// Returns 1 iff the square of the Euclidean distance from point to is <=threshold. +int isDistanceSqrLeq(RealT *p1, RealT *p2, RealT threshold){ + RealT result = 0; + + for (int i = 0; i < dimension; i++){ + result += SQR(p1[i] - p2[i]); + } + + return result <= threshold; +} + +int main(int nargs, char **args){ + if (nargs < 7) { + usage(args[0]); + exit(1); + } + + nPoints = atoi(args[1]); + nQueries = atoi(args[2]); + dimension = atoi(args[3]); + p = atof(args[4]); + + char* endPtr[1]; + RealT thresholdR = strtod(args[5], endPtr); + if (thresholdR == 0 || endPtr[1] == args[5]){ + // The value for R is not specified, instead there is a file + // specifying multiple R's. + thresholdR = 0; + + // Read in the file + FILE *radiiFile = fopen(args[5], "rt"); + FAILIF(radiiFile == NULL); + fscanf(radiiFile, "%d\n", &nRadii); + ASSERT(nRadii > 0); + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + for(IntT i = 0; i < nRadii; i++){ + FSCANF_REAL(radiiFile, &listOfRadii[i]); + ASSERT(listOfRadii[i] > 0); + RealT r; + FSCANF_REAL(radiiFile, &r); + } + }else{ + nRadii = 1; + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + listOfRadii[0] = thresholdR; + } + DPRINTF("No. radii: %d\n", nRadii); + + readPoints(args[6]); + + nearNeighbors = (int*)malloc(nPoints * sizeof(int)); + + FILE *queryFile = fopen(args[7], "rt"); + //fscanf(queryFile, "%d\n", &nQueries); + query = (RealT*)malloc(dimension * sizeof(RealT)); + printf("nPoints = %d\n", nPoints); + //printf("nQueries = %d\n", nQueries); + for(int i = 0; i < nQueries; i++){ + // read in the query point. + for(int d = 0; d < dimension; d++){ + FSCANF_REAL(queryFile, &(query[d])); + } + //printRealVector1("Query: ", dimension, query); + + IntT nNNs; + for(int r = 0; r < nRadii; r++){ + TimeVarT time = 0; + nNNs = 0; + RealT sqrR = SQR(listOfRadii[r]); + TIMEV_START(time); + for(int j = 0; j < nPoints; j++){ + if (isDistanceSqrLeq(query, points[j], sqrR)) { + nearNeighbors[nNNs] = j; + nNNs++; + } + //printf("Distance[dist] (%d): %lf\n", j, dist(query, points[j])); + //printRealVector1("X: ", dimension, points[j]); + } + TIMEV_END(time); // time only finding the near neighbors, and exclude printing from timing. + printf("Total time for R-NN query at radius %0.6lf (radius no. %d):\t%0.6lf\n", (double)(listOfRadii[r]), r, time); + + if (nNNs > 0){ + printf("Query point %d: found %d NNs at distance %0.6lf (radius no. %d). NNs are:\n", i, nNNs, (double)(listOfRadii[r]), r); + for(int j = 0; j < nNNs; j++){ + printf("%09d\n", nearNeighbors[j]); + //printRealVector1("NN: ", dimension, points[nearNeighbors[j]]); + } + break; + } + } + if (nNNs == 0){ + printf("Query point %d: no NNs found.\n", i); + } + } + +} diff --git a/src/lsh/sources/exploreBuckets.cpp b/src/lsh/sources/exploreBuckets.cpp new file mode 100755 index 0000000..b1aaa23 --- /dev/null +++ b/src/lsh/sources/exploreBuckets.cpp @@ -0,0 +1,431 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +/* + The main entry file containing the main() function. The main() + function parses the command line parameters and depending on them + calls the correspondin functions. + */ + +#include +#include +#include +#include "headers.h" + +#define N_SAMPLE_QUERY_POINTS 100 + +// The data set containing all the points. +PPointT *dataSetPoints = NULL; +// Number of points in the data set. +IntT nPoints = 0; +// The dimension of the points. +IntT pointsDimension = 0; +// The value of parameter R (a near neighbor of a point is any +// point

from the data set that is the within distance +// ). +//RealT thresholdR = 1.0; + +// The succes probability of each point (each near neighbor is +// reported by the algorithm with probability ). +RealT successProbability = 0.9; + +// Same as , only an array of R's (for the case when +// multiple R's are specified). +RealT *listOfRadii = NULL; +IntT nRadii = 0; + +RealT *memRatiosForNNStructs = NULL; + +char sBuffer[600000]; + +/* + Prints the usage of the LSHMain. + */ +void usage(char *programName){ + printf("Usage: %s #pts_in_data_set #queries dimension successProbability radius data_set_file query_points_file max_available_memory [-c|-p params_file]\n", programName); +} + +inline PPointT readPoint(FILE *fileHandle){ + PPointT p; + RealT sqrLength = 0; + FAILIF(NULL == (p = (PPointT)MALLOC(sizeof(PointT)))); + FAILIF(NULL == (p->coordinates = (RealT*)MALLOC(pointsDimension * sizeof(RealT)))); + for(IntT d = 0; d < pointsDimension; d++){ + FSCANF_REAL(fileHandle, &(p->coordinates[d])); + sqrLength += SQR(p->coordinates[d]); + } + fscanf(fileHandle, "%[^\n]", sBuffer); + p->index = -1; + p->sqrLength = sqrLength; + return p; +} + +// Reads in the data set points from in the array +// . Each point get a unique number in the field +// to be easily indentifiable. +void readDataSetFromFile(char *filename){ + FILE *f = fopen(filename, "rt"); + FAILIF(f == NULL); + + //fscanf(f, "%d %d ", &nPoints, &pointsDimension); + //FSCANF_DOUBLE(f, &thresholdR); + //FSCANF_DOUBLE(f, &successProbability); + //fscanf(f, "\n"); + FAILIF(NULL == (dataSetPoints = (PPointT*)MALLOC(nPoints * sizeof(PPointT)))); + for(IntT i = 0; i < nPoints; i++){ + dataSetPoints[i] = readPoint(f); + dataSetPoints[i]->index = i; + } +} + +// Tranforming from +// to +// . +void transformMemRatios(){ + RealT sum = 0; + for(IntT i = nRadii - 1; i >= 0; i--){ + sum += memRatiosForNNStructs[i]; + memRatiosForNNStructs[i] = memRatiosForNNStructs[i] / sum; + //DPRINTF("%0.6lf\n", memRatiosForNNStructs[i]); + } + ASSERT(sum <= 1.000001); +} + + +int compareInt32T(const void *a, const void *b){ + Int32T *x = (Int32T*)a; + Int32T *y = (Int32T*)b; + return (*x > *y) - (*x < *y); +} + +/* + The main entry to LSH package. Depending on the command line + parameters, the function computes the R-NN data structure optimal + parameters and/or construct the R-NN data structure and runs the + queries on the data structure. + */ +#define ENUM_BUCKETS + +int main(int nargs, char **args){ +#ifndef ENUM_BUCKETS + if(nargs < 9){ + usage(args[0]); + exit(1); + } +#endif + + //initializeLSHGlobal(); + + // Parse part of the command-line parameters. +#ifdef ENUM_BUCKETS + nPoints = atoi(args[1]); + pointsDimension = atoi(args[2]); + successProbability = atof(args[3]); + char* endPtr[1]; + RealT thresholdR = strtod(args[4], endPtr); +#else + nPoints = atoi(args[1]); + IntT nQueries = atoi(args[2]); + pointsDimension = atoi(args[3]); + successProbability = atof(args[4]); + char* endPtr[1]; + RealT thresholdR = strtod(args[5], endPtr); + if (thresholdR == 0 || endPtr[1] == args[5]){ + // The value for R is not specified, instead there is a file + // specifying multiple R's. + thresholdR = 0; + + // Read in the file + FILE *radiiFile = fopen(args[5], "rt"); + FAILIF(radiiFile == NULL); + fscanf(radiiFile, "%d\n", &nRadii); + ASSERT(nRadii > 0); + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + FAILIF(NULL == (memRatiosForNNStructs = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + for(IntT i = 0; i < nRadii; i++){ + FSCANF_REAL(radiiFile, &listOfRadii[i]); + ASSERT(listOfRadii[i] > 0); + FSCANF_REAL(radiiFile, &memRatiosForNNStructs[i]); + ASSERT(memRatiosForNNStructs[i] > 0); + } + }else{ +#endif + nRadii = 1; + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + FAILIF(NULL == (memRatiosForNNStructs = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + listOfRadii[0] = thresholdR; + memRatiosForNNStructs[0] = 1; +#ifndef ENUM_BUCKETS + } +#endif + DPRINTF("No. radii: %d\n", nRadii); + //thresholdR = atof(args[5]); + availableTotalMemory = atol(args[6]); + + if (nPoints > MAX_N_POINTS) { + printf("Error: the structure supports at most %d points (%d were specified).\n", MAX_N_POINTS, nPoints); + fprintf(ERROR_OUTPUT, "Error: the structure supports at most %d points (%d were specified).\n", MAX_N_POINTS, nPoints); + exit(1); + } + + readDataSetFromFile(args[5]); + DPRINTF("Allocated memory (after reading data set): %d\n", totalAllocatedMemory); + + Int32T nSampleQueries = N_SAMPLE_QUERY_POINTS; + PPointT sampleQueries[nSampleQueries]; + Int32T sampleQBoundaryIndeces[nSampleQueries]; +#ifndef ENUM_BUCKETS + if ((nargs < 9) || (strcmp("-c", args[9]) == 0)){ + // In this cases, we need to generate a sample query set for + // computing the optimal parameters. + + // Generate a sample query set. + FILE *queryFile = fopen(args[7], "rt"); + if (strcmp(args[7], ".") == 0 || queryFile == NULL || nQueries <= 0){ +#endif + // Choose several data set points for the sample query points. + for(IntT i = 0; i < nSampleQueries; i++){ + sampleQueries[i] = dataSetPoints[genRandomInt(0, nPoints - 1)]; + } +#ifndef ENUM_BUCKETS + }else{ + // Choose several actual query points for the sample query points. + nSampleQueries = MIN(nSampleQueries, nQueries); + Int32T sampleIndeces[nSampleQueries]; + for(IntT i = 0; i < nSampleQueries; i++){ + sampleIndeces[i] = genRandomInt(0, nQueries - 1); + } + qsort(sampleIndeces, nSampleQueries, sizeof(*sampleQueries), compareInt32T); + //printIntVector("sampleIndeces: ", nSampleQueries, sampleIndeces); + Int32T j = 0; + for(Int32T i = 0; i < nQueries; i++){ + if (i == sampleIndeces[j]){ + sampleQueries[j] = readPoint(queryFile); + j++; + while (i == sampleIndeces[j]){ + sampleQueries[j] = sampleQueries[j - 1]; + j++; + } + }else{ + fscanf(queryFile, "%[^\n]", sBuffer); + fscanf(queryFile, "\n"); + } + } + nSampleQueries = j; + fclose(queryFile); + } +#endif + + // Compute the array sampleQBoundaryIndeces that specifies how to + // segregate the sample query points according to their distance + // to NN. + sortQueryPointsByRadii(pointsDimension, + nSampleQueries, + sampleQueries, + nPoints, + dataSetPoints, + nRadii, + listOfRadii, + sampleQBoundaryIndeces); +#ifndef ENUM_BUCKETS + } +#endif + + RNNParametersT *algParameters = NULL; + PRNearNeighborStructT *nnStructs = NULL; +#ifndef ENUM_BUCKETS + if (nargs > 9) { + // Additional command-line parameter is specified. + if (strcmp("-c", args[9]) == 0) { + // Only compute the R-NN DS parameters and output them to stdout. + + printf("%d\n", nRadii); + transformMemRatios(); + for(IntT i = 0; i < nRadii; i++){ + // which sample queries to use + Int32T segregatedQStart = (i == 0) ? 0 : sampleQBoundaryIndeces[i - 1]; + Int32T segregatedQNumber = nSampleQueries - segregatedQStart; + if (segregatedQNumber == 0) { + // XXX: not the right answer + segregatedQNumber = nSampleQueries; + segregatedQStart = 0; + } + ASSERT(segregatedQStart < nSampleQueries); + ASSERT(segregatedQStart >= 0); + ASSERT(segregatedQStart + segregatedQNumber <= nSampleQueries); + ASSERT(segregatedQNumber >= 0); + RNNParametersT optParameters = computeOptimalParameters(listOfRadii[i], + successProbability, + nPoints, + pointsDimension, + dataSetPoints, + segregatedQNumber, + sampleQueries + segregatedQStart, + (Uns32T)((availableTotalMemory - totalAllocatedMemory) * memRatiosForNNStructs[i])); + printRNNParameters(stdout, optParameters); + } + exit(0); + } else if (strcmp("-p", args[9]) == 0) { + // Read the R-NN DS parameters from the given file and run the + // queries on the constructed data structure. + if (nargs < 10){ + usage(args[0]); + exit(1); + } +#endif + FILE *pFile = fopen(args[7], "rt"); + FAILIFWR(pFile == NULL, "Could not open the params file."); + fscanf(pFile, "%d\n", &nRadii); + DPRINTF1("Using the following R-NN DS parameters:\n"); + DPRINTF("N radii = %d\n", nRadii); + FAILIF(NULL == (nnStructs = (PRNearNeighborStructT*)MALLOC(nRadii * sizeof(PRNearNeighborStructT)))); + FAILIF(NULL == (algParameters = (RNNParametersT*)MALLOC(nRadii * sizeof(RNNParametersT)))); + for(IntT i = 0; i < nRadii; i++){ + algParameters[i] = readRNNParameters(pFile); + printRNNParameters(stderr, algParameters[i]); + nnStructs[i] = initLSH_WithDataSet(algParameters[i], nPoints, dataSetPoints); + } + + pointsDimension = algParameters[0].dimension; + FREE(listOfRadii); + FAILIF(NULL == (listOfRadii = (RealT*)MALLOC(nRadii * sizeof(RealT)))); + for(IntT i = 0; i < nRadii; i++){ + listOfRadii[i] = algParameters[i].parameterR; + } +#ifndef ENUM_BUCKETS + } else{ + // Wrong option. + usage(args[0]); + exit(1); + } + } else { + FAILIF(NULL == (nnStructs = (PRNearNeighborStructT*)MALLOC(nRadii * sizeof(PRNearNeighborStructT)))); + // Determine the R-NN DS parameters, construct the DS and run the queries. + transformMemRatios(); + for(IntT i = 0; i < nRadii; i++){ + // XXX: segregate the sample queries... + nnStructs[i] = initSelfTunedRNearNeighborWithDataSet(listOfRadii[i], + successProbability, + nPoints, + pointsDimension, + dataSetPoints, + nSampleQueries, + sampleQueries, + (Uns32T)((availableTotalMemory - totalAllocatedMemory) * memRatiosForNNStructs[i])); + } + } +#endif + +#if 1 +#ifdef ENUM_BUCKETS + printf("========================= Structure built =========================\n"); + + PRNearNeighborStructT nnStruct = nnStructs[0]; + printf("TypeHT: %d\n", nnStruct->hashedBuckets[0]->typeHT); + + for (int i = 0; i < 1 /*nnStruct->parameterL*/; i++) { + PUHashStructureT htable = nnStruct->hashedBuckets[i]; + PHybridChainEntryT entry = htable->hybridChainsStorage; +#if 1 + for (int j = 0; j < htable->nHashedBuckets; j++) { + int refIndex = entry[1].point.pointIndex; + do { + entry++; + int idx = entry->point.pointIndex; + printf("%d\t", idx); + RealT dist = 0; + for (int x = 0; x < nnStruct->dimension; x++) { + RealT tmp = nnStruct->points[idx]->coordinates[x] - nnStruct->points[refIndex]->coordinates[x]; + //printf("%f ", tmp); //nnStruct->points[idx]); + tmp = tmp * tmp; + dist += tmp; + } + printf("%f\t%g\n", dist, dist); + } while (entry->point.isLastPoint == 0); + entry++; + printf("\n"); + } +#else + for (int j = 0; j < 1000; j++) { + printf("%d\t%d\t%d\t%d\n", + entry->point.isLastBucket, + entry->point.isLastPoint, + entry->point.bucketLength, + entry->point.pointIndex); + if (entry->point.isLastPoint == 1) entry++; + entry++; + } +#endif + } + +#else + DPRINTF1("X\n"); + + IntT resultSize = nPoints; + PPointT *result = (PPointT*)MALLOC(resultSize * sizeof(*result)); + PPointT queryPoint; + FAILIF(NULL == (queryPoint = (PPointT)MALLOC(sizeof(PointT)))); + FAILIF(NULL == (queryPoint->coordinates = (RealT*)MALLOC(pointsDimension * sizeof(RealT)))); + + FILE *queryFile = fopen(args[7], "rt"); + FAILIF(queryFile == NULL); + TimeVarT meanQueryTime = 0; + for(IntT i = 0; i < nQueries; i++){ + + RealT sqrLength = 0; + // read in the query point. + for(IntT d = 0; d < pointsDimension; d++){ + FSCANF_REAL(queryFile, &(queryPoint->coordinates[d])); + sqrLength += SQR(queryPoint->coordinates[d]); + } + queryPoint->sqrLength = sqrLength; + //printRealVector("Query: ", pointsDimension, queryPoint->coordinates); + + // get the near neighbors. + IntT nNNs = 0; + for(IntT r = 0; r < nRadii; r++){ + nNNs = getRNearNeighbors(nnStructs[r], queryPoint, result, resultSize); + printf("Total time for R-NN query at radius %0.6lf (radius no. %d):\t%0.6lf\n", (double)(listOfRadii[r]), r, timeRNNQuery); + meanQueryTime += timeRNNQuery; + + if (nNNs > 0){ + printf("Query point %d: found %d NNs at distance %0.6lf (radius no. %d). NNs are:\n", i, nNNs, (double)(listOfRadii[r]), r); + for(IntT j = 0; j < nNNs; j++){ + ASSERT(result[j] != NULL); + printf("%09d\tdist:%0.6lf\n", result[j]->index, distance(pointsDimension, queryPoint, result[j])); + CR_ASSERT(distance(pointsDimension, queryPoint, result[j]) <= listOfRadii[r]); + //DPRINTF("Distance: %lf\n", distance(pointsDimension, queryPoint, result[j])); + //printRealVector("NN: ", pointsDimension, result[j]->coordinates); + } + break; + } + } + if (nNNs == 0){ + printf("Query point %d: no NNs found.\n", i); + } + } + if (nQueries > 0){ + meanQueryTime = meanQueryTime / nQueries; + printf("Mean query time: %0.6lf\n", (double)meanQueryTime); + } + + //freePRNearNeighborStruct(nnStruct); +#endif +#endif + + return 0; +} diff --git a/src/lsh/sources/genDS.cpp b/src/lsh/sources/genDS.cpp new file mode 100755 index 0000000..2ab655f --- /dev/null +++ b/src/lsh/sources/genDS.cpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +/* + This program generates the input data set. The arguments are: + dimension M #_of_points R probability_of_success + + The points are generated uniformly at random in the range [-M, M]^d. + */ + +#include +#include +#include +#include "BasicDefinitions.h" +#include "Random.h" + +void usage(char *programName){ + printf("Usage: %s dimension max_size #_of_points R probability_of_success\n", programName); +} + +int main(int nargs, char **args){ + if (nargs < 5){ + usage(args[0]); + exit(1); + } + + IntT dimension = atoi(args[1]); + RealT M = atof(args[2]); + IntT nPoints = atoi(args[3]); + RealT R = atof(args[4]); + //IntT nExpNNs = atoi(args[4]); + RealT probability = atof(args[5]); + + // for d even, Vol_d = pi^{d/2} / (d/2)! + + + // RealT vol_d_const = 1.0 / SQRT(M_PI * (RealT)dimension); + + // RealT R = 2.0 * M * POW((RealT)nExpNNs / (RealT)nPoints / vol_d_const, 1.0/(RealT)dimension) / SQRT(M_PI) * SQRT((RealT)dimension / 2.0 / M_E); + +// RealT vol_d_const = 1; +// for(IntT i = 1; i <= dimension / 2; i++) +// vol_d_const = vol_d_const * M_PI / (RealT)i; + +// RealT R = 2.0 * M * POW((RealT)nExpNNs / (RealT)nPoints / vol_d_const, 1.0/(RealT)dimension); + + //printf("%Lf %Lf \n", POW(R/2.0/M, dimension), (RealT)nExpNNs / (RealT)nPoints / vol_d_const); + printf("%d %d ", nPoints, dimension); + FPRINTF_REAL(stdout, R); + printf(" "); + FPRINTF_REAL(stdout, probability); + printf("\n"); + + for(IntT i = 0; i < nPoints; i++){ + for(IntT d = 0; d < dimension; d++){ + FPRINTF_REAL(stdout, genUniformRandom(-M, M)); + printf(" "); + } + printf("\n"); + } +} diff --git a/src/lsh/sources/genPlantedDS.cpp b/src/lsh/sources/genPlantedDS.cpp new file mode 100755 index 0000000..016f877 --- /dev/null +++ b/src/lsh/sources/genPlantedDS.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +/* + This program generates the input data set. The arguments are: + dimension M #_of_points R probability_of_success + + The points are generated uniformly at random in the range [-M, M]^d. + */ + +#include +#include +#include +#include "BasicDefinitions.h" +#include "Random.h" + +void usage(char *programName){ + printf("Usage: %s dimension max_size #_of_points R probability_of_success\n", programName); +} + +int main(int nargs, char **args){ + if (nargs < 5){ + usage(args[0]); + exit(1); + } + + IntT dimension = atoi(args[1]); + RealT M = atof(args[2]); + IntT nPoints = atoi(args[3]); + RealT R = atof(args[4]); + //IntT nExpNNs = atoi(args[4]); + RealT probability = atof(args[5]); + + // for d even, Vol_d = pi^{d/2} / (d/2)! + + + // RealT vol_d_const = 1.0 / SQRT(M_PI * (RealT)dimension); + + // RealT R = 2.0 * M * POW((RealT)nExpNNs / (RealT)nPoints / vol_d_const, 1.0/(RealT)dimension) / SQRT(M_PI) * SQRT((RealT)dimension / 2.0 / M_E); + +// RealT vol_d_const = 1; +// for(IntT i = 1; i <= dimension / 2; i++) +// vol_d_const = vol_d_const * M_PI / (RealT)i; + +// RealT R = 2.0 * M * POW((RealT)nExpNNs / (RealT)nPoints / vol_d_const, 1.0/(RealT)dimension); + + //printf("%Lf %Lf \n", POW(R/2.0/M, dimension), (RealT)nExpNNs / (RealT)nPoints / vol_d_const); + printf("%d %d ", nPoints, dimension); + FPRINTF_REAL(stdout, R); + printf(" "); + FPRINTF_REAL(stdout, probability); + printf("\n"); + + for(IntT d = 0; d < dimension; d++){ + FPRINTF_REAL(stdout, 0.99 * R / SQRT(dimension)); + printf(" "); + } + printf("\n"); + + for(IntT i = 0; i < nPoints - 1; i++){ + for(IntT d = 0; d < dimension; d++){ + FPRINTF_REAL(stdout, genUniformRandom(-M, M)); + printf(" "); + } + printf("\n"); + } +} diff --git a/src/lsh/sources/headers.h b/src/lsh/sources/headers.h new file mode 100755 index 0000000..49b10de --- /dev/null +++ b/src/lsh/sources/headers.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include +#include +#include +#include +#include + +#include "BasicDefinitions.h" +#include "Random.h" +#include "Geometry.h" +#include "Util.h" +#include "BucketHashing.h" +#include "LocalitySensitiveHashing.h" +#include "SelfTuning.h" +#include "NearNeighbors.h" + +#ifdef DEBUG_MEM +#include +#endif + +#ifdef DEBUG_TIMINGS +#include +#endif + +#include "GlobalVars.h" diff --git a/src/lsh/sources/testFloat.cpp b/src/lsh/sources/testFloat.cpp new file mode 100755 index 0000000..6221745 --- /dev/null +++ b/src/lsh/sources/testFloat.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2004-2005 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * MIT grants permission to use, copy, modify, and distribute this software and + * its documentation for NON-COMMERCIAL purposes and without fee, provided that + * this copyright notice appears in all copies. + * + * MIT provides this software "as is," without representations or warranties of + * any kind, either expressed or implied, including but not limited to the + * implied warranties of merchantability, fitness for a particular purpose, and + * noninfringement. MIT shall not be liable for any damages arising from any + * use of this software. + * + * Author: Alexandr Andoni (andoni@mit.edu), Piotr Indyk (indyk@mit.edu) + */ + +#include "headers.h" + +int main(){ + float f; + f = sqrtf(2); + return 0; +} diff --git a/src/main/Makefile b/src/main/Makefile new file mode 100755 index 0000000..20372ef --- /dev/null +++ b/src/main/Makefile @@ -0,0 +1,118 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +TREEHEADER=../include/ptree.h +TREESRC=ptree.cc + +CTREEOBJS=ptreeC.o +JTREEOBJS=ptreeJ.o +PHPTREEOBJS=ptreePHP.o + +CVGENOBJS=${CTREEOBJS} ../vgen/treeTra/libvgen.a # TODO: libvgen may also be language dependent, especially the parts that involve "contextual nodes" (e.g., bugfinding, context locating code) +# A better architecture would make 'vgen' language independent, put all language-dependent code into ptgen/ptree but provide a language-independent interface for accessing tree structures; +# 'bugfinding' (the buggy code scoring algorithm in particular) should also be language-independent, accessing language-dependent tree structures only through a language-independent interface. +JVGENOBJS=${JTREEOBJS} ../vgen/treeTra/libvgen.a +PHPVGENOBJS=${PHPTREEOBJS} ../vgen/treeTra/libvgen.a + +COBJS=${CVGENOBJS} ../ptgen/gcc/gccptgen.a +JOBJS=${JVGENOBJS} ../ptgen/java/javaptgen.a +PHPOBJS=${PHPVGENOBJS} ../ptgen/php5/phpptgen.a + +CHEADERS=../ptgen/gcc/crelevantNodes.h ../ptgen/gcc/catomicNodes.h ../ptgen/gcc/cparentNodes.h ../ptgen/gcc/ccontextualNodes.h +JHEADERS=../ptgen/java/jrelevantNodes.h ../ptgen/java/jatomicNodes.h ../ptgen/java/jparentNodes.h ../ptgen/java/jcontextualNodes.h +PHPHEADERS=../ptgen/php5/phprelevantNodes.h ../ptgen/php5/phpatomicNodes.h ../ptgen/php5/phpparentNodes.h ../ptgen/php5/phpcontextualNodes.h + +CXX= g++ -I../include -I../vgen/treeTra # -g +CXXFLAGS= -O3 + +EXES=cvecgen jvecgen cbugfilters jbugfilters out2html \ + phpvecgen phpbugfilters out2xml \ + cParseTreeMain jParseTreeMain phpParseTreeMain + + +TARGET:${EXES} + +#NOTE: this Makefile may not be good for parallel compilation, +# because intermediate files may be overwritten. + +${CTREEOBJS}:${TREESRC} ${TREEHEADER} + $(CXX) -o $@ $(CXXFLAGS) -c -DCLANG ${TREESRC} + +${JTREEOBJS}:${TREESRC} ${TREEHEADER} + $(CXX) -o $@ $(CXXFLAGS) -c -DJAVA ${TREESRC} + +${PHPTREEOBJS}:${TREESRC} ${TREEHEADER} + $(CXX) -o $@ $(CXXFLAGS) -c -DPHP ${TREESRC} + +cvecgen:${COBJS} ${CHEADERS} main.cc + $(CXX) $(CXXFLAGS) -c -DCLANG main.cc + $(CXX) -o $@ $(CXXFLAGS) main.o ${COBJS} + +jvecgen:${JOBJS} ${JHEADERS} main.cc + $(CXX) $(CXXFLAGS) -c -DJAVA main.cc + $(CXX) -o $@ $(CXXFLAGS) main.o ${JOBJS} + +phpvecgen:${PHPOBJS} ${PHPHEADERS} main.cc + $(CXX) $(CXXFLAGS) -c -DPHP main.cc + $(CXX) -o $@ $(CXXFLAGS) main.o ${PHPOBJS} + +cbugfilters:${COBJS} ${CHEADERS} bugmain.cc + ${CXX} ${CXXFLAGS} -c -DCLANG bugmain.cc + ${CXX} ${CXXFLAGS} -o $@ bugmain.o ${COBJS} + +jbugfilters:${JOBJS} ${JHEADERS} bugmain.cc + ${CXX} ${CXXFLAGS} -c -DJAVA bugmain.cc + ${CXX} ${CXXFLAGS} -o $@ bugmain.o ${JOBJS} + +phpbugfilters:${PHPOBJS} ${PHPHEADERS} bugmain.cc + ${CXX} ${CXXFLAGS} -c -DPHP bugmain.cc + ${CXX} ${CXXFLAGS} -o $@ bugmain.o ${PHPOBJS} + +out2html:${COBJS} out2html.C # don't matter to use COBJS or others + ${CXX} ${CXXFLAGS} -o $@ out2html.C ${COBJS} + +out2xml:${COBJS} out2xml.C # don't matter to use COBJS or others + ${CXX} ${CXXFLAGS} -o $@ out2xml.C ${COBJS} + +cParseTreeMain:${COBJS} ${CHEADERS} parseTreeMain.cc + ${CXX} ${CXXFLAGS} -DCLANG -o $@ parseTreeMain.cc ${COBJS} + +jParseTreeMain:${JOBJS} ${JHEADERS} parseTreeMain.cc + ${CXX} ${CXXFLAGS} -DJAVA -o $@ parseTreeMain.cc ${JOBJS} + +phpParseTreeMain:${PHPOBJS} ${PHPHEADERS} parseTreeMain.cc + ${CXX} ${CXXFLAGS} -DPHP -o $@ parseTreeMain.cc ${PHPOBJS} + +.PHONY: clean +clean: + rm -f *.o ${EXES} + diff --git a/src/main/bugmain.cc b/src/main/bugmain.cc new file mode 100755 index 0000000..4bca7dd --- /dev/null +++ b/src/main/bugmain.cc @@ -0,0 +1,172 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +map name2id; +map id2name; +#ifdef JAVA +string identifierTypeName = "ID_TK"; +#else +#ifdef PHP +string identifierTypeName = "T_VARIABLE"; +#else +string identifierTypeName = "IDENTIFIER"; +#endif +#endif + +static const char *cxtNodes[] = { +#ifdef JAVA +#include "../ptgen/java/jcontextualNodes.h" +#else +#ifdef PHP +#include "../ptgen/php5/phpcontextualNodes.h" +#else +#include "../ptgen/gcc/ccontextualNodes.h" +#endif +#endif +}; + +/* For debugging use only */ +ParseTree* global_tree_for_debugging; + +static bool cloneClusterFiltered(TokenTreeMap & cls) { + /* Some managieral problems caused a lot of confusion about which + code is for which filters...and the several versions of the + filters (especially for filter 4) caused the most trouble, the + data (# bugs detected) in the paper may be a little off (not + essential though. Fixed). + + The code for filters should be better organized to avoid the + above confusions. TODO: use command-line options instead of + re-compiling the code for enabling different filters. However, + since the current code for filtering is scattered among serveral source + files, it'd better to re-architect the filtering code before + implementing the command line options. + */ + + if ( cls.rank>=4 /* <=> cls.buggy_score[1]>0 */ && cls.buggy_score[1]>2 ) /* Filter ALL */ + cls.rank -= 4; + if ( cls.rank==0 // || cls.buggy_score[0]==0 + // individual filters should or should not preserve cases where other things are buggy? + || ( /* cls.rank<4 && */ cls.buggy_score[0]==-8 ) /* Filter 1: none vs. none. 1 */ + || ( /* cls.rank<4 && */ (cls.buggy_score[0]==-4 || cls.buggy_score[0]==-5) ) /* Filter 2: loop/switch vs. none; deep "if" vs. none. 7.2 */ + || ( /* cls.rank<4 && */ (cls.buggy_score[0]==0 || cls.buggy_score[0]==1) && cls.buggy_score[3]<1 ) /* Filter 3: loop vs. cond+none. 7.1 */ + // filter 4 is in token-tree-map.C. + || ( /* cls.rank==4 && */ cls.buggy_score[1]>2 ) /* Filter 5: nVARs differ too much. 4.1 */ + || ( cls.buggy_score[2]<10 ) /* Filter 6: clones are too spatially close. 4.2 */ + ) + return true; + else + return false; +} + +/* Decide whether to filter a clone cluster: for ONE cluster only; + use a shell script to pass every clone cluster to this class. */ +int main( int argc, char **argv ) +{ + if ( argc>2 ) { + cerr << "Usage: " << argv[0] << " [filter ID=0] [a clone cluster from stdin]" << endl + << "\t stdin is the input channel; stdout is the output channel." << endl; + return 1; + } + cerr << "NOTE: Command line options for filter IDs not implemented" << endl; +/* + int fid = 0; + if ( argc==2 ) + fid = atoi(argv[1]); + if ( fid<0 || fid>2 ) + fid = 0; +*/ + id_init(); + TokenTreeMap::init_shared_data(); +#ifdef JAVA + TokenTreeMap_Java tt; +#else +#ifdef PHP + ContextInconsistency_PHP tt; +#else + TokenTreeMap tt; +#endif +#endif + tt.initNodes(cxtNodes); + tt.createFN2Tree(); + +/* + bool filtered = false; + switch (fid) { + case 0: // iff all filters agree: + filtered = tt.isAllFiltered(); + break; + case 1: + isFilteredI(tt, 1, filtered); + break; + case 2: + isFilteredI(tt, 2, filtered); + break; + case 3: + isFilteredI(tt, 3, filtered); + break; + default: + filtered = tt.isAnyFiltered(); + break; + } +*/ + + if ( !cloneClusterFiltered(tt) ) { +#ifdef HTML + cout << "Rank score: " << tt.rank << " * " << tt.clusterbuffer.size() << " =" << tt.rank*tt.clusterbuffer.size() + << " buggy score: "; + for (int i=0; i < NUM_BUGGY_SCORES; i++) + cout << tt.buggy_score[i] << " "; + cout << endl; + for (int i=0; i < tt.clusterbuffer.size(); i++) { + tt.clusterbuffer[i].out2html(cout); + } + cout << "

" << endl; +#else + tt.outputCluster(cout); + cout << endl; +#endif + } + + return 0; +} + diff --git a/src/main/build.sh b/src/main/build.sh new file mode 100755 index 0000000..8fb74d9 --- /dev/null +++ b/src/main/build.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +#export CXXFLAGS="-pg -g" +#export CFLAGS="-pg -g" +export CXXFLAGS="-O3" +export CFLAGS="-O3" + +( +cd ../ptgen/ || exit 1 +make clean +make +errcode=$? +if [ $errcode -ne 0 ]; then + echo "Error: ptgen make failed. Exit." + exit $errcode +fi +) + +( +cd ../vgen/treeTra/ || exit 1 +make clean +make +errcode=$? +if [ $errcode -ne 0 ]; then + echo "Error: vcgen make failed. Exit." + exit 1 +fi +cd ../vgrouping/ || exit 1 +make clean +make +errcode=$? +if [ $errcode -ne 0 ]; then + echo "Error: vgrouping make failed. Exit." + exit $errcode +fi +) + +make clean +make +errcode=$? +if [ $errcode -ne 0 ]; then + echo "Error: main make failed. Exit." + exit $errcode +fi + +( +cd ../lsh/ || exit 1 +make clean_all +make +errcode=$? +if [ $errcode -ne 0 ]; then + echo "Error: lsh make failed. Exit." + exit $errcode +fi +) + diff --git a/src/main/clean.sh b/src/main/clean.sh new file mode 100644 index 0000000..373a073 --- /dev/null +++ b/src/main/clean.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +#export CXXFLAGS="-pg -g" +#export CFLAGS="-pg -g" +export CXXFLAGS="-O3" +export CFLAGS="-O3" + +( +cd ../ptgen/ || exit 1 +make clean +) + +( +cd ../vgen/treeTra/ || exit 1 +make clean +cd ../vgrouping/ || exit 1 +make clean +) + +make clean + +( +cd ../lsh/ || exit 1 +make clean_all +) + diff --git a/src/main/main.cc b/src/main/main.cc new file mode 100755 index 0000000..100bae9 --- /dev/null +++ b/src/main/main.cc @@ -0,0 +1,296 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/** This is the main entry for vector generation for one source file. + * With different defines, the file can be compiled for different languages. + * + * Need to use with scripts if wanting vector generation for many source files. + */ + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +map name2id; +map id2name; +#ifdef JAVA +string identifierTypeName = "ID_TK"; +#else +#ifdef PHP +string identifierTypeName = "T_VARIABLE"; +#else +string identifierTypeName = "IDENTIFIER"; +#endif +#endif + +int yyparse(); + +extern Tree *root; + +void id_init(); + +extern FILE *yyin; + +static const char *relNodes[] = { +#ifdef JAVA +#include "../ptgen/java/jrelevantNodes.h" +#else +#ifdef PHP +#include "../ptgen/php5/phprelevantNodes.h" +#else +#include "../ptgen/gcc/crelevantNodes.h" +#endif +#endif +}; + +static const char *atomicNodes[] = { +#ifdef JAVA +#include "../ptgen/java/jatomicNodes.h" +#else +#ifdef PHP +#include "../ptgen/php5/phpatomicNodes.h" +#else +#include "../ptgen/gcc/catomicNodes.h" +#endif +#endif +}; + +static const char *valParents[] = { +#ifdef JAVA +#include "../ptgen/java/jparentNodes.h" +#else +#ifdef PHP +#include "../ptgen/php5/phpparentNodes.h" +#else +#include "../ptgen/gcc/cparentNodes.h" +#endif +#endif +}; + + +void initNodes( vector & nodes, const char **nodeconfig) +{ + for (const char **s= nodeconfig; *s != NULL; s++) { + map::iterator i= name2id.find(*s); + if (i == name2id.end()) { + cerr << "unknown node type: " << *s << endl; + continue; + } + nodes.push_back(i->second); + } +} + +/* For debugging use only */ +ParseTree* global_tree_for_debugging; + +int main( int argc, char **argv ) +{ + const char * inputfilename = NULL; + const char * outputfilename = NULL; + const char * configfilename = NULL; + int mergeTokens = 30, mergeStride = 1, mergeLists = 3; + int startline = 0, endline = -1; /* default: all lines */ + + // use getopt_long; should also work under cygwin + static const struct option longOpts[] = { + { "input-file", required_argument, NULL, 'i' }, + { "output-file", required_argument, NULL, 'o' }, + { "config-file", required_argument, NULL, 'c' }, + { "mim-token-number", required_argument, NULL, 'm' }, + { "stride", required_argument, NULL, 't' }, + { "merge-list-size", required_argument, NULL, 0 }, + { "start-line-number", required_argument, NULL, 0 }, + { "end-line-number", required_argument, NULL, 0 }, + { "help", no_argument, NULL, 'h'}, + { NULL, no_argument, NULL, 0 } + }; + + int c = 0, longIndex = 0; + while ((c = getopt_long (argc, argv, "i:o:c:m:t:h", longOpts, &longIndex)) != -1) { + switch (c) { + case 'i': + inputfilename = optarg; + break; + case 'o': + outputfilename = optarg; + break; + case 'f': + configfilename = optarg; + TraGenMain::getParameters(configfilename, mergeTokens, mergeStride, mergeLists); + break; + case 'm': + if ( sscanf(optarg, "%d", &mergeTokens)<=0 ) { + cerr << "Warning: invalid mergeTokens in option -m (default 30): " << optarg << endl; + mergeTokens = 30; + } + break; + case 't': + if ( sscanf(optarg, "%d", &mergeStride)<=0 ) { + cerr << "Warning: invalid mergeStride in option -t (default 1): " << optarg << endl; + mergeStride = 1; + } + break; + case 0: /* for long options w/o a short name */ + if( stricmp( "merge-list-size", longOpts[longIndex].name ) == 0 ) { + if ( sscanf(optarg, "%d", &mergeLists)<=0 ) { + cerr << "Warning: invalid mergeLists in option --merge-list-size (default 3): " << optarg << endl; + mergeLists = 3; + } + } else if ( stricmp( "start-line-number", longOpts[longIndex].name ) == 0 ) { + if ( sscanf(optarg, "%d", &startline)<=0 ) { + cerr << "Warning: invalid startline in option --start-line-number (default: all lines): " << optarg << endl; + startline = 0; + } + } else if ( stricmp( "end-line-number", longOpts[longIndex].name ) == 0 ) { + if ( sscanf(optarg, "%d", &endline)<=0 ) { + cerr << "Warning: invalid endline in option --end-line-number (default: same as --start-line-number): " << optarg << endl; + endline = -1; + } + } + break; + case 'h': + case '?': + /* getopt_long should have already printed an error message. */ + cerr << "Usage: " << argv[0] << " [options] [filename] " << endl; + cerr << "--input-file , or -i " << endl; + cerr << "--output-file , or -o (default: source name plus '.vec')" << endl; + cerr << "--config-file , or -c (default: not use)" << endl; + cerr << "--mim-token-number , or -m (default: 30)" << endl; + cerr << "--stride , or -t (default: 1)" << endl; + cerr << "--merge-list-size (default: not used)" << endl; + cerr << "--start-line-number (default: all lines)" << endl; + cerr << "--end-line-number (default: same as start-line-number)" << endl; + cerr << "--help or -h" << endl; + cerr << "Later options can override previous options. The actual overriding order is undefined; pls avoid specifying the same parameter twice." << endl; + return 0; + default: + /* unreachable */; + abort(); + } + } + + // additional processing of options: + if(endline<0) + endline = startline; + + // more vec gen parameters (for backward compatibility only) + if (optind < argc) { + cerr << "Warning: unparsed options may be used to overload the parameters: [src file] [config-file | [min-tokens][stride][merge-list]]" << endl; + inputfilename = argv[optind++]; + if (optind < argc) + configfilename = argv[optind++]; + if (optind < argc) { + configfilename = NULL; + optind--; + if ( sscanf(argv[optind], "%d", &mergeTokens)<=0 ) { + cerr << "Error: invalid mergeTokens: " << argv[optind] << endl; + exit(1); + } + optind++; + } else { + TraGenMain::getParameters(configfilename, mergeTokens, mergeStride, mergeLists); + } + if (optind < argc) { + if ( sscanf(argv[optind], "%d", &mergeStride)<=0 ) { + cerr << "Error: invalid mergeStride: " << argv[optind] << endl; + exit(1); + } + optind++; + } + if (optind < argc) { + if ( sscanf(argv[optind], "%d", &mergeLists)<=0 ) { + cerr << "Error: invalid mergeLists: " << argv[optind] << endl; + exit(1); + } + optind++; + } + } + cerr << "Merging parameters: " << mergeTokens << ", " << mergeStride << ", " << mergeLists << endl; + + // parse the input file + yyin= fopen(inputfilename,"r"); + if (!yyin) { + cerr << "invalid filename: " << inputfilename << endl; + return 1; + } + id_init(); + yyparse(); + if (!root) { + cerr << "failed to parse file: " << inputfilename << endl; + return 65; + } + + root->lineRange(); + +#if 0 + cerr << "Terminal count: " << root->countTerminals() << endl; +#endif + + // prepare for vector generation + ParseTree p(root,id2name.size(),&id2name,&name2id); + + initNodes(p.relevantNodes, relNodes); + initNodes(p.leafNodes, atomicNodes); + initNodes(p.validParents, valParents); + //initNodes(p.mergeableNodes, mergeableNodes); + + p.filename = inputfilename; + + // for debugging use only in vector generator. + global_tree_for_debugging = &p; + cerr << "typeCount before init() = " << p.typeCount() << endl; + + // setup vec file + string outfilestring = outputfilename==NULL ? string(inputfilename) + ".vec" : outputfilename; + FILE * outfile = NULL; + outfile = fopen(outfilestring.c_str(), "w"); + if(outfile==NULL) { + cerr << "Warning: Can't open file for writing vectors; skip: " << outfilestring << endl; + return 65; + } + TraGenMain t(&p, mergeTokens, mergeStride, mergeLists, outfile); + t.run(startline, endline); + fclose(outfile); + global_tree_for_debugging = NULL; + + //root->printTok(); + //root->print(); + return 0; +} + diff --git a/src/main/out2html.C b/src/main/out2html.C new file mode 100755 index 0000000..cdc6b72 --- /dev/null +++ b/src/main/out2html.C @@ -0,0 +1,86 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include +#include +#include + +using namespace std; + +// These are not really used by "out2html"; just to resolve dependancy. +map name2id; +map id2name; +string identifierTypeName = "no need for such name"; + +/* read in a file line-by-line: + if the line is a valid clone point, transform it to html; + otherwise, simply output the line. */ +int main( int argc, char **argv ) +{ + if ( argc!=2 ) { + cerr << "usage: [a clone cluster file]" << endl; + return 1; + } + + id_init(); + TokenTreeMap::init_shared_data(); + ClonePointT tt; + ifstream inf(argv[1], ios::in); + if ( ! inf.is_open() ) { + cerr << "Can't open file: " << argv[1] << endl; + return 1; + } + + string line; + int linecount = 0; + + while ( !inf.eof() ) { + getline(inf, line); + linecount++; + char * charline = new char[line.length()+1]; + strcpy(charline, line.c_str()); + if ( strcmp(charline, "")==0 || strncmp(charline, "\n", 1)==0 ) + cout << "

" << endl << "
" << endl; + else if ( tt.parse(charline, TokenTreeMap::clone_patterns) ) { + tt.out2html0(cout); + } else + cout << tt << "
" << endl; + delete charline; + } + inf.close(); + + return 0; +} + diff --git a/src/main/out2xml.C b/src/main/out2xml.C new file mode 100755 index 0000000..183eafe --- /dev/null +++ b/src/main/out2xml.C @@ -0,0 +1,95 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include +#include +#include + +using namespace std; + +// These are not really used by "out2html"; just to resolve dependancy. +map name2id; +map id2name; +string identifierTypeName = "no need for such name"; + +/* read in a file line-by-line: + if the line is a valid clone point, transform it to html; + otherwise, simply output the line. */ +int main( int argc, char **argv ) +{ + if ( argc!=2 ) { + cerr << "usage: [a clone cluster file]" << endl; + return 1; + } + + id_init(); + TokenTreeMap::init_shared_data(); + ClonePointT tt; + ifstream inf(argv[1], ios::in); + if ( ! inf.is_open() ) { + cerr << "Can't open file: " << argv[1] << endl; + return 1; + } + + string line; + int linecount = 0; + int clonesetcount = 0; + + cout <<"" << endl; + cout <<""<"<"<"<< endl; + } + else if ( tt.parse(charline, TokenTreeMap::clone_patterns) ) + { + cout <<""<"<"< + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include +#include + +using namespace std; + +map name2id; +map id2name; +#ifdef JAVA +string identifierTypeName = "ID_TK"; +#else +#ifdef PHP +string identifierTypeName = "T_VARIABLE"; +#else +string identifierTypeName = "IDENTIFIER"; +#endif +#endif + +template bool from_string(T& t, + const std::string& s, + std::ios_base& (*f)(std::ios_base&)) +{ + std::istringstream iss(s); + return !(iss >> f >> t).fail(); +} + +int main( int argc, char **argv ) +{ + if ( argc<4 ) { + cerr << "Usage: " << argv[0] << " [contextual node level [overide files]]" << endl; + return 1; + } + + FILE * fin = NULL; + if ( (fin=fopen(argv[1], "r"))==NULL ) { + cerr << "Can't open file: " << argv[1] << endl; + return 2; + } + fclose(fin); + fin = NULL; + +#ifdef JAVA + const char * file_suffix = ".java"; +#else +#ifdef PHP + const char * file_suffix = ".php"; +#else + const char * file_suffix = ".c"; +#endif +#endif + string fn = string(argv[1]); + unsigned int sn = fn.rfind(file_suffix); + if ( sn==string::npos || sn+strlen(file_suffix)" << tbid << endl; + return 1; + } + if(!from_string(teid, string(argv[3]), dec) || teid<=0 ) { + cerr << "Error: end token id incorrect: " << argv[3] << "-->" << teid << endl; + return 1; + } + + long contextlevel = 0; // 0 means no context + if( argc>=5 ) { + if( !from_string(contextlevel, string(argv[4]), dec) || contextlevel<0 ) { + cerr << "Error: context level incorrect: " << argv[4] << "-->" << contextlevel << endl; + return 1; + } else if ( contextlevel>1 ) { + cerr << "Warning: only 1-level context is valid for now" << endl; + contextlevel = 1; + } + } + + id_init(); + + ParseTree* pt = parseFile(argv[1]); + if ( pt==NULL ) { + cerr << "Error: no parse tree created for file: " << argv[1] << endl; + return 1; + } + + if(argc>=6) { + pt->dumpParseTree(true); // to overide existing file + } else { + pt->dumpParseTree(false); + } + + Tree* node = pt->tokenRange2Tree(tbid, teid); + long i = pt->tree2sn(node); + if(i<=0) { + cerr << "Warining: incorrect tree node order number: " << endl; + } + cout << pt->filename << " " << tbid << " " << teid << " " << i; + if( contextlevel>0 ) { + Tree* contextnode = pt->getContextualNode(node); + long ci = pt->tree2sn(contextnode); + cout << " " << ci; + } + cout << endl; + + return 0; +} + diff --git a/src/main/ptree.cc b/src/main/ptree.cc new file mode 100755 index 0000000..edf6c53 --- /dev/null +++ b/src/main/ptree.cc @@ -0,0 +1,423 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include + +using namespace std; + +ParseTree::ParseTree(Tree *root, int nTypes, + map *typeNames, + map *typeIds ) +{ + this->root= root; + this->nTypes= nTypes; + this->typeNames= typeNames; + this->typeIDs= typeIds; +} + +ParseTree::~ParseTree() +{ + if ( root!=NULL ) { + delete root; + root = NULL; + } + // TODO: possible mem leak on typeNames, typeIds +} + +Tree *ParseTree::getRoot() +{ + return root; +} + +int ParseTree::typeCount() +{ + return nTypes; +} + +const string & ParseTree::getTypeName(int id) +{ + assert(id=0); + map::iterator i= typeNames->find(id); + if (i== typeNames->end()) { + throw "not found"; + } else { + return i->second; + } +} + +int ParseTree::getTypeID( const string &name) +{ + map::iterator i= typeIDs->find(name); + + if (i== typeIDs->end()) { + return -1; + } else { + return i->second; + } +} + +bool ParseTree::dumpParseTree(bool toOveride) +{ + ifstream inp; + ofstream out; + string outputfn = filename + ".grp"; + + // prepare the output file: + if(!toOveride) { + inp.open(outputfn.c_str(), ifstream::in); + inp.close(); + if(!inp.fail()) { + cerr << "Warning: parse tree dump file exists already: " << outputfn << " ...skip" << endl; + return false; + } + inp.clear(ios::failbit); + } + out.open(outputfn.c_str(), ofstream::out); + if(out.fail()) { + cerr << "Error: cannot open dump file: " << outputfn << endl; + return false; + } + + // dump the tree to the file: + out << "# " << filename << endl; + long ncount = 1; + ncount = root->dumpTree(out, ncount); + + // close the file: + out.close(); + return true; +} + +Tree* ParseTree::line2Tree(int ln) +{ + // precondition: the line range of each node is set, and the line range of a parent node contains the line ranges of all of its children + // Note that there may be more than one token in the same line + return line2Tree(ln, ln); +} + +Tree* ParseTree::line2Tree(int startln, int endln) +{ + // precondition: the line range of each node is set, and the line range of a parent node contains the line ranges of all of its children + if(startln>endln || startln<0) + return NULL; + if(startln==0) + return root; + //else + Tree* itr = root; + while(itr!=NULL) { +// cerr << "children size: " << itr->children.size() << ". Comparing with line range: " << itr->min << ":" << itr->max << endl; + if(itr->min<=endln && startln<=itr->max) { + int inRangeCount = 0; + Tree* inRangeNode = NULL; + for(int i=0; ichildren.size(); i++) { +// cerr << "comparing with children " << i << "'s line range: " << itr->children[i]->min << ":" << itr->children[i]->max << endl; + if(itr->children[i]->min<=endln && startln<=itr->children[i]->max) { + inRangeCount++; + if(inRangeCount>=2) + break; + inRangeNode = itr->children[i]; + } + } + if(inRangeCount==1) + itr = inRangeNode; + else + break; + } else + itr = NULL; + } + return itr; +} + +Tree* ParseTree::tokenRange2Tree(long startTokenId, long endTokenId) +{ + if ( root == NULL ) + return NULL; + + list* path1 = root2Token(startTokenId); + list* path2 = root2Token(endTokenId); + Tree * rsl = NULL; + + if ( path1==NULL || path2==NULL ) + rsl = root; + else { + // compare the two paths starting from the root: + list::iterator itr1 = path1->begin(); + list::iterator itr2 = path2->begin(); + for (; itr1 != path1->end() && itr2 != path2->end(); ++itr1, ++itr2) { + if ( (*itr1) != (*itr2) ) // find the smallest common ancestor + break; + rsl = (*itr1); + } + // print the paths for debugging: +#define print_path_list(path_name) \ + for (list::iterator path_itr = (path_name)->begin(); \ + path_itr != (path_name)->end(); ++path_itr) \ + cerr << "NODE: %X" << (*path_itr) << " type:" << getTypeName(id2name, (*path_itr)->type) << endl; + /* print_path_list(path1); + cerr << endl; + print_path_list(path2); */ + } + if ( rsl == NULL ) + rsl = root; + + if ( path1!=NULL ) + delete path1; + if ( path2!=NULL ) + delete path2; + + return rsl; // shouldn't return NULL here. +} + +Tree* ParseTree::getContextualNode(Tree* node) +{ + // assert 'node' in this parse tree + if ( node==NULL ) + return root; + + map::iterator attr_itr = node->attributes.find(NODE_TOKEN_ID); + assert ( attr_itr != node->attributes.end() ); + pair* startrange = (pair*)(*attr_itr).second; + if (node->parent==NULL) + return root; + + Tree* startnode = node->parent; + while ( startnode!=NULL ) { + if ( isContextualNode(startnode) ) { // this condition is language-dependant + break; + } else + startnode = startnode->parent; + } + if ( startnode==NULL ) + return root; + return startnode; +} + +Tree* ParseTree::getContextualNode(long startTokenId, long endTokenId) +{ + Tree* node = tokenRange2Tree(startTokenId, endTokenId); + return getContextualNode(node); +} + +list* ParseTree::root2Token(long tid) +{ + list* path = new list(); + root2TokenAux(tid, root, *path); + if ( path->empty() ) { + delete path; + path = NULL; + } + return path; +} + +bool ParseTree::root2TokenAux(long tid, Tree* node, list& path) +{ + if ( node == NULL ) + return false; + + map::iterator attr_itr = node->attributes.find(NODE_TOKEN_ID); + assert( attr_itr!=node->attributes.end() ); + const pair* tokens = (pair*)(*attr_itr).second; + if ( tid >= tokens->first && tid <= tokens->second ) { + // add the containing node to the path: + path.push_back(node); + // select a child with the containing range: + for (int i=0; i < node->children.size(); i++) { + if ( root2TokenAux(tid, node->children[i], path) ) + return true; // within range. + } // the path must exist and be unique (it's true for well-formed + // ASTs), otherwise the code may return a wrong list. + return false; // out of range. should NOT be reachable for well-formed ASTs. + } else + return false; // out of range. +} + +long ParseTree::tree2sn(Tree* nd) +{ + if (root == NULL || nd == NULL) + return -1; + + long n = 1; + n = root->tree2sn(nd, n); + + return n; +} + + +long Tree::dumpTree(ofstream & out, long n) +{ + long c = n++; + out << "n " << c << " " << getTypeName(id2name, type) << endl; + for (int i= 0; i < children.size(); i++) { + out << "e " << c << " " << n << endl; + n = children[i]->dumpTree(out, n); + } + return n; +} + +long Tree::tree2sn(Tree* t, long& n) +{ + long c = n++; + if(this==t) + return c; + + for (int i= 0; i < children.size(); i++) { + c = children[i]->tree2sn(t, n); + if(c>0) + return c; + } + return -1; +} + +/* Valid type ids range from 0 to typeCount-1 */ +int typeCount(map& id2name) +{ + return id2name.size(); +} + +int typeCount(map& name2id) +{ + return name2id.size(); +} + +const string & getTypeName(map& id2name, int id) +{ + assert( id=0 ); + map::iterator i= id2name.find(id); + if (i == id2name.end()) { + throw "not found"; + } else { + return i->second; + } +} + +int getTypeID(map& name2id, const string& name) +{ + map::iterator i= name2id.find(name); + if (i == name2id.end()) { + return -1; + } else { + return i->second; + } +} + + +bool compareTree(Tree* t1, Tree * t2) +{ + if ( t1==NULL && t2==NULL ) + return true; + else if ( t1==NULL || t2==NULL ) + return false; + else if ( t1->type != t2->type ) + return false; + else if ( t1->children.size() != t2->children.size() ) + return false; + else { + for ( int i=0; i < t1->children.size(); i++ ) { + if ( compareTree(t1->children[i], t2->children[i]) ) + continue; + else + return false; + } + return true; + } +} + +ParseTree* parseFile(const char * fn) +{ + // NOTE: yyparse is NOT re-entrant, which may cause unexpected behavior if called for more than one files. + yyin = fopen(fn, "r"); + if (!yyin) { + cerr << "Error: Can't open file for yyin: " << fn << endl; + } + yyrestart(yyin); // This may be unnecessary because BISON's manual + // says this is equivalent to (but I doubt it) + // changing yyin directly. + yyparse(); + fclose(yyin); + yyin = NULL; + if (!root) { + cerr << "Error: failed to parse file: " << fn << endl; + return NULL; + } + + Tree* initial_inh = NULL; + root->lineRange(); + ParseTree* pt = new ParseTree(root, id2name.size(), &id2name, &name2id); + TraGenConfiguration * vecGen_config = new TraGenConfiguration((const char*)NULL); // no actual use; just a dummy + TokenRangeCounter * token_range_counter = new TokenRangeCounter(*vecGen_config); + token_range_counter->reinit(); + token_range_counter->traverse(root, initial_inh); + pt->filename = fn; + return pt; +} + +static vector ctxNodes; /* internal use only: ctxNodes[i]==true iff the node kind is considered as contexts */ +static const char * contextualNodes[] = { +#ifdef JAVA +#include "../ptgen/java/jcontextualNodes.h" +#else +#ifdef PHP +#include "../ptgen/php5/phpcontextualNodes.h" +#else +#include "../ptgen/gcc/ccontextualNodes.h" +#endif +#endif +}; + +bool setContextualNodes() /* internal use only */ +{ + bool errflag = false; + assert ( name2id.size() > 0 ); + ctxNodes = vector(id2name.size(), false); + for (const char **s= contextualNodes; *s != NULL; s++) { + map::iterator i= name2id.find(*s); + if (i == name2id.end()) { + cerr << "unknown node type when setting contextual nodes: " << *s << endl; + errflag = true; + continue; + } + ctxNodes[i->second] = true; + } + return errflag; +} + +bool isContextualNode(Tree* node) +{ + assert( node->type >= 0 && node->type < id2name.size() ); + if ( ctxNodes.empty() ) + setContextualNodes(); + return ctxNodes[node->type]; +} + diff --git a/src/ptgen/Makefile b/src/ptgen/Makefile new file mode 100644 index 0000000..f1371d2 --- /dev/null +++ b/src/ptgen/Makefile @@ -0,0 +1,47 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +TARGET: + make -C gcc + make -C java + make -C php5 + +.PHONY: clean +clean: + rm -f *.pyc + make -C simple clean + make -C gcc clean + make -C java clean + make -C php5 clean + + diff --git a/src/ptgen/YaccLexer.py b/src/ptgen/YaccLexer.py new file mode 100755 index 0000000..59e5657 --- /dev/null +++ b/src/ptgen/YaccLexer.py @@ -0,0 +1,483 @@ +### $ANTLR 2.7.6 (2005-12-22): "yacc.g" -> "YaccLexer.py"$ +### import antlr and other modules .. +import sys +import antlr + +version = sys.version.split()[0] +if version < '2.2.1': + False = 0 +if version < '2.3': + True = not False +### header action >>> + +### header action <<< +### preamble action >>> + +### preamble action <<< +### >>>The Literals<<< +literals = {} + + +### import antlr.Token +from antlr import Token +### >>>The Known Token Types <<< +SKIP = antlr.SKIP +INVALID_TYPE = antlr.INVALID_TYPE +EOF_TYPE = antlr.EOF_TYPE +EOF = antlr.EOF +NULL_TREE_LOOKAHEAD = antlr.NULL_TREE_LOOKAHEAD +MIN_USER_TYPE = antlr.MIN_USER_TYPE +ID = 4 +COLON = 5 +SEMICOLON = 6 +CHAR = 7 +STRING = 8 +ERROR = 9 +PREC = 10 +ACTION = 11 +OR = 12 +HYPHEN = 13 +CARROT = 14 +BANG = 15 +LETTER = 16 +DIGIT = 17 +COMMENT = 18 +WS = 19 + +class Lexer(antlr.CharScanner) : + ### user action >>> + ### user action <<< + def __init__(self, *argv, **kwargs) : + antlr.CharScanner.__init__(self, *argv, **kwargs) + self.caseSensitiveLiterals = True + self.setCaseSensitive(True) + self.literals = literals + + def nextToken(self): + while True: + try: ### try again .. + while True: + _token = None + _ttype = INVALID_TYPE + self.resetText() + try: ## for char stream error handling + try: ##for lexical error handling + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in u':': + pass + self.mCOLON(True) + theRetToken = self._returnToken + elif la1 and la1 in u';': + pass + self.mSEMICOLON(True) + theRetToken = self._returnToken + elif la1 and la1 in u'-': + pass + self.mHYPHEN(True) + theRetToken = self._returnToken + elif la1 and la1 in u'^': + pass + self.mCARROT(True) + theRetToken = self._returnToken + elif la1 and la1 in u'!': + pass + self.mBANG(True) + theRetToken = self._returnToken + elif la1 and la1 in u'|': + pass + self.mOR(True) + theRetToken = self._returnToken + elif la1 and la1 in u'%': + pass + self.mPREC(True) + theRetToken = self._returnToken + elif la1 and la1 in u'\'': + pass + self.mCHAR(True) + theRetToken = self._returnToken + elif la1 and la1 in u'"': + pass + self.mSTRING(True) + theRetToken = self._returnToken + elif la1 and la1 in u'{': + pass + self.mACTION(True) + theRetToken = self._returnToken + elif la1 and la1 in u'/': + pass + self.mCOMMENT(True) + theRetToken = self._returnToken + elif la1 and la1 in u'\t\n\r ': + pass + self.mWS(True) + theRetToken = self._returnToken + else: + if (self.LA(1)==u'e') and (self.LA(2)==u'r') and (self.LA(3)==u'r') and (self.LA(4)==u'o') and (self.LA(5)==u'r') and (True) and (True): + pass + self.mERROR(True) + theRetToken = self._returnToken + elif (_tokenSet_0.member(self.LA(1))) and (True) and (True) and (True) and (True) and (True) and (True): + pass + self.mID(True) + theRetToken = self._returnToken + else: + self.default(self.LA(1)) + + if not self._returnToken: + raise antlr.TryAgain ### found SKIP token + ### option { testLiterals=true } + self.testForLiteral(self._returnToken) + ### return token to caller + return self._returnToken + ### handle lexical errors .... + except antlr.RecognitionException, e: + raise antlr.TokenStreamRecognitionException(e) + ### handle char stream errors ... + except antlr.CharStreamException,cse: + if isinstance(cse, antlr.CharStreamIOException): + raise antlr.TokenStreamIOException(cse.io) + else: + raise antlr.TokenStreamException(str(cse)) + except antlr.TryAgain: + pass + + def mCOLON(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = COLON + _saveIndex = 0 + pass + self.match(':') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mSEMICOLON(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = SEMICOLON + _saveIndex = 0 + pass + self.match(';') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mHYPHEN(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = HYPHEN + _saveIndex = 0 + pass + self.match('-') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mCARROT(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = CARROT + _saveIndex = 0 + pass + self.match('^') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mBANG(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = BANG + _saveIndex = 0 + pass + self.match('!') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mOR(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = OR + _saveIndex = 0 + pass + self.match('|') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mPREC(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = PREC + _saveIndex = 0 + pass + self.match("%prec") + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mERROR(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = ERROR + _saveIndex = 0 + pass + self.match("error") + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mID(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = ID + _saveIndex = 0 + pass + self.mLETTER(False) + while True: + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in u'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz': + pass + self.mLETTER(False) + elif la1 and la1 in u'0123456789': + pass + self.mDIGIT(False) + elif la1 and la1 in u'_': + pass + self.match('_') + elif la1 and la1 in u'.': + pass + self.match('.') + else: + break + + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mLETTER(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = LETTER + _saveIndex = 0 + pass + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in u'abcdefghijklmnopqrstuvwxyz': + pass + self.matchRange(u'a', u'z') + elif la1 and la1 in u'ABCDEFGHIJKLMNOPQRSTUVWXYZ': + pass + self.matchRange(u'A', u'Z') + else: + self.raise_NoViableAlt(self.LA(1)) + + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mDIGIT(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = DIGIT + _saveIndex = 0 + pass + pass + self.matchRange(u'0', u'9') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mCHAR(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = CHAR + _saveIndex = 0 + pass + self.match('\'') + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in u'\\': + pass + pass + self.match('\\') + self.matchNot(antlr.EOF_CHAR) + elif la1 and la1 in u'\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\t\n\u000b\u000c\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f': + pass + pass + self.match(_tokenSet_1) + else: + self.raise_NoViableAlt(self.LA(1)) + + self.match('\'') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mSTRING(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = STRING + _saveIndex = 0 + pass + self.match('"') + while True: + if (_tokenSet_2.member(self.LA(1))): + pass + self.matchNot('"') + else: + break + + self.match('"') + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mACTION(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = ACTION + _saveIndex = 0 + pass + self.match('{') + lcount= 1 + incomment= False + indquote= False + insquote= False + while lcount != 0: + if self.LA(1) == '\\': + self.consume() + elif self.LA(1) == '/' and self.LA(2) == '*': + if not indquote and not insquote: + incomment= True + self.consume() + elif self.LA(1) == '*' and self.LA(2) == '/': + if not indquote and not insquote: + incomment= False + self.consume() + elif self.LA(1) == '\'': + if not indquote and not incomment: + insquote= not insquote + elif self.LA(1) == '"': + if not insquote and not incomment: + indquote= not indquote + elif self.LA(1) == antlr.EOF: + _ttype = antlr.EOF + elif self.LA(1) == '\n': + self.newline() + elif not indquote and not insquote and not incomment: + if self.LA(1)== '}': + lcount -= 1 + elif self.LA(1)== '{': + lcount += 1 + self.consume() + _ttype = antlr.SKIP; + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mCOMMENT(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = COMMENT + _saveIndex = 0 + pass + self.match("/*") + while True: + if (self.LA(1)==u'*') and (_tokenSet_3.member(self.LA(2))): + pass + pass + self.match('*') + self.match(_tokenSet_3) + elif (_tokenSet_4.member(self.LA(1))): + pass + self.match(_tokenSet_4) + else: + break + + self.match("*/") + _ttype = antlr.SKIP + self.set_return_token(_createToken, _token, _ttype, _begin) + + def mWS(self, _createToken): + _ttype = 0 + _token = None + _begin = self.text.length() + _ttype = WS + _saveIndex = 0 + pass + _cnt50= 0 + while True: + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in u' ': + pass + self.match(' ') + elif la1 and la1 in u'\r': + pass + self.match('\r') + self.match('\n') + self.newline() + elif la1 and la1 in u'\n': + pass + self.match('\n') + self.newline() + elif la1 and la1 in u'\t': + pass + self.match('\t') + else: + break + + _cnt50 += 1 + if _cnt50 < 1: + self.raise_NoViableAlt(self.LA(1)) + _ttype = antlr.SKIP + self.set_return_token(_createToken, _token, _ttype, _begin) + + + +### generate bit set +def mk_tokenSet_0(): + ### var1 + data = [ 0L, 576460743847706622L, 0L, 0L] + return data +_tokenSet_0 = antlr.BitSet(mk_tokenSet_0()) + +### generate bit set +def mk_tokenSet_1(): + ### var1 + data = [ -549755813889L, -268435457L, 0L, 0L] + return data +_tokenSet_1 = antlr.BitSet(mk_tokenSet_1()) + +### generate bit set +def mk_tokenSet_2(): + ### var1 + data = [ -17179869185L, -1L, 0L, 0L] + return data +_tokenSet_2 = antlr.BitSet(mk_tokenSet_2()) + +### generate bit set +def mk_tokenSet_3(): + ### var1 + data = [ -140737488355329L, -1L, 0L, 0L] + return data +_tokenSet_3 = antlr.BitSet(mk_tokenSet_3()) + +### generate bit set +def mk_tokenSet_4(): + ### var1 + data = [ -4398046511105L, -1L, 0L, 0L] + return data +_tokenSet_4 = antlr.BitSet(mk_tokenSet_4()) + +### __main__ header action >>> +if __name__ == '__main__' : + import sys + import antlr + import YaccLexer + + ### create lexer - shall read from stdin + try: + for token in YaccLexer.Lexer(): + print token + + except antlr.TokenStreamException, e: + print "error: exception caught while lexing: ", e +### __main__ header action <<< diff --git a/src/ptgen/YaccParser.py b/src/ptgen/YaccParser.py new file mode 100755 index 0000000..b918299 --- /dev/null +++ b/src/ptgen/YaccParser.py @@ -0,0 +1,269 @@ +### $ANTLR 2.7.6 (2005-12-22): "yacc.g" -> "YaccParser.py"$ +### import antlr and other modules .. +import sys +import antlr + +version = sys.version.split()[0] +if version < '2.2.1': + False = 0 +if version < '2.3': + True = not False +### header action >>> + +### header action <<< +### preamble action>>> + +### preamble action <<< + +### import antlr.Token +from antlr import Token +### >>>The Known Token Types <<< +SKIP = antlr.SKIP +INVALID_TYPE = antlr.INVALID_TYPE +EOF_TYPE = antlr.EOF_TYPE +EOF = antlr.EOF +NULL_TREE_LOOKAHEAD = antlr.NULL_TREE_LOOKAHEAD +MIN_USER_TYPE = antlr.MIN_USER_TYPE +ID = 4 +COLON = 5 +SEMICOLON = 6 +CHAR = 7 +STRING = 8 +ERROR = 9 +PREC = 10 +ACTION = 11 +OR = 12 +HYPHEN = 13 +CARROT = 14 +BANG = 15 +LETTER = 16 +DIGIT = 17 +COMMENT = 18 +WS = 19 + +class Parser(antlr.LLkParser): + ### user action >>> + ### user action <<< + + def __init__(self, *args, **kwargs): + antlr.LLkParser.__init__(self, *args, **kwargs) + self.tokenNames = _tokenNames + ### __init__ header action >>> + self.NonTerminals= set([]) + self.Terminals= set([]) + self.Rules=[] + ### __init__ header action <<< + + def grammar(self): + + try: ## for error handling + pass + _cnt3= 0 + while True: + if (self.LA(1)==ID): + pass + self.rule() + else: + break + + _cnt3 += 1 + if _cnt3 < 1: + raise antlr.NoViableAltException(self.LT(1), self.getFilename()) + self.match(EOF_TYPE) + + except antlr.RecognitionException, ex: + self.reportError(ex) + self.consume() + self.consumeUntil(_tokenSet_0) + + + def rule(self): + + id = None + try: ## for error handling + pass + pass + id = self.LT(1) + self.match(ID) + self.NonTerminals.add(id.getText()) + self.match(COLON) + self.rhs(id.getText()) + self.match(SEMICOLON) + + except antlr.RecognitionException, ex: + self.reportError(ex) + self.consume() + self.consumeUntil(_tokenSet_1) + + + def rhs(self, + lhs + ): + + id = None + c = None + str = None + pi = None + pc = None + right=[] + try: ## for error handling + pass + while True: + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in [ID]: + pass + pass + id = self.LT(1) + self.match(ID) + right.append(("node",id.getText())) + if id.getText() == id.getText().lower(): + self.NonTerminals.add(id.getText()) + else: + self.Terminals.add(id.getText()) + elif la1 and la1 in [CHAR]: + pass + pass + c = self.LT(1) + self.match(CHAR) + right.append(("node",c.getText())) + self.Terminals.add(c.getText()) + elif la1 and la1 in [STRING]: + pass + pass + str = self.LT(1) + self.match(STRING) + right.append(("node",str.getText())) + self.Terminals.add(str.getText()) + elif la1 and la1 in [ERROR]: + pass + self.match(ERROR) + right.append(("error","error")) + elif la1 and la1 in [PREC]: + pass + self.match(PREC) + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in [ID]: + pass + pass + pi = self.LT(1) + self.match(ID) + right.append(("%prec","%prec "+pi.getText())) + elif la1 and la1 in [CHAR]: + pass + pass + pc = self.LT(1) + self.match(CHAR) + right.append(("%prec","%prec "+pc.getText())) + else: + raise antlr.NoViableAltException(self.LT(1), self.getFilename()) + + elif la1 and la1 in [ACTION]: + pass + self.match(ACTION) + else: + break + + self.Rules.append( (lhs,right) ) + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in [OR]: + pass + self.match(OR) + self.rhs(lhs) + elif la1 and la1 in [SEMICOLON]: + pass + else: + raise antlr.NoViableAltException(self.LT(1), self.getFilename()) + + + except antlr.RecognitionException, ex: + self.reportError(ex) + self.consume() + self.consumeUntil(_tokenSet_2) + + + def rulespec(self): + + try: ## for error handling + pass + self.match(HYPHEN) + + except antlr.RecognitionException, ex: + self.reportError(ex) + self.consume() + self.consumeUntil(_tokenSet_0) + + + def treespec(self): + + try: ## for error handling + la1 = self.LA(1) + if False: + pass + elif la1 and la1 in [CARROT]: + pass + self.match(CARROT) + elif la1 and la1 in [BANG]: + pass + self.match(BANG) + else: + raise antlr.NoViableAltException(self.LT(1), self.getFilename()) + + + except antlr.RecognitionException, ex: + self.reportError(ex) + self.consume() + self.consumeUntil(_tokenSet_0) + + + +_tokenNames = [ + "<0>", + "EOF", + "<2>", + "NULL_TREE_LOOKAHEAD", + "ID", + "COLON", + "SEMICOLON", + "CHAR", + "STRING", + "ERROR", + "PREC", + "ACTION", + "OR", + "HYPHEN", + "CARROT", + "BANG", + "LETTER", + "DIGIT", + "COMMENT", + "WS" +] + + +### generate bit set +def mk_tokenSet_0(): + ### var1 + data = [ 2L, 0L] + return data +_tokenSet_0 = antlr.BitSet(mk_tokenSet_0()) + +### generate bit set +def mk_tokenSet_1(): + ### var1 + data = [ 18L, 0L] + return data +_tokenSet_1 = antlr.BitSet(mk_tokenSet_1()) + +### generate bit set +def mk_tokenSet_2(): + ### var1 + data = [ 64L, 0L] + return data +_tokenSet_2 = antlr.BitSet(mk_tokenSet_2()) + diff --git a/src/ptgen/antlr.py b/src/ptgen/antlr.py new file mode 100755 index 0000000..3adee3e --- /dev/null +++ b/src/ptgen/antlr.py @@ -0,0 +1,2833 @@ +## This file is part of PyANTLR. See LICENSE.txt for license +## details..........Copyright (C) Wolfgang Haefelinger, 2004. + +## get sys module +import sys + +version = sys.version.split()[0] +if version < '2.2.1': + False = 0 +if version < '2.3': + True = not False + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### global symbols ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +### ANTLR Standard Tokens +SKIP = -1 +INVALID_TYPE = 0 +EOF_TYPE = 1 +EOF = 1 +NULL_TREE_LOOKAHEAD = 3 +MIN_USER_TYPE = 4 + +### ANTLR's EOF Symbol +EOF_CHAR = '' + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### general functions ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +## Version should be automatically derived from configure.in. For now, +## we need to bump it ourselfs. Don't remove the tags. +## +def version(): + r = { + 'major' : '2', + 'minor' : '7', + 'micro' : '5', + 'patch' : '' , + 'version': '2.7.5' + } + return r +## + +def error(fmt,*args): + if fmt: + print "error: ", fmt % tuple(args) + +def ifelse(cond,_then,_else): + if cond : + r = _then + else: + r = _else + return r + +def is_string_type(x): + return (isinstance(x,str) or isinstance(x,unicode)) + +def assert_string_type(x): + assert is_string_type(x) + pass + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### ANTLR Exceptions ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class ANTLRException(Exception): + + def __init__(self, *args): + Exception.__init__(self, *args) + + +class RecognitionException(ANTLRException): + + def __init__(self, *args): + ANTLRException.__init__(self, *args) + self.fileName = None + self.line = -1 + self.column = -1 + if len(args) >= 2: + self.fileName = args[1] + if len(args) >= 3: + self.line = args[2] + if len(args) >= 4: + self.column = args[3] + + def __str__(self): + buf = [''] + if self.fileName: + buf.append(self.fileName + ":") + if self.line != -1: + if not self.fileName: + buf.append("line ") + buf.append(str(self.line)) + if self.column != -1: + buf.append(":" + str(self.column)) + buf.append(":") + buf.append(" ") + return str('').join(buf) + + __repr__ = __str__ + + +class NoViableAltException(RecognitionException): + + def __init__(self, *args): + RecognitionException.__init__(self, *args) + self.token = None + self.node = None + if isinstance(args[0],AST): + self.node = args[0] + elif isinstance(args[0],Token): + self.token = args[0] + else: + raise TypeError("NoViableAltException requires Token or AST argument") + + def __str__(self): + if self.token: + line = self.token.getLine() + col = self.token.getColumn() + text = self.token.getText() + return "unexpected symbol at line %s (column %s): \"%s\"" % (line,col,text) + if self.node == ASTNULL: + return "unexpected end of subtree" + assert self.node + ### hackish, we assume that an AST contains method getText + return "unexpected node: %s" % (self.node.getText()) + + __repr__ = __str__ + + +class NoViableAltForCharException(RecognitionException): + + def __init__(self, *args): + self.foundChar = None + if len(args) == 2: + self.foundChar = args[0] + scanner = args[1] + RecognitionException.__init__(self, "NoViableAlt", + scanner.getFilename(), + scanner.getLine(), + scanner.getColumn()) + elif len(args) == 4: + self.foundChar = args[0] + fileName = args[1] + line = args[2] + column = args[3] + RecognitionException.__init__(self, "NoViableAlt", + fileName, line, column) + else: + RecognitionException.__init__(self, "NoViableAlt", + '', -1, -1) + + def __str__(self): + mesg = "unexpected char: " + if self.foundChar >= ' ' and self.foundChar <= '~': + mesg += "'" + self.foundChar + "'" + elif self.foundChar: + mesg += "0x" + hex(ord(self.foundChar)).upper()[2:] + else: + mesg += "" + return mesg + + __repr__ = __str__ + + +class SemanticException(RecognitionException): + + def __init__(self, *args): + RecognitionException.__init__(self, *args) + + +class MismatchedCharException(RecognitionException): + + NONE = 0 + CHAR = 1 + NOT_CHAR = 2 + RANGE = 3 + NOT_RANGE = 4 + SET = 5 + NOT_SET = 6 + + def __init__(self, *args): + self.args = args + if len(args) == 5: + # Expected range / not range + if args[3]: + self.mismatchType = MismatchedCharException.NOT_RANGE + else: + self.mismatchType = MismatchedCharException.RANGE + self.foundChar = args[0] + self.expecting = args[1] + self.upper = args[2] + self.scanner = args[4] + RecognitionException.__init__(self, "Mismatched char range", + self.scanner.getFilename(), + self.scanner.getLine(), + self.scanner.getColumn()) + elif len(args) == 4 and is_string_type(args[1]): + # Expected char / not char + if args[2]: + self.mismatchType = MismatchedCharException.NOT_CHAR + else: + self.mismatchType = MismatchedCharException.CHAR + self.foundChar = args[0] + self.expecting = args[1] + self.scanner = args[3] + RecognitionException.__init__(self, "Mismatched char", + self.scanner.getFilename(), + self.scanner.getLine(), + self.scanner.getColumn()) + elif len(args) == 4 and isinstance(args[1], BitSet): + # Expected BitSet / not BitSet + if args[2]: + self.mismatchType = MismatchedCharException.NOT_SET + else: + self.mismatchType = MismatchedCharException.SET + self.foundChar = args[0] + self.set = args[1] + self.scanner = args[3] + RecognitionException.__init__(self, "Mismatched char set", + self.scanner.getFilename(), + self.scanner.getLine(), + self.scanner.getColumn()) + else: + self.mismatchType = MismatchedCharException.NONE + RecognitionException.__init__(self, "Mismatched char") + + ## Append a char to the msg buffer. If special, + # then show escaped version + # + def appendCharName(self, sb, c): + if not c or c == 65535: + # 65535 = (char) -1 = EOF + sb.append("''") + elif c == '\n': + sb.append("'\\n'") + elif c == '\r': + sb.append("'\\r'"); + elif c == '\t': + sb.append("'\\t'") + else: + sb.append('\'' + c + '\'') + + ## + # Returns an error message with line number/column information + # + def __str__(self): + sb = [''] + sb.append(RecognitionException.__str__(self)) + + if self.mismatchType == MismatchedCharException.CHAR: + sb.append("expecting ") + self.appendCharName(sb, self.expecting) + sb.append(", found ") + self.appendCharName(sb, self.foundChar) + elif self.mismatchType == MismatchedCharException.NOT_CHAR: + sb.append("expecting anything but '") + self.appendCharName(sb, self.expecting) + sb.append("'; got it anyway") + elif self.mismatchType in [MismatchedCharException.RANGE, MismatchedCharException.NOT_RANGE]: + sb.append("expecting char ") + if self.mismatchType == MismatchedCharException.NOT_RANGE: + sb.append("NOT ") + sb.append("in range: ") + appendCharName(sb, self.expecting) + sb.append("..") + appendCharName(sb, self.upper) + sb.append(", found ") + appendCharName(sb, self.foundChar) + elif self.mismatchType in [MismatchedCharException.SET, MismatchedCharException.NOT_SET]: + sb.append("expecting ") + if self.mismatchType == MismatchedCharException.NOT_SET: + sb.append("NOT ") + sb.append("one of (") + for i in range(len(self.set)): + self.appendCharName(sb, self.set[i]) + sb.append("), found ") + self.appendCharName(sb, self.foundChar) + + return str().join(sb).strip() + + __repr__ = __str__ + + +class MismatchedTokenException(RecognitionException): + + NONE = 0 + TOKEN = 1 + NOT_TOKEN = 2 + RANGE = 3 + NOT_RANGE = 4 + SET = 5 + NOT_SET = 6 + + def __init__(self, *args): + self.args = args + self.tokenNames = [] + self.token = None + self.tokenText = '' + self.node = None + if len(args) == 6: + # Expected range / not range + if args[3]: + self.mismatchType = MismatchedTokenException.NOT_RANGE + else: + self.mismatchType = MismatchedTokenException.RANGE + self.tokenNames = args[0] + self.expecting = args[2] + self.upper = args[3] + self.fileName = args[5] + + elif len(args) == 4 and isinstance(args[2], int): + # Expected token / not token + if args[3]: + self.mismatchType = MismatchedTokenException.NOT_TOKEN + else: + self.mismatchType = MismatchedTokenException.TOKEN + self.tokenNames = args[0] + self.expecting = args[2] + + elif len(args) == 4 and isinstance(args[2], BitSet): + # Expected BitSet / not BitSet + if args[3]: + self.mismatchType = MismatchedTokenException.NOT_SET + else: + self.mismatchType = MismatchedTokenException.SET + self.tokenNames = args[0] + self.set = args[2] + + else: + self.mismatchType = MismatchedTokenException.NONE + RecognitionException.__init__(self, "Mismatched Token: expecting any AST node", "", -1, -1) + + if len(args) >= 2: + if isinstance(args[1],Token): + self.token = args[1] + self.tokenText = self.token.getText() + RecognitionException.__init__(self, "Mismatched Token", + self.fileName, + self.token.getLine(), + self.token.getColumn()) + elif isinstance(args[1],AST): + self.node = args[1] + self.tokenText = str(self.node) + RecognitionException.__init__(self, "Mismatched Token", + "", + self.node.getLine(), + self.node.getColumn()) + else: + self.tokenText = "" + RecognitionException.__init__(self, "Mismatched Token", + "", -1, -1) + + def appendTokenName(self, sb, tokenType): + if tokenType == INVALID_TYPE: + sb.append("") + elif tokenType < 0 or tokenType >= len(self.tokenNames): + sb.append("<" + str(tokenType) + ">") + else: + sb.append(self.tokenNames[tokenType]) + + ## + # Returns an error message with line number/column information + # + def __str__(self): + sb = [''] + sb.append(RecognitionException.__str__(self)) + + if self.mismatchType == MismatchedTokenException.TOKEN: + sb.append("expecting ") + self.appendTokenName(sb, self.expecting) + sb.append(", found " + self.tokenText) + elif self.mismatchType == MismatchedTokenException.NOT_TOKEN: + sb.append("expecting anything but '") + self.appendTokenName(sb, self.expecting) + sb.append("'; got it anyway") + elif self.mismatchType in [MismatchedTokenException.RANGE, MismatchedTokenException.NOT_RANGE]: + sb.append("expecting token ") + if self.mismatchType == MismatchedTokenException.NOT_RANGE: + sb.append("NOT ") + sb.append("in range: ") + appendTokenName(sb, self.expecting) + sb.append("..") + appendTokenName(sb, self.upper) + sb.append(", found " + self.tokenText) + elif self.mismatchType in [MismatchedTokenException.SET, MismatchedTokenException.NOT_SET]: + sb.append("expecting ") + if self.mismatchType == MismatchedTokenException.NOT_SET: + sb.append("NOT ") + sb.append("one of (") + for i in range(len(self.set)): + self.appendTokenName(sb, self.set[i]) + sb.append("), found " + self.tokenText) + + return str().join(sb).strip() + + __repr__ = __str__ + + +class TokenStreamException(ANTLRException): + + def __init__(self, *args): + ANTLRException.__init__(self, *args) + + +# Wraps an Exception in a TokenStreamException +class TokenStreamIOException(TokenStreamException): + + def __init__(self, *args): + if args and isinstance(args[0], Exception): + io = args[0] + TokenStreamException.__init__(self, str(io)) + self.io = io + else: + TokenStreamException.__init__(self, *args) + self.io = self + + +# Wraps a RecognitionException in a TokenStreamException +class TokenStreamRecognitionException(TokenStreamException): + + def __init__(self, *args): + if args and isinstance(args[0], RecognitionException): + recog = args[0] + TokenStreamException.__init__(self, str(recog)) + self.recog = recog + else: + raise TypeError("TokenStreamRecognitionException requires RecognitionException argument") + + def __str__(self): + return str(self.recog) + + __repr__ = __str__ + + +class TokenStreamRetryException(TokenStreamException): + + def __init__(self, *args): + TokenStreamException.__init__(self, *args) + + +class CharStreamException(ANTLRException): + + def __init__(self, *args): + ANTLRException.__init__(self, *args) + + +# Wraps an Exception in a CharStreamException +class CharStreamIOException(CharStreamException): + + def __init__(self, *args): + if args and isinstance(args[0], Exception): + io = args[0] + CharStreamException.__init__(self, str(io)) + self.io = io + else: + CharStreamException.__init__(self, *args) + self.io = self + + +class TryAgain(Exception): + pass + + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### Token ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class Token(object): + SKIP = -1 + INVALID_TYPE = 0 + EOF_TYPE = 1 + EOF = 1 + NULL_TREE_LOOKAHEAD = 3 + MIN_USER_TYPE = 4 + + def __init__(self,**argv): + try: + self.type = argv['type'] + except: + self.type = INVALID_TYPE + try: + self.text = argv['text'] + except: + self.text = "" + + def isEOF(self): + return (self.type == EOF_TYPE) + + def getColumn(self): + return 0 + + def getLine(self): + return 0 + + def getFilename(self): + return None + + def setFilename(self,name): + return self + + def getText(self): + return "" + + def setText(self,text): + if is_string_type(text): + pass + else: + raise TypeError("Token.setText requires string argument") + return self + + def setColumn(self,column): + return self + + def setLine(self,line): + return self + + def getType(self): + return self.type + + def setType(self,type): + if isinstance(type,int): + self.type = type + else: + raise TypeError("Token.setType requires integer argument") + return self + + def toString(self): + ## not optimal + type_ = self.type + if type_ == 3: + tval = 'NULL_TREE_LOOKAHEAD' + elif type_ == 1: + tval = 'EOF_TYPE' + elif type_ == 0: + tval = 'INVALID_TYPE' + elif type_ == -1: + tval = 'SKIP' + else: + tval = type_ + return '["%s",<%s>]' % (self.getText(),tval) + + __str__ = toString + __repr__ = toString + +### static attribute .. +Token.badToken = Token( type=INVALID_TYPE, text="") + +if __name__ == "__main__": + print "testing .." + T = Token.badToken + print T + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### CommonToken ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class CommonToken(Token): + + def __init__(self,**argv): + Token.__init__(self,**argv) + self.line = 0 + self.col = 0 + try: + self.line = argv['line'] + except: + pass + try: + self.col = argv['col'] + except: + pass + + def getLine(self): + return self.line + + def getText(self): + return self.text + + def getColumn(self): + return self.col + + def setLine(self,line): + self.line = line + return self + + def setText(self,text): + self.text = text + return self + + def setColumn(self,col): + self.col = col + return self + + def toString(self): + ## not optimal + type_ = self.type + if type_ == 3: + tval = 'NULL_TREE_LOOKAHEAD' + elif type_ == 1: + tval = 'EOF_TYPE' + elif type_ == 0: + tval = 'INVALID_TYPE' + elif type_ == -1: + tval = 'SKIP' + else: + tval = type_ + d = { + 'text' : self.text, + 'type' : tval, + 'line' : self.line, + 'colm' : self.col + } + + fmt = '["%(text)s",<%(type)s>,line=%(line)s,col=%(colm)s]' + return fmt % d + + __str__ = toString + __repr__ = toString + + +if __name__ == '__main__' : + T = CommonToken() + print T + T = CommonToken(col=15,line=1,text="some text", type=5) + print T + T = CommonToken() + T.setLine(1).setColumn(15).setText("some text").setType(5) + print T + print T.getLine() + print T.getColumn() + print T.getText() + print T.getType() + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### CommonHiddenStreamToken ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class CommonHiddenStreamToken(CommonToken): + def __init__(self,*args): + CommonToken.__init__(self,*args) + self.hiddenBefore = None + self.hiddenAfter = None + + def getHiddenAfter(self): + return self.hiddenAfter + + def getHiddenBefore(self): + return self.hiddenBefore + + def setHiddenAfter(self,t): + self.hiddenAfter = t + + def setHiddenBefore(self, t): + self.hiddenBefore = t + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### Queue ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +## Shall be a circular buffer on tokens .. +class Queue(object): + + def __init__(self): + self.buffer = [] # empty list + + def append(self,item): + self.buffer.append(item) + + def elementAt(self,index): + return self.buffer[index] + + def reset(self): + self.buffer = [] + + def removeFirst(self): + self.buffer.pop(0) + + def length(self): + return len(self.buffer) + + def __str__(self): + return str(self.buffer) + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### InputBuffer ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class InputBuffer(object): + def __init__(self): + self.nMarkers = 0 + self.markerOffset = 0 + self.numToConsume = 0 + self.queue = Queue() + + def __str__(self): + return "(%s,%s,%s,%s)" % ( + self.nMarkers, + self.markerOffset, + self.numToConsume, + self.queue) + + def __repr__(self): + return str(self) + + def commit(self): + self.nMarkers -= 1 + + def consume(self) : + self.numToConsume += 1 + + ## probably better to return a list of items + ## because of unicode. Or return a unicode + ## string .. + def getLAChars(self) : + i = self.markerOffset + n = self.queue.length() + s = '' + while i 0: + if self.nMarkers > 0: + # guess mode -- leave leading characters and bump offset. + self.markerOffset += 1 + else: + # normal mode -- remove first character + self.queue.removeFirst() + self.numToConsume -= 1 + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### CharBuffer ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class CharBuffer(InputBuffer): + def __init__(self,reader): + ##assert isinstance(reader,file) + super(CharBuffer,self).__init__() + ## a reader is supposed to be anything that has + ## a method 'read(int)'. + self.input = reader + + def __str__(self): + base = super(CharBuffer,self).__str__() + return "CharBuffer{%s,%s" % (base,str(input)) + + def fill(self,amount): + try: + self.syncConsume() + while self.queue.length() < (amount + self.markerOffset) : + ## retrieve just one char - what happend at end + ## of input? + c = self.input.read(1) + ### python's behaviour is to return the empty string on + ### EOF, ie. no exception whatsoever is thrown. An empty + ### python string has the nice feature that it is of + ### type 'str' and "not ''" would return true. Contrary, + ### one can't do this: '' in 'abc'. This should return + ### false, but all we get is then a TypeError as an + ### empty string is not a character. + + ### Let's assure then that we have either seen a + ### character or an empty string (EOF). + assert len(c) == 0 or len(c) == 1 + + ### And it shall be of type string (ASCII or UNICODE). + assert is_string_type(c) + + ### Just append EOF char to buffer. Note that buffer may + ### contain then just more than one EOF char .. + + ### use unicode chars instead of ASCII .. + self.queue.append(c) + except Exception,e: + raise CharStreamIOException(e) + ##except: # (mk) Cannot happen ... + ##error ("unexpected exception caught ..") + ##assert 0 + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### LexerSharedInputState ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class LexerSharedInputState(object): + def __init__(self,ibuf): + assert isinstance(ibuf,InputBuffer) + self.input = ibuf + self.column = 1 + self.line = 1 + self.tokenStartColumn = 1 + self.tokenStartLine = 1 + self.guessing = 0 + self.filename = None + + def reset(self): + self.column = 1 + self.line = 1 + self.tokenStartColumn = 1 + self.tokenStartLine = 1 + self.guessing = 0 + self.filename = None + self.input.reset() + + def LA(self,k): + return self.input.LA(k) + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TokenStream ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TokenStream(object): + def nextToken(self): + pass + + def __iter__(self): + return TokenStreamIterator(self) + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TokenStreamIterator ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TokenStreamIterator(object): + def __init__(self,inst): + if isinstance(inst,TokenStream): + self.inst = inst + return + raise TypeError("TokenStreamIterator requires TokenStream object") + + def next(self): + assert self.inst + item = self.inst.nextToken() + if not item or item.isEOF(): + raise StopIteration() + return item + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TokenStreamSelector ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TokenStreamSelector(TokenStream): + + def __init__(self): + self._input = None + self._stmap = {} + self._stack = [] + + def addInputStream(self,stream,key): + self._stmap[key] = stream + + def getCurrentStream(self): + return self._input + + def getStream(self,sname): + try: + stream = self._stmap[sname] + except: + raise ValueError("TokenStream " + sname + " not found"); + return stream; + + def nextToken(self): + while 1: + try: + return self._input.nextToken() + except TokenStreamRetryException,r: + ### just retry "forever" + pass + + def pop(self): + stream = self._stack.pop(); + self.select(stream); + return stream; + + def push(self,arg): + self._stack.append(self._input); + self.select(arg) + + def retry(self): + raise TokenStreamRetryException() + + def select(self,arg): + if isinstance(arg,TokenStream): + self._input = arg + return + if is_string_type(arg): + self._input = self.getStream(arg) + return + raise TypeError("TokenStreamSelector.select requires " + + "TokenStream or string argument") + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TokenStreamBasicFilter ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TokenStreamBasicFilter(TokenStream): + + def __init__(self,input): + + self.input = input; + self.discardMask = BitSet() + + def discard(self,arg): + if isinstance(arg,int): + self.discardMask.add(arg) + return + if isinstance(arg,BitSet): + self.discardMark = arg + return + raise TypeError("TokenStreamBasicFilter.discard requires" + + "integer or BitSet argument") + + def nextToken(self): + tok = self.input.nextToken() + while tok and self.discardMask.member(tok.getType()): + tok = self.input.nextToken() + return tok + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TokenStreamHiddenTokenFilter ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TokenStreamHiddenTokenFilter(TokenStreamBasicFilter): + + def __init__(self,input): + TokenStreamBasicFilter.__init__(self,input) + self.hideMask = BitSet() + self.nextMonitoredToken = None + self.lastHiddenToken = None + self.firstHidden = None + + def consume(self): + self.nextMonitoredToken = self.input.nextToken() + + def consumeFirst(self): + self.consume() + + p = None; + while self.hideMask.member(self.LA(1).getType()) or \ + self.discardMask.member(self.LA(1).getType()): + if self.hideMask.member(self.LA(1).getType()): + if not p: + p = self.LA(1) + else: + p.setHiddenAfter(self.LA(1)) + self.LA(1).setHiddenBefore(p) + p = self.LA(1) + self.lastHiddenToken = p + if not self.firstHidden: + self.firstHidden = p + self.consume() + + def getDiscardMask(self): + return self.discardMask + + def getHiddenAfter(self,t): + return t.getHiddenAfter() + + def getHiddenBefore(self,t): + return t.getHiddenBefore() + + def getHideMask(self): + return self.hideMask + + def getInitialHiddenToken(self): + return self.firstHidden + + def hide(self,m): + if isinstance(m,int): + self.hideMask.add(m) + return + if isinstance(m.BitMask): + self.hideMask = m + return + + def LA(self,i): + return self.nextMonitoredToken + + def nextToken(self): + if not self.LA(1): + self.consumeFirst() + + monitored = self.LA(1) + + monitored.setHiddenBefore(self.lastHiddenToken) + self.lastHiddenToken = None + + self.consume() + p = monitored + + while self.hideMask.member(self.LA(1).getType()) or \ + self.discardMask.member(self.LA(1).getType()): + if self.hideMask.member(self.LA(1).getType()): + p.setHiddenAfter(self.LA(1)) + if p != monitored: + self.LA(1).setHiddenBefore(p) + p = self.lastHiddenToken = self.LA(1) + self.consume() + return monitored + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### StringBuffer ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class StringBuffer: + def __init__(self,string=None): + if string: + self.text = list(string) + else: + self.text = [] + + def setLength(self,sz): + if not sz : + self.text = [] + return + assert sz>0 + if sz >= self.length(): + return + ### just reset to empty buffer + self.text = self.text[0:sz] + + def length(self): + return len(self.text) + + def append(self,c): + self.text.append(c) + + ### return buffer as string. Arg 'a' is used as index + ## into the buffer and 2nd argument shall be the length. + ## If 2nd args is absent, we return chars till end of + ## buffer starting with 'a'. + def getString(self,a=None,length=None): + if not a : + a = 0 + assert a>=0 + if a>= len(self.text) : + return "" + + if not length: + ## no second argument + L = self.text[a:] + else: + assert (a+length) <= len(self.text) + b = a + length + L = self.text[a:b] + s = "" + for x in L : s += x + return s + + toString = getString ## alias + + def __str__(self): + return str(self.text) + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### Reader ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +## When reading Japanese chars, it happens that a stream returns a +## 'char' of length 2. This looks like a bug in the appropriate +## codecs - but I'm rather unsure about this. Anyway, if this is +## the case, I'm going to split this string into a list of chars +## and put them on hold, ie. on a buffer. Next time when called +## we read from buffer until buffer is empty. +## wh: nov, 25th -> problem does not appear in Python 2.4.0.c1. + +class Reader(object): + def __init__(self,stream): + self.cin = stream + self.buf = [] + + def read(self,num): + assert num==1 + + if len(self.buf): + return self.buf.pop() + + ## Read a char - this may return a string. + ## Is this a bug in codecs/Python? + c = self.cin.read(1) + + if not c or len(c)==1: + return c + + L = list(c) + L.reverse() + for x in L: + self.buf.append(x) + + ## read one char .. + return self.read(1) + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### CharScanner ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class CharScanner(TokenStream): + ## class members + NO_CHAR = 0 + EOF_CHAR = '' ### EOF shall be the empty string. + + def __init__(self, *argv, **kwargs): + super(CharScanner, self).__init__() + self.saveConsumedInput = True + self.tokenClass = None + self.caseSensitive = True + self.caseSensitiveLiterals = True + self.literals = None + self.tabsize = 8 + self._returnToken = None + self.commitToPath = False + self.traceDepth = 0 + self.text = StringBuffer() + self.hashString = hash(self) + self.setTokenObjectClass(CommonToken) + self.setInput(*argv) + + def __iter__(self): + return CharScannerIterator(self) + + def setInput(self,*argv): + ## case 1: + ## if there's no arg we default to read from + ## standard input + if not argv: + import sys + self.setInput(sys.stdin) + return + + ## get 1st argument + arg1 = argv[0] + + ## case 2: + ## if arg1 is a string, we assume it's a file name + ## and open a stream using 2nd argument as open + ## mode. If there's no 2nd argument we fall back to + ## mode '+rb'. + if is_string_type(arg1): + f = open(arg1,"rb") + self.setInput(f) + self.setFilename(arg1) + return + + ## case 3: + ## if arg1 is a file we wrap it by a char buffer ( + ## some additional checks?? No, can't do this in + ## general). + if isinstance(arg1,file): + self.setInput(CharBuffer(arg1)) + return + + ## case 4: + ## if arg1 is of type SharedLexerInputState we use + ## argument as is. + if isinstance(arg1,LexerSharedInputState): + self.inputState = arg1 + return + + ## case 5: + ## check whether argument type is of type input + ## buffer. If so create a SharedLexerInputState and + ## go ahead. + if isinstance(arg1,InputBuffer): + self.setInput(LexerSharedInputState(arg1)) + return + + ## case 6: + ## check whether argument type has a method read(int) + ## If so create CharBuffer ... + try: + if arg1.read: + rd = Reader(arg1) + cb = CharBuffer(rd) + ss = LexerSharedInputState(cb) + self.inputState = ss + return + except: + pass + + ## case 7: + ## raise wrong argument exception + raise TypeError(argv) + + def setTabSize(self,size) : + self.tabsize = size + + def getTabSize(self) : + return self.tabsize + + def setCaseSensitive(self,t) : + self.caseSensitive = t + + def setCommitToPath(self,commit) : + self.commitToPath = commit + + def setFilename(self,f) : + self.inputState.filename = f + + def setLine(self,line) : + self.inputState.line = line + + def setText(self,s) : + self.resetText() + self.text.append(s) + + def getCaseSensitive(self) : + return self.caseSensitive + + def getCaseSensitiveLiterals(self) : + return self.caseSensitiveLiterals + + def getColumn(self) : + return self.inputState.column + + def setColumn(self,c) : + self.inputState.column = c + + def getCommitToPath(self) : + return self.commitToPath + + def getFilename(self) : + return self.inputState.filename + + def getInputBuffer(self) : + return self.inputState.input + + def getInputState(self) : + return self.inputState + + def setInputState(self,state) : + assert isinstance(state,LexerSharedInputState) + self.inputState = state + + def getLine(self) : + return self.inputState.line + + def getText(self) : + return str(self.text) + + def getTokenObject(self) : + return self._returnToken + + def LA(self,i) : + c = self.inputState.input.LA(i) + if not self.caseSensitive: + ### E0006 + c = c.__class__.lower(c) + return c + + def makeToken(self,type) : + try: + ## dynamically load a class + assert self.tokenClass + tok = self.tokenClass() + tok.setType(type) + tok.setColumn(self.inputState.tokenStartColumn) + tok.setLine(self.inputState.tokenStartLine) + return tok + except: + self.panic("unable to create new token") + return Token.badToken + + def mark(self) : + return self.inputState.input.mark() + + def _match_bitset(self,b) : + if b.member(self.LA(1)): + self.consume() + else: + raise MismatchedCharException(self.LA(1), b, False, self) + + def _match_string(self,s) : + for c in s: + if self.LA(1) == c: + self.consume() + else: + raise MismatchedCharException(self.LA(1), c, False, self) + + def match(self,item): + if is_string_type(item): + return self._match_string(item) + else: + return self._match_bitset(item) + + def matchNot(self,c) : + if self.LA(1) != c: + self.consume() + else: + raise MismatchedCharException(self.LA(1), c, True, self) + + def matchRange(self,c1,c2) : + if self.LA(1) < c1 or self.LA(1) > c2 : + raise MismatchedCharException(self.LA(1), c1, c2, False, self) + else: + self.consume() + + def newline(self) : + self.inputState.line += 1 + self.inputState.column = 1 + + def tab(self) : + c = self.getColumn() + nc = ( ((c-1)/self.tabsize) + 1) * self.tabsize + 1 + self.setColumn(nc) + + def panic(self,s='') : + print "CharScanner: panic: " + s + sys.exit(1) + + def reportError(self,ex) : + print ex + + def reportError(self,s) : + if not self.getFilename(): + print "error: " + str(s) + else: + print self.getFilename() + ": error: " + str(s) + + def reportWarning(self,s) : + if not self.getFilename(): + print "warning: " + str(s) + else: + print self.getFilename() + ": warning: " + str(s) + + def resetText(self) : + self.text.setLength(0) + self.inputState.tokenStartColumn = self.inputState.column + self.inputState.tokenStartLine = self.inputState.line + + def rewind(self,pos) : + self.inputState.input.rewind(pos) + + def setTokenObjectClass(self,cl): + self.tokenClass = cl + + def testForLiteral(self,token): + if not token: + return + assert isinstance(token,Token) + + _type = token.getType() + + ## special tokens can't be literals + if _type in [SKIP,INVALID_TYPE,EOF_TYPE,NULL_TREE_LOOKAHEAD] : + return + + _text = token.getText() + if not _text: + return + + assert is_string_type(_text) + _type = self.testLiteralsTable(_text,_type) + token.setType(_type) + return _type + + def testLiteralsTable(self,*args): + if is_string_type(args[0]): + s = args[0] + i = args[1] + else: + s = self.text.getString() + i = args[0] + + ## check whether integer has been given + if not isinstance(i,int): + assert isinstance(i,int) + + ## check whether we have a dict + assert isinstance(self.literals,dict) + try: + ## E0010 + if not self.caseSensitiveLiterals: + s = s.__class__.lower(s) + i = self.literals[s] + except: + pass + return i + + def toLower(self,c): + return c.__class__.lower() + + def traceIndent(self): + print ' ' * self.traceDepth + + def traceIn(self,rname): + self.traceDepth += 1 + self.traceIndent() + print "> lexer %s c== %s" % (rname,self.LA(1)) + + def traceOut(self,rname): + self.traceIndent() + print "< lexer %s c== %s" % (rname,self.LA(1)) + self.traceDepth -= 1 + + def uponEOF(self): + pass + + def append(self,c): + if self.saveConsumedInput : + self.text.append(c) + + def commit(self): + self.inputState.input.commit() + + def consume(self): + if not self.inputState.guessing: + c = self.LA(1) + if self.caseSensitive: + self.append(c) + else: + # use input.LA(), not LA(), to get original case + # CharScanner.LA() would toLower it. + c = self.inputState.input.LA(1) + self.append(c) + + if c and c in "\t": + self.tab() + else: + self.inputState.column += 1 + self.inputState.input.consume() + + ## Consume chars until one matches the given char + def consumeUntil_char(self,c): + while self.LA(1) != EOF_CHAR and self.LA(1) != c: + self.consume() + + ## Consume chars until one matches the given set + def consumeUntil_bitset(self,bitset): + while self.LA(1) != EOF_CHAR and not self.set.member(self.LA(1)): + self.consume() + + ### If symbol seen is EOF then generate and set token, otherwise + ### throw exception. + def default(self,la1): + if not la1 : + self.uponEOF() + self._returnToken = self.makeToken(EOF_TYPE) + else: + self.raise_NoViableAlt(la1) + + def filterdefault(self,la1,*args): + if not la1: + self.uponEOF() + self._returnToken = self.makeToken(EOF_TYPE) + return + + if not args: + self.consume() + raise TryAgain() + else: + ### apply filter object + self.commit(); + try: + func=args[0] + args=args[1:] + apply(func,args) + except RecognitionException, e: + ## catastrophic failure + self.reportError(e); + self.consume(); + raise TryAgain() + + def raise_NoViableAlt(self,la1=None): + if not la1: la1 = self.LA(1) + fname = self.getFilename() + line = self.getLine() + col = self.getColumn() + raise NoViableAltForCharException(la1,fname,line,col) + + def set_return_token(self,_create,_token,_ttype,_offset): + if _create and not _token and (not _ttype == SKIP): + string = self.text.getString(_offset) + _token = self.makeToken(_ttype) + _token.setText(string) + self._returnToken = _token + return _token + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### CharScannerIterator ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class CharScannerIterator: + + def __init__(self,inst): + if isinstance(inst,CharScanner): + self.inst = inst + return + raise TypeError("CharScannerIterator requires CharScanner object") + + def next(self): + assert self.inst + item = self.inst.nextToken() + if not item or item.isEOF(): + raise StopIteration() + return item + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### BitSet ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +### I'm assuming here that a long is 64bits. It appears however, that +### a long is of any size. That means we can use a single long as the +### bitset (!), ie. Python would do almost all the work (TBD). + +class BitSet(object): + BITS = 64 + NIBBLE = 4 + LOG_BITS = 6 + MOD_MASK = BITS -1 + + def __init__(self,data=None): + if not data: + BitSet.__init__(self,[long(0)]) + return + if isinstance(data,int): + BitSet.__init__(self,[long(data)]) + return + if isinstance(data,long): + BitSet.__init__(self,[data]) + return + if not isinstance(data,list): + raise TypeError("BitSet requires integer, long, or " + + "list argument") + for x in data: + if not isinstance(x,long): + raise TypeError(self,"List argument item is " + + "not a long: %s" % (x)) + self.data = data + + def __str__(self): + bits = len(self.data) * BitSet.BITS + s = "" + for i in xrange(0,bits): + if self.at(i): + s += "1" + else: + s += "o" + if not ((i+1) % 10): + s += '|%s|' % (i+1) + return s + + def __repr__(self): + return str(self) + + def member(self,item): + if not item: + return False + + if isinstance(item,int): + return self.at(item) + + if not is_string_type(item): + raise TypeError(self,"char or unichar expected: %s" % (item)) + + ## char is a (unicode) string with at most lenght 1, ie. + ## a char. + + if len(item) != 1: + raise TypeError(self,"char expected: %s" % (item)) + + ### handle ASCII/UNICODE char + num = ord(item) + + ### check whether position num is in bitset + return self.at(num) + + def wordNumber(self,bit): + return bit >> BitSet.LOG_BITS + + def bitMask(self,bit): + pos = bit & BitSet.MOD_MASK ## bit mod BITS + return (1L << pos) + + def set(self,bit,on=True): + # grow bitset as required (use with care!) + i = self.wordNumber(bit) + mask = self.bitMask(bit) + if i>=len(self.data): + d = i - len(self.data) + 1 + for x in xrange(0,d): + self.data.append(0L) + assert len(self.data) == i+1 + if on: + self.data[i] |= mask + else: + self.data[i] &= (~mask) + + ### make add an alias for set + add = set + + def off(self,bit,off=True): + self.set(bit,not off) + + def at(self,bit): + i = self.wordNumber(bit) + v = self.data[i] + m = self.bitMask(bit) + return v & m + + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### some further funcs ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +def illegalarg_ex(func): + raise ValueError( + "%s is only valid if parser is built for debugging" % + (func.func_name)) + +def runtime_ex(func): + raise RuntimeException( + "%s is only valid if parser is built for debugging" % + (func.func_name)) + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TokenBuffer ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TokenBuffer(object): + def __init__(self,stream): + self.input = stream + self.nMarkers = 0 + self.markerOffset = 0 + self.numToConsume = 0 + self.queue = Queue() + + def reset(self) : + self.nMarkers = 0 + self.markerOffset = 0 + self.numToConsume = 0 + self.queue.reset() + + def consume(self) : + self.numToConsume += 1 + + def fill(self, amount): + self.syncConsume() + while self.queue.length() < (amount + self.markerOffset): + self.queue.append(self.input.nextToken()) + + def getInput(self): + return self.input + + def LA(self,k) : + self.fill(k) + return self.queue.elementAt(self.markerOffset + k - 1).type + + def LT(self,k) : + self.fill(k) + return self.queue.elementAt(self.markerOffset + k - 1) + + def mark(self) : + self.syncConsume() + self.nMarkers += 1 + return self.markerOffset + + def rewind(self,mark) : + self.syncConsume() + self.markerOffset = mark + self.nMarkers -= 1 + + def syncConsume(self) : + while self.numToConsume > 0: + if self.nMarkers > 0: + # guess mode -- leave leading characters and bump offset. + self.markerOffset += 1 + else: + # normal mode -- remove first character + self.queue.removeFirst() + self.numToConsume -= 1 + + def __str__(self): + return "(%s,%s,%s,%s,%s)" % ( + self.input, + self.nMarkers, + self.markerOffset, + self.numToConsume, + self.queue) + + def __repr__(self): + return str(self) + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### ParserSharedInputState ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class ParserSharedInputState(object): + + def __init__(self): + self.input = None + self.reset() + + def reset(self): + self.guessing = 0 + self.filename = None + if self.input: + self.input.reset() + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### Parser ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class Parser(object): + + def __init__(self, *args, **kwargs): + self.tokenNames = None + self.returnAST = None + self.astFactory = None + self.tokenTypeToASTClassMap = {} + self.ignoreInvalidDebugCalls = False + self.traceDepth = 0 + if not args: + self.inputState = ParserSharedInputState() + return + arg0 = args[0] + assert isinstance(arg0,ParserSharedInputState) + self.inputState = arg0 + return + + def getTokenTypeToASTClassMap(self): + return self.tokenTypeToASTClassMap + + + def addMessageListener(self, l): + if not self.ignoreInvalidDebugCalls: + illegalarg_ex(addMessageListener) + + def addParserListener(self,l) : + if (not self.ignoreInvalidDebugCalls) : + illegalarg_ex(addParserListener) + + def addParserMatchListener(self, l) : + if (not self.ignoreInvalidDebugCalls) : + illegalarg_ex(addParserMatchListener) + + def addParserTokenListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + illegalarg_ex(addParserTokenListener) + + def addSemanticPredicateListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + illegalarg_ex(addSemanticPredicateListener) + + def addSyntacticPredicateListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + illegalarg_ex(addSyntacticPredicateListener) + + def addTraceListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + illegalarg_ex(addTraceListener) + + def consume(self): + raise NotImplementedError() + + def _consumeUntil_type(self,tokenType): + while self.LA(1) != EOF_TYPE and self.LA(1) != tokenType: + self.consume() + + def _consumeUntil_bitset(self, set): + while self.LA(1) != EOF_TYPE and not set.member(self.LA(1)): + self.consume() + + def consumeUntil(self,arg): + if isinstance(arg,int): + self._consumeUntil_type(arg) + else: + self._consumeUntil_bitset(arg) + + def defaultDebuggingSetup(self): + pass + + def getAST(self) : + return self.returnAST + + def getASTFactory(self) : + return self.astFactory + + def getFilename(self) : + return self.inputState.filename + + def getInputState(self) : + return self.inputState + + def setInputState(self, state) : + self.inputState = state + + def getTokenName(self,num) : + return self.tokenNames[num] + + def getTokenNames(self) : + return self.tokenNames + + def isDebugMode(self) : + return self.false + + def LA(self, i): + raise NotImplementedError() + + def LT(self, i): + raise NotImplementedError() + + def mark(self): + return self.inputState.input.mark() + + def _match_int(self,t): + if (self.LA(1) != t): + raise MismatchedTokenException( + self.tokenNames, self.LT(1), t, False, self.getFilename()) + else: + self.consume() + + def _match_set(self, b): + if (not b.member(self.LA(1))): + raise MismatchedTokenException( + self.tokenNames,self.LT(1), b, False, self.getFilename()) + else: + self.consume() + + def match(self,set) : + if isinstance(set,int): + self._match_int(set) + return + if isinstance(set,BitSet): + self._match_set(set) + return + raise TypeError("Parser.match requires integer ot BitSet argument") + + def matchNot(self,t): + if self.LA(1) == t: + raise MismatchedTokenException( + tokenNames, self.LT(1), t, True, self.getFilename()) + else: + self.consume() + + def removeMessageListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(removeMessageListener) + + def removeParserListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(removeParserListener) + + def removeParserMatchListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(removeParserMatchListener) + + def removeParserTokenListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(removeParserTokenListener) + + def removeSemanticPredicateListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(removeSemanticPredicateListener) + + def removeSyntacticPredicateListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(removeSyntacticPredicateListener) + + def removeTraceListener(self, l) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(removeTraceListener) + + def reportError(self,x) : + fmt = "syntax error:" + f = self.getFilename() + if f: + fmt = ("%s:" % f) + fmt + if isinstance(x,Token): + line = x.getColumn() + col = x.getLine() + text = x.getText() + fmt = fmt + 'unexpected symbol at line %s (column %s) : "%s"' + print >>sys.stderr, fmt % (line,col,text) + else: + print >>sys.stderr, fmt,str(x) + + def reportWarning(self,s): + f = self.getFilename() + if f: + print "%s:warning: %s" % (f,str(x)) + else: + print "warning: %s" % (str(x)) + + def rewind(self, pos) : + self.inputState.input.rewind(pos) + + def setASTFactory(self, f) : + self.astFactory = f + + def setASTNodeClass(self, cl) : + self.astFactory.setASTNodeType(cl) + + def setASTNodeType(self, nodeType) : + self.setASTNodeClass(nodeType) + + def setDebugMode(self, debugMode) : + if (not self.ignoreInvalidDebugCalls): + runtime_ex(setDebugMode) + + def setFilename(self, f) : + self.inputState.filename = f + + def setIgnoreInvalidDebugCalls(self, value) : + self.ignoreInvalidDebugCalls = value + + def setTokenBuffer(self, t) : + self.inputState.input = t + + def traceIndent(self): + print " " * self.traceDepth + + def traceIn(self,rname): + self.traceDepth += 1 + self.trace("> ", rname) + + def traceOut(self,rname): + self.trace("< ", rname) + self.traceDepth -= 1 + + ### wh: moved from ASTFactory to Parser + def addASTChild(self,currentAST, child): + if not child: + return + if not currentAST.root: + currentAST.root = child + elif not currentAST.child: + currentAST.root.setFirstChild(child) + else: + currentAST.child.setNextSibling(child) + currentAST.child = child + currentAST.advanceChildToEnd() + + ### wh: moved from ASTFactory to Parser + def makeASTRoot(self,currentAST,root) : + if root: + ### Add the current root as a child of new root + root.addChild(currentAST.root) + ### The new current child is the last sibling of the old root + currentAST.child = currentAST.root + currentAST.advanceChildToEnd() + ### Set the new root + currentAST.root = root + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### LLkParser ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class LLkParser(Parser): + + def __init__(self, *args, **kwargs): + try: + arg1 = args[0] + except: + arg1 = 1 + + if isinstance(arg1,int): + super(LLkParser,self).__init__() + self.k = arg1 + return + + if isinstance(arg1,ParserSharedInputState): + super(LLkParser,self).__init__(arg1) + self.set_k(1,*args) + return + + if isinstance(arg1,TokenBuffer): + super(LLkParser,self).__init__() + self.setTokenBuffer(arg1) + self.set_k(1,*args) + return + + if isinstance(arg1,TokenStream): + super(LLkParser,self).__init__() + tokenBuf = TokenBuffer(arg1) + self.setTokenBuffer(tokenBuf) + self.set_k(1,*args) + return + + ### unknown argument + raise TypeError("LLkParser requires integer, " + + "ParserSharedInputStream or TokenStream argument") + + def consume(self): + self.inputState.input.consume() + + def LA(self,i): + return self.inputState.input.LA(i) + + def LT(self,i): + return self.inputState.input.LT(i) + + def set_k(self,index,*args): + try: + self.k = args[index] + except: + self.k = 1 + + def trace(self,ee,rname): + print type(self) + self.traceIndent() + guess = "" + if self.inputState.guessing > 0: + guess = " [guessing]" + print(ee + rname + guess) + for i in xrange(1,self.k+1): + if i != 1: + print(", ") + if self.LT(i) : + v = self.LT(i).getText() + else: + v = "null" + print "LA(%s) == %s" % (i,v) + print("\n") + + def traceIn(self,rname): + self.traceDepth += 1; + self.trace("> ", rname); + + def traceOut(self,rname): + self.trace("< ", rname); + self.traceDepth -= 1; + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TreeParserSharedInputState ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TreeParserSharedInputState(object): + def __init__(self): + self.guessing = 0 + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### TreeParser ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class TreeParser(object): + + def __init__(self, *args, **kwargs): + self.inputState = TreeParserSharedInputState() + self._retTree = None + self.tokenNames = [] + self.returnAST = None + self.astFactory = ASTFactory() + self.traceDepth = 0 + + def getAST(self): + return self.returnAST + + def getASTFactory(self): + return self.astFactory + + def getTokenName(self,num) : + return self.tokenNames[num] + + def getTokenNames(self): + return self.tokenNames + + def match(self,t,set) : + assert isinstance(set,int) or isinstance(set,BitSet) + if not t or t == ASTNULL: + raise MismatchedTokenException(self.getTokenNames(), t,set, False) + + if isinstance(set,int) and t.getType() != set: + raise MismatchedTokenException(self.getTokenNames(), t,set, False) + + if isinstance(set,BitSet) and not set.member(t.getType): + raise MismatchedTokenException(self.getTokenNames(), t,set, False) + + def matchNot(self,t, ttype) : + if not t or (t == ASTNULL) or (t.getType() == ttype): + raise MismatchedTokenException(getTokenNames(), t, ttype, True) + + def reportError(self,ex): + print >>sys.stderr,"error:",ex + + def reportWarning(self, s): + print "warning:",s + + def setASTFactory(self,f): + self.astFactory = f + + def setASTNodeType(self,nodeType): + self.setASTNodeClass(nodeType) + + def setASTNodeClass(self,nodeType): + self.astFactory.setASTNodeType(nodeType) + + def traceIndent(self): + print " " * self.traceDepth + + def traceIn(self,rname,t): + self.traceDepth += 1 + self.traceIndent() + print("> " + rname + "(" + + ifelse(t,str(t),"null") + ")" + + ifelse(self.inputState.guessing>0,"[guessing]","")) + + def traceOut(self,rname,t): + self.traceIndent() + print("< " + rname + "(" + + ifelse(t,str(t),"null") + ")" + + ifelse(self.inputState.guessing>0,"[guessing]","")) + self.traceDepth -= 1 + + ### wh: moved from ASTFactory to TreeParser + def addASTChild(self,currentAST, child): + if not child: + return + if not currentAST.root: + currentAST.root = child + elif not currentAST.child: + currentAST.root.setFirstChild(child) + else: + currentAST.child.setNextSibling(child) + currentAST.child = child + currentAST.advanceChildToEnd() + + ### wh: moved from ASTFactory to TreeParser + def makeASTRoot(self,currentAST,root): + if root: + ### Add the current root as a child of new root + root.addChild(currentAST.root) + ### The new current child is the last sibling of the old root + currentAST.child = currentAST.root + currentAST.advanceChildToEnd() + ### Set the new root + currentAST.root = root + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### funcs to work on trees ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +def rightmost(ast): + if ast: + while(ast.right): + ast = ast.right + return ast + +def cmptree(s,t,partial): + while(s and t): + ### as a quick optimization, check roots first. + if not s.equals(t): + return False + + ### if roots match, do full list match test on children. + if not cmptree(s.getFirstChild(),t.getFirstChild(),partial): + return False + + s = s.getNextSibling() + t = t.getNextSibling() + + r = ifelse(partial,not t,not s and not t) + return r + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### AST ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class AST(object): + def __init__(self): + pass + + def addChild(self, c): + pass + + def equals(self, t): + return False + + def equalsList(self, t): + return False + + def equalsListPartial(self, t): + return False + + def equalsTree(self, t): + return False + + def equalsTreePartial(self, t): + return False + + def findAll(self, tree): + return None + + def findAllPartial(self, subtree): + return None + + def getFirstChild(self): + return self + + def getNextSibling(self): + return self + + def getText(self): + return "" + + def getType(self): + return INVALID_TYPE + + def getLine(self): + return 0 + + def getColumn(self): + return 0 + + def getNumberOfChildren(self): + return 0 + + def initialize(self, t, txt): + pass + + def initialize(self, t): + pass + + def setFirstChild(self, c): + pass + + def setNextSibling(self, n): + pass + + def setText(self, text): + pass + + def setType(self, ttype): + pass + + def toString(self): + self.getText() + + __str__ = toString + + def toStringList(self): + return self.getText() + + def toStringTree(self): + return self.getText() + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### ASTNULLType ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +### There is only one instance of this class **/ +class ASTNULLType(AST): + def __init__(self): + AST.__init__(self) + pass + + def getText(self): + return "" + + def getType(self): + return NULL_TREE_LOOKAHEAD + + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### BaseAST ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class BaseAST(AST): + + verboseStringConversion = False + tokenNames = None + + def __init__(self): + self.down = None ## kid + self.right = None ## sibling + + def addChild(self,node): + if node: + t = rightmost(self.down) + if t: + t.right = node + else: + assert not self.down + self.down = node + + def getNumberOfChildren(self): + t = self.down + n = 0 + while t: + n += 1 + t = t.right + return n + + def doWorkForFindAll(self,v,target,partialMatch): + sibling = self + + while sibling: + c1 = partialMatch and sibling.equalsTreePartial(target) + if c1: + v.append(sibling) + else: + c2 = not partialMatch and sibling.equalsTree(target) + if c2: + v.append(sibling) + + ### regardless of match or not, check any children for matches + if sibling.getFirstChild(): + sibling.getFirstChild().doWorkForFindAll(v,target,partialMatch) + + sibling = sibling.getNextSibling() + + ### Is node t equal to 'self' in terms of token type and text? + def equals(self,t): + if not t: + return False + return self.getText() == t.getText() and self.getType() == t.getType() + + ### Is t an exact structural and equals() match of this tree. The + ### 'self' reference is considered the start of a sibling list. + ### + def equalsList(self, t): + return cmptree(self, t, partial=False) + + ### Is 't' a subtree of this list? + ### The siblings of the root are NOT ignored. + ### + def equalsListPartial(self,t): + return cmptree(self,t,partial=True) + + ### Is tree rooted at 'self' equal to 't'? The siblings + ### of 'self' are ignored. + ### + def equalsTree(self, t): + return self.equals(t) and \ + cmptree(self.getFirstChild(), t.getFirstChild(), partial=False) + + ### Is 't' a subtree of the tree rooted at 'self'? The siblings + ### of 'self' are ignored. + ### + def equalsTreePartial(self, t): + if not t: + return True + return self.equals(t) and cmptree( + self.getFirstChild(), t.getFirstChild(), partial=True) + + ### Walk the tree looking for all exact subtree matches. Return + ### an ASTEnumerator that lets the caller walk the list + ### of subtree roots found herein. + def findAll(self,target): + roots = [] + + ### the empty tree cannot result in an enumeration + if not target: + return None + # find all matches recursively + self.doWorkForFindAll(roots, target, False) + return roots + + ### Walk the tree looking for all subtrees. Return + ### an ASTEnumerator that lets the caller walk the list + ### of subtree roots found herein. + def findAllPartial(self,sub): + roots = [] + + ### the empty tree cannot result in an enumeration + if not sub: + return None + + self.doWorkForFindAll(roots, sub, True) ### find all matches recursively + return roots + + ### Get the first child of this node None if not children + def getFirstChild(self): + return self.down + + ### Get the next sibling in line after this one + def getNextSibling(self): + return self.right + + ### Get the token text for this node + def getText(self): + return "" + + ### Get the token type for this node + def getType(self): + return 0 + + def getLine(self): + return 0 + + def getColumn(self): + return 0 + + ### Remove all children */ + def removeChildren(self): + self.down = None + + def setFirstChild(self,c): + self.down = c + + def setNextSibling(self, n): + self.right = n + + ### Set the token text for this node + def setText(self, text): + pass + + ### Set the token type for this node + def setType(self, ttype): + pass + + ### static + def setVerboseStringConversion(verbose,names): + verboseStringConversion = verbose + tokenNames = names + setVerboseStringConversion = staticmethod(setVerboseStringConversion) + + ### Return an array of strings that maps token ID to it's text. + ## @since 2.7.3 + def getTokenNames(): + return tokenNames + + def toString(self): + return self.getText() + + ### return tree as lisp string - sibling included + def toStringList(self): + ts = self.toStringTree() + sib = self.getNextSibling() + if sib: + ts += sib.toStringList() + return ts + + __str__ = toStringList + + ### return tree as string - siblings ignored + def toStringTree(self): + ts = "" + kid = self.getFirstChild() + if kid: + ts += " (" + ts += " " + self.toString() + if kid: + ts += kid.toStringList() + ts += " )" + return ts + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### CommonAST ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +### Common AST node implementation +class CommonAST(BaseAST): + def __init__(self,token=None): + super(CommonAST,self).__init__() + self.ttype = INVALID_TYPE + self.text = "" + self.line = 0 + self.column= 0 + self.initialize(token) + #assert self.text + + ### Get the token text for this node + def getText(self): + return self.text + + ### Get the token type for this node + def getType(self): + return self.ttype + + ### Get the line for this node + def getLine(self): + return self.line + + ### Get the column for this node + def getColumn(self): + return self.column + + def initialize(self,*args): + if not args: + return + + arg0 = args[0] + + if isinstance(arg0,int): + arg1 = args[1] + self.setType(arg0) + self.setText(arg1) + return + + if isinstance(arg0,AST) or isinstance(arg0,Token): + self.setText(arg0.getText()) + self.setType(arg0.getType()) + self.line = arg0.getLine() + self.column = arg0.getColumn() + return + + ### Set the token text for this node + def setText(self,text_): + assert is_string_type(text_) + self.text = text_ + + ### Set the token type for this node + def setType(self,ttype_): + assert isinstance(ttype_,int) + self.ttype = ttype_ + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### CommonASTWithHiddenTokens ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class CommonASTWithHiddenTokens(CommonAST): + + def __init__(self,*args): + CommonAST.__init__(self,*args) + self.hiddenBefore = None + self.hiddenAfter = None + + def getHiddenAfter(self): + return self.hiddenAfter + + def getHiddenBefore(self): + return self.hiddenBefore + + def initialize(self,*args): + CommonAST.initialize(self,*args) + if args and isinstance(args[0],Token): + assert isinstance(args[0],CommonHiddenStreamToken) + self.hideenBefore = args[0].getHiddenBefore() + self.hiddenAfter = args[0].getHiddenAfter() + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### ASTPair ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class ASTPair(object): + def __init__(self): + self.root = None ### current root of tree + self.child = None ### current child to which siblings are added + + ### Make sure that child is the last sibling */ + def advanceChildToEnd(self): + if self.child: + while self.child.getNextSibling(): + self.child = self.child.getNextSibling() + + ### Copy an ASTPair. Don't call it clone() because we want type-safety */ + def copy(self): + tmp = ASTPair() + tmp.root = self.root + tmp.child = self.child + return tmp + + def toString(self): + r = ifelse(not root,"null",self.root.getText()) + c = ifelse(not child,"null",self.child.getText()) + return "[%s,%s]" % (r,c) + + __str__ = toString + __repr__ = toString + + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### ASTFactory ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class ASTFactory(object): + def __init__(self,table=None): + self._class = None + self._classmap = ifelse(table,table,None) + + def create(self,*args): + if not args: + return self.create(INVALID_TYPE) + + arg0 = args[0] + arg1 = None + arg2 = None + + try: + arg1 = args[1] + arg2 = args[2] + except: + pass + + # ctor(int) + if isinstance(arg0,int) and not arg2: + ### get class for 'self' type + c = self.getASTNodeType(arg0) + t = self.create(c) + if t: + t.initialize(arg0, ifelse(arg1,arg1,"")) + return t + + # ctor(int,something) + if isinstance(arg0,int) and arg2: + t = self.create(arg2) + if t: + t.initialize(arg0,arg1) + return t + + # ctor(AST) + if isinstance(arg0,AST): + t = self.create(arg0.getType()) + if t: + t.initialize(arg0) + return t + + # ctor(token) + if isinstance(arg0,Token) and not arg1: + ttype = arg0.getType() + assert isinstance(ttype,int) + t = self.create(ttype) + if t: + t.initialize(arg0) + return t + + # ctor(token,class) + if isinstance(arg0,Token) and arg1: + assert isinstance(arg1,type) + assert issubclass(arg1,AST) + # this creates instance of 'arg1' using 'arg0' as + # argument. Wow, that's magic! + t = arg1(arg0) + assert t and isinstance(t,AST) + return t + + # ctor(class) + if isinstance(arg0,type): + ### next statement creates instance of type (!) + t = arg0() + assert isinstance(t,AST) + return t + + + def setASTNodeClass(self,className=None): + if not className: + return + assert isinstance(className,type) + assert issubclass(className,AST) + self._class = className + + ### kind of misnomer - use setASTNodeClass instead. + setASTNodeType = setASTNodeClass + + def getASTNodeClass(self): + return self._class + + + + def getTokenTypeToASTClassMap(self): + return self._classmap + + def setTokenTypeToASTClassMap(self,amap): + self._classmap = amap + + def error(self, e): + import sys + print >> sys.stderr, e + + def setTokenTypeASTNodeType(self, tokenType, className): + """ + Specify a mapping between a token type and a (AST) class. + """ + if not self._classmap: + self._classmap = {} + + if not className: + try: + del self._classmap[tokenType] + except: + pass + else: + ### here we should also perform actions to ensure that + ### a. class can be loaded + ### b. class is a subclass of AST + ### + assert isinstance(className,type) + assert issubclass(className,AST) ## a & b + ### enter the class + self._classmap[tokenType] = className + + def getASTNodeType(self,tokenType): + """ + For a given token type return the AST node type. First we + lookup a mapping table, second we try _class + and finally we resolve to "antlr.CommonAST". + """ + + # first + if self._classmap: + try: + c = self._classmap[tokenType] + if c: + return c + except: + pass + # second + if self._class: + return self._class + + # default + return CommonAST + + ### methods that have been moved to file scope - just listed + ### here to be somewhat consistent with original API + def dup(self,t): + return antlr.dup(t,self) + + def dupList(self,t): + return antlr.dupList(t,self) + + def dupTree(self,t): + return antlr.dupTree(t,self) + + ### methods moved to other classes + ### 1. makeASTRoot -> Parser + ### 2. addASTChild -> Parser + + ### non-standard: create alias for longish method name + maptype = setTokenTypeASTNodeType + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### ASTVisitor ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +class ASTVisitor(object): + def __init__(self,*args): + pass + + def visit(self,ast): + pass + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### +### static methods and variables ### +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx### + +ASTNULL = ASTNULLType() + +### wh: moved from ASTFactory as there's nothing ASTFactory-specific +### in this method. +def make(*nodes): + if not nodes: + return None + + for i in xrange(0,len(nodes)): + node = nodes[i] + if node: + assert isinstance(node,AST) + + root = nodes[0] + tail = None + if root: + root.setFirstChild(None) + + for i in xrange(1,len(nodes)): + if not nodes[i]: + continue + if not root: + root = tail = nodes[i] + elif not tail: + root.setFirstChild(nodes[i]) + tail = root.getFirstChild() + else: + tail.setNextSibling(nodes[i]) + tail = tail.getNextSibling() + + ### Chase tail to last sibling + while tail.getNextSibling(): + tail = tail.getNextSibling() + return root + +def dup(t,factory): + if not t: + return None + + if factory: + dup_t = factory.create(t.__class__) + else: + raise TypeError("dup function requires ASTFactory argument") + dup_t.initialize(t) + return dup_t + +def dupList(t,factory): + result = dupTree(t,factory) + nt = result + while t: + ## for each sibling of the root + t = t.getNextSibling() + nt.setNextSibling(dupTree(t,factory)) + nt = nt.getNextSibling() + return result + +def dupTree(t,factory): + result = dup(t,factory) + if t: + result.setFirstChild(dupList(t.getFirstChild(),factory)) + return result + +###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +### $Id: antlr.py,v 1.1.1.1 2005/02/02 10:24:36 geronimo Exp $ + +# Local Variables: *** +# mode: python *** +# py-indent-offset: 4 *** +# End: *** diff --git a/src/ptgen/gcc/Makefile b/src/ptgen/gcc/Makefile new file mode 100755 index 0000000..3228a6d --- /dev/null +++ b/src/ptgen/gcc/Makefile @@ -0,0 +1,60 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +CXX= g++ -I../../include +OBJS= lex.yy.o pt_c.tab.o head.o +#TARGET=c_ptgen # this is just a test driver +TARGET=gccptgen.a + +all: $(TARGET) + +gccptgen.a: $(OBJS) + ar -qcs $@ $(OBJS) + +c_ptgen: $(OBJS) main.o + $(CXX) -o c_ptgen $(OBJS) main.o + +#${TARGET}: ${OBJS} main.o +# $(CXX) -o ${TARGET} ${OBJS} main.o + +lex.yy.cc: c.l pt_c.tab.cc + flex -olex.yy.cc c.l + +pt_c.tab.cc: pt_c.y + bison -d pt_c.y -o pt_c.tab.cc + +head.cc pt_c.y: c.y c.y.head c.y.foot + ./mainc.py c.y + +.PHONY: clean +clean: + rm -f *.o lex.yy.cc pt_c.tab* pt_c.y head.cc $(TARGET) diff --git a/src/ptgen/gcc/c.l b/src/ptgen/gcc/c.l new file mode 100755 index 0000000..28345d1 --- /dev/null +++ b/src/ptgen/gcc/c.l @@ -0,0 +1,408 @@ +D [0-9] +L [a-zA-Z_] +H [a-fA-F0-9] +E [Ee][+-]?{D}+ +FS (f|F|l|L) +IS (u|U|l|L)* + +%{ +#include +#include +#include "pt_c.tab.hh" +#include +#include +using namespace std; + +extern map name2id; +void count(); +void comment(); +void cpp_comment(); +void macro(); + +// The global variables are not good for reentrance. +// But for now, the wrong line number doesn't hurt our bug finding because +// we rely on terminal IDs instead of line numbers. +int column = 0; +int line = 1; + +#define YY_DECL int yylex(YYSTYPE *yylvalp) + +%} + +%x COMMENT + +%% + +"/*" { count(); comment();} +"//" { count(); cpp_comment(); } +^"#" {count(); macro();} + +break { + count(); + yylvalp->t = new Terminal(name2id["BREAK"],yytext,line); + return(BREAK); } +case { + count(); + yylvalp->t = new Terminal(name2id["CASE"],yytext,line); + return(CASE); } +continue { + count(); + yylvalp->t = new Terminal(name2id["CONTINUE"],yytext,line); + return(CONTINUE); } +default { + count(); + yylvalp->t = new Terminal(name2id["DEFAULT"],yytext,line); + return(DEFAULT); } +"do" { + count(); + yylvalp->t = new Terminal(name2id["DO"],yytext,line); + return(DO); } +"else" { + count(); + yylvalp->t = new Terminal(name2id["ELSE"],yytext,line); + return(ELSE); } +"enum" { + count(); + yylvalp->t = new Terminal(name2id["ENUM"],yytext,line); + return(ENUM); } +"extern"|"register"|"auto"|"typedef"|"__thread"|"inline"|"__inline__"|"__inline" { + count(); + yylvalp->t = new Terminal(name2id["SCSPEC"],yytext,line); + return(SCSPEC); } +"for" { + count(); + yylvalp->t = new Terminal(name2id["FOR"],yytext,line); + return(FOR); } +"goto" { + count(); + yylvalp->t = new Terminal(name2id["GOTO"],yytext,line); + return(GOTO); } +"if" { + count(); + yylvalp->t = new Terminal(name2id["IF"],yytext,line); + return(IF); } +"return" { + count(); + yylvalp->t = new Terminal(name2id["RETURN"],yytext,line); + return(RETURN); } +"sizeof" { + count(); + yylvalp->t = new Terminal(name2id["SIZEOF"],yytext,line); + return(SIZEOF); } +"static" { + count(); + yylvalp->t = new Terminal(name2id["STATIC"],yytext,line); + return(STATIC); } +"struct" { + count(); + yylvalp->t = new Terminal(name2id["STRUCT"],yytext,line); + return(STRUCT); } +"switch" { + count(); + yylvalp->t = new Terminal(name2id["SWITCH"],yytext,line); + return(SWITCH); } +"union" { + count(); + yylvalp->t = new Terminal(name2id["UNION"],yytext,line); + return(UNION); } +"unsigned"|"long"|"short"|"signed"|"int"|"char"|"float"|"double"|"void"|"bool" { + count(); + yylvalp->t = new Terminal(name2id["TYPESPEC"],yytext,line); + return(TYPESPEC); + } +"const"|"__const__"|"__const"|"volatile"|"__volatile__"|"__volatile"|"restrict"|"__restrict__"|"__restrict" { + count(); + yylvalp->t = new Terminal(name2id["TYPE_QUAL"],yytext,line); + return(TYPE_QUAL); } +"while" { + count(); + yylvalp->t = new Terminal(name2id["WHILE"],yytext,line); + return(WHILE); } +"asm"|"__asm__"|"__asm" { + count(); + yylvalp->t = new Terminal(name2id["ASM_KEYWORD"],yytext,line); + return(ASM_KEYWORD); } +"typeof"|"__typeof__"|"__typeof" { + count(); + yylvalp->t = new Terminal(name2id["TYPEOF"],yytext,line); + return(TYPEOF); } + +"__alignof__"|"__alignof" { + count(); + yylvalp->t = new Terminal(name2id["ALIGNOF"],yytext,line); + return(ALIGNOF); } + +"__attribute__"|"__attribute" { + count(); + yylvalp->t = new Terminal(name2id["ATTRIBUTE"],yytext,line); + return(ATTRIBUTE); } + +"__extension__" { + count(); + yylvalp->t = new Terminal(name2id["EXTENSION"],yytext,line); + return(EXTENSION); } +{L}({L}|{D})*"_t" { + count(); + yylvalp->t = new Terminal(name2id["TYPENAME"],yytext,line); + return(TYPENAME); } +"__init"|"__iomem"|"__user"|"__exit"|"__devexit"|"__devinit"|"__cpuinit"|"__cpuexit"|"__INLINE__"|"__kprobes"|"__inline__"|"__lockfunc"|"__force"|"__sched"|"__deprecated"|"__memzero"|"__pminit"|"__weak"|"__xipram"|"__attribute_used__"|"__apicdebuginit"|"__unused"|"__initdata" {count();} + +{L}({L}|{D})* { + count(); + yylvalp->t = new Terminal(name2id["IDENTIFIER"],yytext,line); + return(IDENTIFIER); } +0[xX]{H}+{IS}? { + count(); + yylvalp->t = new Terminal(name2id["CONSTANT"],yytext,line); + return(CONSTANT); } +0{D}+{IS}? { + count(); + yylvalp->t = new Terminal(name2id["CONSTANT"],yytext,line); + return(CONSTANT); } +{D}+{IS}? { + count(); + yylvalp->t = new Terminal(name2id["CONSTANT"],yytext,line); + return(CONSTANT); } +L?'(\\.|[^\\'])+' { + count(); + yylvalp->t = new Terminal(name2id["CONSTANT"],yytext,line); + return(CONSTANT); } + +{D}+{E}{FS}? { + count(); + yylvalp->t = new Terminal(name2id["CONSTANT"],yytext,line); + return(CONSTANT); } +{D}*"."{D}+({E})?{FS}? { + count(); + yylvalp->t = new Terminal(name2id["CONSTANT"],yytext,line); + return(CONSTANT); } +{D}+"."{D}*({E})?{FS}? { + count(); + yylvalp->t = new Terminal(name2id["CONSTANT"],yytext,line); + return(CONSTANT); } + +L?\"(\\.|[^\\"])*\" { + count(); + yylvalp->t = new Terminal(name2id["STRING"],yytext,line); + return(STRING); } +"..." { + count(); + yylvalp->t = new Terminal(name2id["ELLIPSIS"],yytext,line); + return(ELLIPSIS); } +">>="|"<<="|"+="|"-="|"*="|"/="|"%="|"&="|"^="|"|=" { + count(); + yylvalp->t = new Terminal(name2id["ASSIGN"],yytext,line); + return(ASSIGN); } +">>" { + count(); + yylvalp->t = new Terminal(name2id["RSHIFT"],yytext,line); + return(RSHIFT); } +"<<" { + count(); + yylvalp->t = new Terminal(name2id["LSHIFT"],yytext,line); + return(LSHIFT); } +"++" { + count(); + yylvalp->t = new Terminal(name2id["PLUSPLUS"],yytext,line); + return(PLUSPLUS); } +"--" { + count(); + yylvalp->t = new Terminal(name2id["MINUSMINUS"],yytext,line); + return(MINUSMINUS); } +"->" { + count(); + yylvalp->t = new Terminal(name2id["POINTSAT"],yytext,line); + return(POINTSAT); } +"&&" { + count(); + yylvalp->t = new Terminal(name2id["ANDAND"],yytext,line); + return(ANDAND); } +"||" { + count(); + yylvalp->t = new Terminal(name2id["OROR"],yytext,line); + return(OROR); } +"<="|">=" { + count(); + yylvalp->t = new Terminal(name2id["ARITHCOMPARE"],yytext,line); + return(ARITHCOMPARE); } +"=="|"!=" { + count(); + yylvalp->t = new Terminal(name2id["EOCOMPARE"],yytext,line); + return(EQCOMPARE); } +";" { + count(); + yylvalp->t = new Terminal(name2id["';'"],yytext,line); + return(';'); } +("{"|"<%") { + count(); + yylvalp->t = new Terminal(name2id["'{'"],yytext,line); + return('{'); } +("}"|"%>") { + count(); + yylvalp->t = new Terminal(name2id["'}'"],yytext,line); + return('}'); } +"," { + count(); + yylvalp->t = new Terminal(name2id["','"],yytext,line); + return(','); } +"=" { + count(); + yylvalp->t = new Terminal(name2id["'='"],yytext,line); + return('='); } +"(" { + count(); + yylvalp->t = new Terminal(name2id["'('"],yytext,line); + return('('); } +")" { + count(); + yylvalp->t = new Terminal(name2id["')'"],yytext,line); + return(')'); } +("["|"<:") { + count(); + yylvalp->t = new Terminal(name2id["'['"],yytext,line); + return('['); } +("]"|":>") { + count(); + yylvalp->t = new Terminal(name2id["']'"],yytext,line); + return(']'); } +":" { + count(); + yylvalp->t = new Terminal(name2id["':'"],yytext,line); + return(':'); } +"." { + count(); + yylvalp->t = new Terminal(name2id["'.'"],yytext,line); + return('.'); } +"&" { + count(); + yylvalp->t = new Terminal(name2id["'&'"],yytext,line); + return('&'); } +"!" { + count(); + yylvalp->t = new Terminal(name2id["'!'"],yytext,line); + return('!'); } +"~" { + count(); + yylvalp->t = new Terminal(name2id["'~'"],yytext,line); + return('~'); } +"-" { + count(); + yylvalp->t = new Terminal(name2id["'-'"],yytext,line); + return('-'); } +"+" { + count(); + yylvalp->t = new Terminal(name2id["'+'"],yytext,line); + return('+'); } +"*" { + count(); + yylvalp->t = new Terminal(name2id["'*'"],yytext,line); + return('*'); } +"/" { + count(); + yylvalp->t = new Terminal(name2id["'/'"],yytext,line); + return('/'); } +"%" { + count(); + yylvalp->t = new Terminal(name2id["'%'"],yytext,line); + return('%'); } +"<"|">" { + count(); + yylvalp->t = new Terminal(name2id["ARITHCOMPARE"],yytext,line); + return(ARITHCOMPARE); } +"^" { + count(); + yylvalp->t = new Terminal(name2id["'^'"],yytext,line); + return('^'); } +"|" { + count(); + yylvalp->t = new Terminal(name2id["'|'"],yytext,line); + return('|'); } +"?" { + count(); + yylvalp->t = new Terminal(name2id["'?'"],yytext,line); + return('?'); } + +[ \t\v\n\r] { + count(); + } +. {count();} + +%% + +int yywrap() +{ + return(1); +} + + + +void comment() +{ + int c; + + for (;;) { + while ( (c = yyinput()) != '*' && c != EOF ) { + if (c=='\n') { + line++;column=0; + } else { + column++; + } + } + + if ( c == '*' ) { + while ( (c = yyinput()) == '*' ) + column++; + column++; + if (c =='\n') {line++;column=0;} + if ( c == '/' ) + break; + } + + if ( c == EOF ) { + break; + } + } +} + +void cpp_comment() +{ + int c; + while ((c = yyinput()) != '\n' && c != 0 && c!=EOF) + column++; + line++; + column= 0; +} + +void macro() +{ + int c,last=0; + again: + last= 0; + while ((c = yyinput()) != '\n' && c != 0 && c!=EOF) { + last= c; + } + if (c == '\n' && last == '\\') { + line++; + goto again; + } + + line++; + column= 0; +} + +void count() +{ + int i; + + for (i = 0; yytext[i] != '\0'; i++) + if (yytext[i] == '\n') { + column = 0; + line++; + } else if (yytext[i] == '\t') + column += 4; + else + column++; + + //ECHO; +} + diff --git a/src/ptgen/gcc/c.y b/src/ptgen/gcc/c.y new file mode 100755 index 0000000..4e38da3 --- /dev/null +++ b/src/ptgen/gcc/c.y @@ -0,0 +1,2233 @@ +program: /* empty */ + { if (pedantic) + pedwarn ("ISO C forbids an empty source file"); + } + | extdefs + ; + +/* the reason for the strange actions in this rule + is so that notype_initdecls when reached via datadef + can find a valid list of type and sc specs in $0. */ + +extdefs: + {$$ = NULL_TREE; } extdef + | extdefs {$$ = NULL_TREE; ggc_collect(); } extdef + ; + +extdef: + extdef_1 + { parsing_iso_function_signature = false; } /* Reset after any external definition. */ + ; + +extdef_1: + fndef + | datadef + | ASM_KEYWORD '(' expr ')' ';' + { STRIP_NOPS ($3); + if ((TREE_CODE ($3) == ADDR_EXPR + && TREE_CODE (TREE_OPERAND ($3, 0)) == STRING_CST) + || TREE_CODE ($3) == STRING_CST) + assemble_asm ($3); + else + error ("argument of `asm' is not a constant string"); } + | extension extdef + { RESTORE_EXT_FLAGS ($1); } + ; + +datadef: + setspecs notype_initdecls ';' + { if (pedantic) + error ("ISO C forbids data definition with no type or storage class"); + else + warning ("data definition has no type or storage class"); + + POP_DECLSPEC_STACK; } + | declspecs_nots setspecs notype_initdecls ';' + { POP_DECLSPEC_STACK; } + | declspecs_ts setspecs initdecls ';' + { POP_DECLSPEC_STACK; } + | declspecs ';' + { shadow_tag ($1); } + | error ';' + | error '}' + | ';' + { if (pedantic) + pedwarn ("ISO C does not allow extra `;' outside of a function"); } + ; + +fndef: + declspecs_ts setspecs declarator + { if (! start_function (current_declspecs, $3, + all_prefix_attributes)) + YYERROR1; + } + old_style_parm_decls save_location + { DECL_SOURCE_LOCATION (current_function_decl) = $6; + store_parm_decls (); } + compstmt_or_error + { finish_function (); + POP_DECLSPEC_STACK; } + | declspecs_ts setspecs declarator error + { POP_DECLSPEC_STACK; } + | declspecs_nots setspecs notype_declarator + { if (! start_function (current_declspecs, $3, + all_prefix_attributes)) + YYERROR1; + } + old_style_parm_decls save_location + { DECL_SOURCE_LOCATION (current_function_decl) = $6; + store_parm_decls (); } + compstmt_or_error + { finish_function (); + POP_DECLSPEC_STACK; } + | declspecs_nots setspecs notype_declarator error + { POP_DECLSPEC_STACK; } + | setspecs notype_declarator + { if (! start_function (NULL_TREE, $2, + all_prefix_attributes)) + YYERROR1; + } + old_style_parm_decls save_location + { DECL_SOURCE_LOCATION (current_function_decl) = $5; + store_parm_decls (); } + compstmt_or_error + { finish_function (); + POP_DECLSPEC_STACK; } + | setspecs notype_declarator error + { POP_DECLSPEC_STACK; } + ; + +identifier: + IDENTIFIER + | TYPENAME + ; + +unop: '&' + { $$ = ADDR_EXPR; } + | '-' + { $$ = NEGATE_EXPR; } + | '+' + { $$ = CONVERT_EXPR; + if (warn_traditional && !in_system_header) + warning ("traditional C rejects the unary plus operator"); + } + | PLUSPLUS + { $$ = PREINCREMENT_EXPR; } + | MINUSMINUS + { $$ = PREDECREMENT_EXPR; } + | '~' + { $$ = BIT_NOT_EXPR; } + | '!' + { $$ = TRUTH_NOT_EXPR; } + ; + +expr: nonnull_exprlist + { $$ = build_compound_expr ($1); } + ; + +exprlist: + /* empty */ + { $$ = NULL_TREE; } + | nonnull_exprlist + ; + +nonnull_exprlist: + expr_no_commas + { $$ = build_tree_list (NULL_TREE, $1); } + | nonnull_exprlist ',' expr_no_commas + { chainon ($1, build_tree_list (NULL_TREE, $3)); } + ; + +unary_expr: + primary + | '*' cast_expr %prec UNARY + { $$ = build_indirect_ref ($2, "unary *"); } + /* __extension__ turns off -pedantic for following primary. */ + | extension cast_expr %prec UNARY + { $$ = $2; + RESTORE_EXT_FLAGS ($1); } + | unop cast_expr %prec UNARY + { $$ = build_unary_op ($1, $2, 0); + overflow_warning ($$); } + /* Refer to the address of a label as a pointer. */ + | ANDAND identifier + { $$ = finish_label_address_expr ($2); } + | sizeof unary_expr %prec UNARY + { skip_evaluation--; + if (TREE_CODE ($2) == COMPONENT_REF + && DECL_C_BIT_FIELD (TREE_OPERAND ($2, 1))) + error ("`sizeof' applied to a bit-field"); + $$ = c_sizeof (TREE_TYPE ($2)); } + | sizeof '(' typename ')' %prec HYPERUNARY + { skip_evaluation--; + $$ = c_sizeof (groktypename ($3)); } + | alignof unary_expr %prec UNARY + { skip_evaluation--; + $$ = c_alignof_expr ($2); } + | alignof '(' typename ')' %prec HYPERUNARY + { skip_evaluation--; + $$ = c_alignof (groktypename ($3)); } + | REALPART cast_expr %prec UNARY + { $$ = build_unary_op (REALPART_EXPR, $2, 0); } + | IMAGPART cast_expr %prec UNARY + { $$ = build_unary_op (IMAGPART_EXPR, $2, 0); } + ; + +sizeof: + SIZEOF { skip_evaluation++; } + ; + +alignof: + ALIGNOF { skip_evaluation++; } + ; + +typeof: + TYPEOF { skip_evaluation++; } + ; + +cast_expr: + unary_expr + | '(' typename ')' cast_expr %prec UNARY + { $$ = c_cast_expr ($2, $4); } + ; + +expr_no_commas: + cast_expr + | expr_no_commas '+' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas '-' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas '*' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas '/' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas '%' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas LSHIFT expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas RSHIFT expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas ARITHCOMPARE expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas EQCOMPARE expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas '&' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas '|' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas '^' expr_no_commas + { $$ = parser_build_binary_op ($2, $1, $3); } + | expr_no_commas ANDAND + { $1 = c_common_truthvalue_conversion + (default_conversion ($1)); + skip_evaluation += $1 == truthvalue_false_node; } + expr_no_commas + { skip_evaluation -= $1 == truthvalue_false_node; + $$ = parser_build_binary_op (TRUTH_ANDIF_EXPR, $1, $4); } + | expr_no_commas OROR + { $1 = c_common_truthvalue_conversion + (default_conversion ($1)); + skip_evaluation += $1 == truthvalue_true_node; } + expr_no_commas + { skip_evaluation -= $1 == truthvalue_true_node; + $$ = parser_build_binary_op (TRUTH_ORIF_EXPR, $1, $4); } + | expr_no_commas '?' + { $1 = c_common_truthvalue_conversion + (default_conversion ($1)); + skip_evaluation += $1 == truthvalue_false_node; } + expr ':' + { skip_evaluation += (($1 == truthvalue_true_node) + - ($1 == truthvalue_false_node)); } + expr_no_commas + { skip_evaluation -= $1 == truthvalue_true_node; + $$ = build_conditional_expr ($1, $4, $7); } + | expr_no_commas '?' + { if (pedantic) + pedwarn ("ISO C forbids omitting the middle term of a ?: expression"); + /* Make sure first operand is calculated only once. */ + $2 = save_expr ($1); + $1 = c_common_truthvalue_conversion + (default_conversion ($2)); + skip_evaluation += $1 == truthvalue_true_node; } + ':' expr_no_commas + { skip_evaluation -= $1 == truthvalue_true_node; + $$ = build_conditional_expr ($1, $2, $5); } + | expr_no_commas '=' expr_no_commas + { char class; + $$ = build_modify_expr ($1, NOP_EXPR, $3); + class = TREE_CODE_CLASS (TREE_CODE ($$)); + if (IS_EXPR_CODE_CLASS (class)) + C_SET_EXP_ORIGINAL_CODE ($$, MODIFY_EXPR); + } + | expr_no_commas ASSIGN expr_no_commas + { char class; + $$ = build_modify_expr ($1, $2, $3); + /* This inhibits warnings in + c_common_truthvalue_conversion. */ + class = TREE_CODE_CLASS (TREE_CODE ($$)); + if (IS_EXPR_CODE_CLASS (class)) + C_SET_EXP_ORIGINAL_CODE ($$, ERROR_MARK); + } + ; + +primary: + IDENTIFIER + { + if (yychar == YYEMPTY) + yychar = YYLEX; + $$ = build_external_ref ($1, yychar == '('); + } + | IDENTIFIER STRING + | IDENTIFIER STRING IDENTIFIER + | STRING IDENTIFIER + | CONSTANT + | STRING + | FUNC_NAME + { $$ = fname_decl (C_RID_CODE ($$), $$); } + | '(' typename ')' '{' + { start_init (NULL_TREE, NULL, 0); + $2 = groktypename ($2); + really_start_incremental_init ($2); } + initlist_maybe_comma '}' %prec UNARY + { tree constructor = pop_init_level (0); + tree type = $2; + finish_init (); + + if (pedantic && ! flag_isoc99) + pedwarn ("ISO C90 forbids compound literals"); + $$ = build_compound_literal (type, constructor); + } + | '(' expr ')' + { char class = TREE_CODE_CLASS (TREE_CODE ($2)); + if (IS_EXPR_CODE_CLASS (class)) + C_SET_EXP_ORIGINAL_CODE ($2, ERROR_MARK); + $$ = $2; } + | '(' error ')' + { $$ = error_mark_node; } + | compstmt_primary_start compstmt_nostart ')' + { tree saved_last_tree; + + if (pedantic) + pedwarn ("ISO C forbids braced-groups within expressions"); + saved_last_tree = COMPOUND_BODY ($1); + RECHAIN_STMTS ($1, COMPOUND_BODY ($1)); + last_tree = saved_last_tree; + TREE_CHAIN (last_tree) = NULL_TREE; + if (!last_expr_type) + last_expr_type = void_type_node; + $$ = build1 (STMT_EXPR, last_expr_type, $1); + TREE_SIDE_EFFECTS ($$) = 1; + } + | compstmt_primary_start error ')' + { + last_tree = COMPOUND_BODY ($1); + TREE_CHAIN (last_tree) = NULL_TREE; + $$ = error_mark_node; + } + | primary '(' exprlist ')' %prec '.' + { $$ = build_function_call ($1, $3); } + | VA_ARG '(' expr_no_commas ',' typename ')' + { $$ = build_va_arg ($3, groktypename ($5)); } + + | CHOOSE_EXPR '(' expr_no_commas ',' expr_no_commas ',' expr_no_commas ')' + { + tree c; + + c = fold ($3); + STRIP_NOPS (c); + if (TREE_CODE (c) != INTEGER_CST) + error ("first argument to __builtin_choose_expr not a constant"); + $$ = integer_zerop (c) ? $7 : $5; + } + | TYPES_COMPATIBLE_P '(' typename ',' typename ')' + { + tree e1, e2; + + e1 = TYPE_MAIN_VARIANT (groktypename ($3)); + e2 = TYPE_MAIN_VARIANT (groktypename ($5)); + + $$ = comptypes (e1, e2, COMPARE_STRICT) + ? build_int_2 (1, 0) : build_int_2 (0, 0); + } + | primary '[' expr ']' %prec '.' + { $$ = build_array_ref ($1, $3); } + | primary '.' identifier + { + $$ = build_component_ref ($1, $3); + } + | primary POINTSAT identifier + { + tree expr = build_indirect_ref ($1, "->"); + + $$ = build_component_ref (expr, $3); + } + | primary PLUSPLUS + { $$ = build_unary_op (POSTINCREMENT_EXPR, $1, 0); } + | primary MINUSMINUS + { $$ = build_unary_op (POSTDECREMENT_EXPR, $1, 0); } + ; + +old_style_parm_decls: + old_style_parm_decls_1 + { + parsing_iso_function_signature = false; /* Reset after decls. */ + } + ; + +old_style_parm_decls_1: + /* empty */ + { + if (warn_traditional && !in_system_header + && parsing_iso_function_signature) + warning ("traditional C rejects ISO C style function definitions"); + if (warn_old_style_definition && !in_system_header + && !parsing_iso_function_signature) + warning ("old-style parameter declaration"); + parsing_iso_function_signature = false; /* Reset after warning. */ + } + | datadecls + { + if (warn_old_style_definition && !in_system_header) + warning ("old-style parameter declaration"); + } + ; + +/* The following are analogous to lineno_decl, decls and decl + except that they do not allow nested functions. + They are used for old-style parm decls. */ +lineno_datadecl: + save_location datadecl + { } + ; + +datadecls: + lineno_datadecl + | errstmt + | datadecls lineno_datadecl + | lineno_datadecl errstmt + ; + +/* We don't allow prefix attributes here because they cause reduce/reduce + conflicts: we can't know whether we're parsing a function decl with + attribute suffix, or function defn with attribute prefix on first old + style parm. */ +datadecl: + declspecs_ts_nosa setspecs initdecls ';' + { POP_DECLSPEC_STACK; } + | declspecs_nots_nosa setspecs notype_initdecls ';' + { POP_DECLSPEC_STACK; } + | declspecs_ts_nosa ';' + { shadow_tag_warned ($1, 1); + pedwarn ("empty declaration"); } + | declspecs_nots_nosa ';' + { pedwarn ("empty declaration"); } + ; + +/* This combination which saves a lineno before a decl + is the normal thing to use, rather than decl itself. + This is to avoid shift/reduce conflicts in contexts + where statement labels are allowed. */ +lineno_decl: + save_location decl + { } + ; + +/* records the type and storage class specs to use for processing + the declarators that follow. + Maintains a stack of outer-level values of current_declspecs, + for the sake of parm declarations nested in function declarators. */ +setspecs: /* empty */ + { pending_xref_error (); + PUSH_DECLSPEC_STACK; + split_specs_attrs ($0, + ¤t_declspecs, &prefix_attributes); + all_prefix_attributes = prefix_attributes; } + ; + +/* Possibly attributes after a comma, which should reset all_prefix_attributes + to prefix_attributes with these ones chained on the front. */ +maybe_resetattrs: + maybe_attribute + { all_prefix_attributes = chainon ($1, prefix_attributes); } + ; + +decl: + declspecs_ts setspecs initdecls ';' + { POP_DECLSPEC_STACK; } + | declspecs_nots setspecs notype_initdecls ';' + { POP_DECLSPEC_STACK; } + | declspecs_ts setspecs nested_function + { POP_DECLSPEC_STACK; } + | declspecs_nots setspecs notype_nested_function + { POP_DECLSPEC_STACK; } + | declspecs ';' + { shadow_tag ($1); } + | extension decl + { RESTORE_EXT_FLAGS ($1); } + ; + +/* A list of declaration specifiers. These are: + + - Storage class specifiers (scspec), which for GCC currently includes + function specifiers ("inline"). + + - Type specifiers (typespec_*). + + - Type qualifiers (TYPE_QUAL). + + - Attribute specifier lists (attributes). + + These are stored as a TREE_LIST; the head of the list is the last + item in the specifier list. Each entry in the list has either a + TREE_PURPOSE that is an attribute specifier list, or a TREE_VALUE that + is a single other specifier or qualifier; and a TREE_CHAIN that is the + rest of the list. TREE_STATIC is set on the list if something other + than a storage class specifier or attribute has been seen; this is used + to warn for the obsolescent usage of storage class specifiers other than + at the start of the list. (Doing this properly would require function + specifiers to be handled separately from storage class specifiers.) + + The various cases below are classified according to: + + (a) Whether a storage class specifier is included or not; some + places in the grammar disallow storage class specifiers (_sc or _nosc). + + (b) Whether a type specifier has been seen; after a type specifier, + a typedef name is an identifier to redeclare (_ts or _nots). + + (c) Whether the list starts with an attribute; in certain places, + the grammar requires specifiers that don't start with an attribute + (_sa or _nosa). + + (d) Whether the list ends with an attribute (or a specifier such that + any following attribute would have been parsed as part of that specifier); + this avoids shift-reduce conflicts in the parsing of attributes + (_ea or _noea). + + TODO: + + (i) Distinguish between function specifiers and storage class specifiers, + at least for the purpose of warnings about obsolescent usage. + + (ii) Halve the number of productions here by eliminating the _sc/_nosc + distinction and instead checking where required that storage class + specifiers aren't present. */ + +/* Declspecs which contain at least one type specifier or typedef name. + (Just `const' or `volatile' is not enough.) + A typedef'd name following these is taken as a name to be declared. + Declspecs have a non-NULL TREE_VALUE, attributes do not. */ + +declspecs_nosc_nots_nosa_noea: + TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $1, NULL_TREE); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_nosa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_nosa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +declspecs_nosc_nots_nosa_ea: + declspecs_nosc_nots_nosa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_nosc_nots_sa_noea: + declspecs_nosc_nots_sa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_sa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +declspecs_nosc_nots_sa_ea: + attributes + { $$ = tree_cons ($1, NULL_TREE, NULL_TREE); + TREE_STATIC ($$) = 0; } + | declspecs_nosc_nots_sa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_nosc_ts_nosa_noea: + typespec_nonattr + { $$ = tree_cons (NULL_TREE, $1, NULL_TREE); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_nosa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_nosa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_nosa_noea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_nosa_ea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_nosa_noea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_nosa_ea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +declspecs_nosc_ts_nosa_ea: + typespec_attr + { $$ = tree_cons (NULL_TREE, $1, NULL_TREE); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_nosa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_nosc_ts_nosa_noea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_nosa_ea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_nosa_noea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_nosa_ea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +declspecs_nosc_ts_sa_noea: + declspecs_nosc_ts_sa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_sa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_sa_noea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_sa_ea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_sa_noea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_sa_ea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +declspecs_nosc_ts_sa_ea: + declspecs_nosc_ts_sa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_nosc_ts_sa_noea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_sa_ea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_sa_noea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_sa_ea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +declspecs_sc_nots_nosa_noea: + scspec + { $$ = tree_cons (NULL_TREE, $1, NULL_TREE); + TREE_STATIC ($$) = 0; } + | declspecs_sc_nots_nosa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_nosa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_nosa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_nosc_nots_nosa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_nots_nosa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_nots_nosa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_sc_nots_nosa_ea: + declspecs_sc_nots_nosa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_sc_nots_sa_noea: + declspecs_sc_nots_sa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_sa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_nots_sa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_nosc_nots_sa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_nots_sa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_nots_sa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_sc_nots_sa_ea: + declspecs_sc_nots_sa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_sc_ts_nosa_noea: + declspecs_sc_ts_nosa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_nosa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_nosa_noea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_nosa_ea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_nosa_noea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_nosa_ea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_nosa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_nosc_ts_nosa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_ts_nosa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_ts_nosa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_sc_ts_nosa_ea: + declspecs_sc_ts_nosa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_ts_nosa_noea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_nosa_ea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_nosa_noea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_nosa_ea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +declspecs_sc_ts_sa_noea: + declspecs_sc_ts_sa_noea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_sa_ea TYPE_QUAL + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_sa_noea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_sa_ea typespec_reserved_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_sa_noea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_sa_ea typespec_nonattr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_nosc_ts_sa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_nosc_ts_sa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_ts_sa_noea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_ts_sa_ea scspec + { if (extra_warnings && TREE_STATIC ($1)) + warning ("`%s' is not at beginning of declaration", + IDENTIFIER_POINTER ($2)); + $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + ; + +declspecs_sc_ts_sa_ea: + declspecs_sc_ts_sa_noea attributes + { $$ = tree_cons ($2, NULL_TREE, $1); + TREE_STATIC ($$) = TREE_STATIC ($1); } + | declspecs_sc_ts_sa_noea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_ts_sa_ea typespec_reserved_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_sa_noea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + | declspecs_sc_nots_sa_ea typespec_attr + { $$ = tree_cons (NULL_TREE, $2, $1); + TREE_STATIC ($$) = 1; } + ; + +/* Particular useful classes of declspecs. */ +declspecs_ts: + declspecs_nosc_ts_nosa_noea + | declspecs_nosc_ts_nosa_ea + | declspecs_nosc_ts_sa_noea + | declspecs_nosc_ts_sa_ea + | declspecs_sc_ts_nosa_noea + | declspecs_sc_ts_nosa_ea + | declspecs_sc_ts_sa_noea + | declspecs_sc_ts_sa_ea + ; + +declspecs_nots: + declspecs_nosc_nots_nosa_noea + | declspecs_nosc_nots_nosa_ea + | declspecs_nosc_nots_sa_noea + | declspecs_nosc_nots_sa_ea + | declspecs_sc_nots_nosa_noea + | declspecs_sc_nots_nosa_ea + | declspecs_sc_nots_sa_noea + | declspecs_sc_nots_sa_ea + ; + +declspecs_ts_nosa: + declspecs_nosc_ts_nosa_noea + | declspecs_nosc_ts_nosa_ea + | declspecs_sc_ts_nosa_noea + | declspecs_sc_ts_nosa_ea + ; + +declspecs_nots_nosa: + declspecs_nosc_nots_nosa_noea + | declspecs_nosc_nots_nosa_ea + | declspecs_sc_nots_nosa_noea + | declspecs_sc_nots_nosa_ea + ; + +declspecs_nosc_ts: + declspecs_nosc_ts_nosa_noea + | declspecs_nosc_ts_nosa_ea + | declspecs_nosc_ts_sa_noea + | declspecs_nosc_ts_sa_ea + ; + +declspecs_nosc_nots: + declspecs_nosc_nots_nosa_noea + | declspecs_nosc_nots_nosa_ea + | declspecs_nosc_nots_sa_noea + | declspecs_nosc_nots_sa_ea + ; + +declspecs_nosc: + declspecs_nosc_ts_nosa_noea + | declspecs_nosc_ts_nosa_ea + | declspecs_nosc_ts_sa_noea + | declspecs_nosc_ts_sa_ea + | declspecs_nosc_nots_nosa_noea + | declspecs_nosc_nots_nosa_ea + | declspecs_nosc_nots_sa_noea + | declspecs_nosc_nots_sa_ea + ; + +declspecs: + declspecs_nosc_nots_nosa_noea + | declspecs_nosc_nots_nosa_ea + | declspecs_nosc_nots_sa_noea + | declspecs_nosc_nots_sa_ea + | declspecs_nosc_ts_nosa_noea + | declspecs_nosc_ts_nosa_ea + | declspecs_nosc_ts_sa_noea + | declspecs_nosc_ts_sa_ea + | declspecs_sc_nots_nosa_noea + | declspecs_sc_nots_nosa_ea + | declspecs_sc_nots_sa_noea + | declspecs_sc_nots_sa_ea + | declspecs_sc_ts_nosa_noea + | declspecs_sc_ts_nosa_ea + | declspecs_sc_ts_sa_noea + | declspecs_sc_ts_sa_ea + ; + +/* A (possibly empty) sequence of type qualifiers and attributes. */ +maybe_type_quals_attrs: + /* empty */ + { $$ = NULL_TREE; } + | declspecs_nosc_nots + { $$ = $1; } + ; + +/* A type specifier (but not a type qualifier). + Once we have seen one of these in a declaration, + if a typedef name appears then it is being redeclared. + + The _reserved versions start with a reserved word and may appear anywhere + in the declaration specifiers; the _nonreserved versions may only + appear before any other type specifiers, and after that are (if names) + being redeclared. + + FIXME: should the _nonreserved version be restricted to names being + redeclared only? The other entries there relate only the GNU extensions + and Objective C, and are historically parsed thus, and don't make sense + after other type specifiers, but it might be cleaner to count them as + _reserved. + + _attr means: specifiers that either end with attributes, + or are such that any following attributes would + be parsed as part of the specifier. + + _nonattr: specifiers. */ + +typespec_nonattr: + typespec_reserved_nonattr + | typespec_nonreserved_nonattr + ; + +typespec_attr: + typespec_reserved_attr + ; + +typespec_reserved_nonattr: + TYPESPEC + { OBJC_NEED_RAW_IDENTIFIER (1); } + | structsp_nonattr + ; + +typespec_reserved_attr: + structsp_attr + ; + +typespec_nonreserved_nonattr: + TYPENAME + { /* For a typedef name, record the meaning, not the name. + In case of `foo foo, bar;'. */ + $$ = lookup_name ($1); } + | typeof '(' expr ')' + { skip_evaluation--; + if (TREE_CODE ($3) == COMPONENT_REF + && DECL_C_BIT_FIELD (TREE_OPERAND ($3, 1))) + error ("`typeof' applied to a bit-field"); + $$ = TREE_TYPE ($3); } + | typeof '(' typename ')' + { skip_evaluation--; $$ = groktypename ($3); } + ; + +/* typespec_nonreserved_attr does not exist. */ + +initdecls: + initdcl + | initdecls ',' maybe_resetattrs initdcl + ; + +notype_initdecls: + notype_initdcl + | notype_initdecls ',' maybe_resetattrs notype_initdcl + ; + +maybeasm: + /* empty */ + { $$ = NULL_TREE; } + | ASM_KEYWORD '(' STRING ')' + { $$ = $3; } + ; + +initdcl: + declarator maybeasm maybe_attribute '=' + { $$ = start_decl ($1, current_declspecs, 1, + chainon ($3, all_prefix_attributes)); + start_init ($$, $2, global_bindings_p ()); } + init +/* Note how the declaration of the variable is in effect while its init is parsed! */ + { finish_init (); + finish_decl ($5, $6, $2); } + | declarator maybeasm maybe_attribute + { tree d = start_decl ($1, current_declspecs, 0, + chainon ($3, all_prefix_attributes)); + finish_decl (d, NULL_TREE, $2); + } + ; + +notype_initdcl: + notype_declarator maybeasm maybe_attribute '=' + { $$ = start_decl ($1, current_declspecs, 1, + chainon ($3, all_prefix_attributes)); + start_init ($$, $2, global_bindings_p ()); } + init +/* Note how the declaration of the variable is in effect while its init is parsed! */ + { finish_init (); + finish_decl ($5, $6, $2); } + | notype_declarator maybeasm maybe_attribute + { tree d = start_decl ($1, current_declspecs, 0, + chainon ($3, all_prefix_attributes)); + finish_decl (d, NULL_TREE, $2); } + ; +/* the * rules are dummies to accept the Apollo extended syntax + so that the header files compile. */ +maybe_attribute: + /* empty */ + { $$ = NULL_TREE; } + | attributes + { $$ = $1; } + ; + +attributes: + attribute + { $$ = $1; } + | attributes attribute + { $$ = chainon ($1, $2); } + ; + +attribute: + ATTRIBUTE '(' '(' attribute_list ')' ')' + { $$ = $4; } + ; + +attribute_list: + attrib + { $$ = $1; } + | attribute_list ',' attrib + { $$ = chainon ($1, $3); } + ; + +attrib: + /* empty */ + { $$ = NULL_TREE; } + | any_word + { $$ = build_tree_list ($1, NULL_TREE); } + | any_word '(' IDENTIFIER ')' + { $$ = build_tree_list ($1, build_tree_list (NULL_TREE, $3)); } + | any_word '(' IDENTIFIER ',' nonnull_exprlist ')' + { $$ = build_tree_list ($1, tree_cons (NULL_TREE, $3, $5)); } + | any_word '(' exprlist ')' + { $$ = build_tree_list ($1, $3); } + ; + +/* This still leaves out most reserved keywords, + shouldn't we include them? */ + +any_word: + identifier + | scspec + | TYPESPEC + | TYPE_QUAL + ; + +scspec: + STATIC + | SCSPEC + ; + +/* Initializers. `init' is the entry point. */ + +init: + expr_no_commas + | '{' + { really_start_incremental_init (NULL_TREE); } + initlist_maybe_comma '}' + { $$ = pop_init_level (0); } + | error + { $$ = error_mark_node; } + ; + +/* `initlist_maybe_comma' is the guts of an initializer in braces. */ +initlist_maybe_comma: + /* empty */ + { if (pedantic) + pedwarn ("ISO C forbids empty initializer braces"); } + | initlist1 maybecomma + ; + +initlist1: + initelt + | initlist1 ',' initelt + ; + +/* `initelt' is a single element of an initializer. + It may use braces. */ +initelt: + designator_list '=' initval + { if (pedantic && ! flag_isoc99) + pedwarn ("ISO C90 forbids specifying subobject to initialize"); } + | designator initval + { if (pedantic) + pedwarn ("obsolete use of designated initializer without `='"); } + | identifier ':' + { set_init_label ($1); + if (pedantic) + pedwarn ("obsolete use of designated initializer with `:'"); } + initval + {} + | initval + ; + +initval: + '{' + { push_init_level (0); } + initlist_maybe_comma '}' + { process_init_element (pop_init_level (0)); } + | expr_no_commas + { process_init_element ($1); } + | error + ; + +designator_list: + designator + | designator_list designator + ; + +designator: + '.' identifier + { set_init_label ($2); } + | '[' expr_no_commas ELLIPSIS expr_no_commas ']' + { set_init_index ($2, $4); + if (pedantic) + pedwarn ("ISO C forbids specifying range of elements to initialize"); } + | '[' expr_no_commas ']' + { set_init_index ($2, NULL_TREE); } + ; + +nested_function: + declarator + { if (pedantic) + pedwarn ("ISO C forbids nested functions"); + + push_function_context (); + if (! start_function (current_declspecs, $1, + all_prefix_attributes)) + { + pop_function_context (); + YYERROR1; + } + parsing_iso_function_signature = false; /* Don't warn about nested functions. */ + } + old_style_parm_decls save_location + { tree decl = current_function_decl; + DECL_SOURCE_LOCATION (decl) = $4; + store_parm_decls (); } +/* This used to use compstmt_or_error. + That caused a bug with input `f(g) int g {}', + where the use of YYERROR1 above caused an error + which then was handled by compstmt_or_error. + There followed a repeated execution of that same rule, + which called YYERROR1 again, and so on. */ + compstmt + { tree decl = current_function_decl; + finish_function (); + pop_function_context (); + add_decl_stmt (decl); } + ; + +notype_nested_function: + notype_declarator + { if (pedantic) + pedwarn ("ISO C forbids nested functions"); + + push_function_context (); + if (! start_function (current_declspecs, $1, + all_prefix_attributes)) + { + pop_function_context (); + YYERROR1; + } + parsing_iso_function_signature = false; /* Don't warn about nested functions. */ + } + old_style_parm_decls save_location + { tree decl = current_function_decl; + DECL_SOURCE_LOCATION (decl) = $4; + store_parm_decls (); } +/* This used to use compstmt_or_error. + That caused a bug with input `f(g) int g {}', + where the use of YYERROR1 above caused an error + which then was handled by compstmt_or_error. + There followed a repeated execution of that same rule, + which called YYERROR1 again, and so on. */ + compstmt + { tree decl = current_function_decl; + finish_function (); + pop_function_context (); + add_decl_stmt (decl); } + ; + +/* Any kind of declarator (thus, all declarators allowed + after an explicit typespec). */ + +declarator: + after_type_declarator + | notype_declarator + ; + +/* A declarator that is allowed only after an explicit typespec. */ + +after_type_declarator: + '(' maybe_attribute after_type_declarator ')' + { $$ = $2 ? tree_cons ($2, $3, NULL_TREE) : $3; } + | after_type_declarator '(' parmlist_or_identifiers %prec '.' + { $$ = build_nt (CALL_EXPR, $1, $3, NULL_TREE); } +/* | after_type_declarator '(' error ')' %prec '.' + { $$ = build_nt (CALL_EXPR, $1, NULL_TREE, NULL_TREE); + poplevel (0, 0, 0); } */ + | after_type_declarator array_declarator %prec '.' + { $$ = set_array_declarator_type ($2, $1, 0); } + | '*' maybe_type_quals_attrs after_type_declarator %prec UNARY + { $$ = make_pointer_declarator ($2, $3); } + | TYPENAME + ; + +/* Kinds of declarator that can appear in a parameter list + in addition to notype_declarator. This is like after_type_declarator + but does not allow a typedef name in parentheses as an identifier + (because it would conflict with a function with that typedef as arg). */ +parm_declarator: + parm_declarator_starttypename + | parm_declarator_nostarttypename + ; + +parm_declarator_starttypename: + parm_declarator_starttypename '(' parmlist_or_identifiers %prec '.' + { $$ = build_nt (CALL_EXPR, $1, $3, NULL_TREE); } +/* | parm_declarator_starttypename '(' error ')' %prec '.' + { $$ = build_nt (CALL_EXPR, $1, NULL_TREE, NULL_TREE); + poplevel (0, 0, 0); } */ + | parm_declarator_starttypename array_declarator %prec '.' + { $$ = set_array_declarator_type ($2, $1, 0); } + | TYPENAME + ; + +parm_declarator_nostarttypename: + parm_declarator_nostarttypename '(' parmlist_or_identifiers %prec '.' + { $$ = build_nt (CALL_EXPR, $1, $3, NULL_TREE); } +/* | parm_declarator_nostarttypename '(' error ')' %prec '.' + { $$ = build_nt (CALL_EXPR, $1, NULL_TREE, NULL_TREE); + poplevel (0, 0, 0); } */ + | parm_declarator_nostarttypename array_declarator %prec '.' + { $$ = set_array_declarator_type ($2, $1, 0); } + | '*' maybe_type_quals_attrs parm_declarator_starttypename %prec UNARY + { $$ = make_pointer_declarator ($2, $3); } + | '*' maybe_type_quals_attrs parm_declarator_nostarttypename %prec UNARY + { $$ = make_pointer_declarator ($2, $3); } + | '(' maybe_attribute parm_declarator_nostarttypename ')' + { $$ = $2 ? tree_cons ($2, $3, NULL_TREE) : $3; } + ; + +/* A declarator allowed whether or not there has been + an explicit typespec. These cannot redeclare a typedef-name. */ + +notype_declarator: + notype_declarator '(' parmlist_or_identifiers %prec '.' + { $$ = build_nt (CALL_EXPR, $1, $3, NULL_TREE); } +/* | notype_declarator '(' error ')' %prec '.' + { $$ = build_nt (CALL_EXPR, $1, NULL_TREE, NULL_TREE); + poplevel (0, 0, 0); } */ + | '(' maybe_attribute notype_declarator ')' + { $$ = $2 ? tree_cons ($2, $3, NULL_TREE) : $3; } + | '*' maybe_type_quals_attrs notype_declarator %prec UNARY + { $$ = make_pointer_declarator ($2, $3); } + | notype_declarator array_declarator %prec '.' + { $$ = set_array_declarator_type ($2, $1, 0); } + | IDENTIFIER + ; + +struct_head: + STRUCT + { $$ = NULL_TREE; } + | STRUCT attributes + { $$ = $2; } + ; + +union_head: + UNION + { $$ = NULL_TREE; } + | UNION attributes + { $$ = $2; } + ; + +enum_head: + ENUM + { $$ = NULL_TREE; } + | ENUM attributes + { $$ = $2; } + ; + +/* structsp_attr: struct/union/enum specifiers that either + end with attributes, or are such that any following attributes would + be parsed as part of the struct/union/enum specifier. + + structsp_nonattr: other struct/union/enum specifiers. */ + +structsp_attr: + struct_head identifier '{' + { $$ = start_struct (RECORD_TYPE, $2); + /* Start scope of tag before parsing components. */ + } + component_decl_list '}' maybe_attribute + { $$ = finish_struct ($4, nreverse ($5), + chainon ($1, $7)); } + | struct_head '{' component_decl_list '}' maybe_attribute + { $$ = finish_struct (start_struct (RECORD_TYPE, NULL_TREE), + nreverse ($3), chainon ($1, $5)); + } + | union_head identifier '{' + { $$ = start_struct (UNION_TYPE, $2); } + component_decl_list '}' maybe_attribute + { $$ = finish_struct ($4, nreverse ($5), + chainon ($1, $7)); } + | union_head '{' component_decl_list '}' maybe_attribute + { $$ = finish_struct (start_struct (UNION_TYPE, NULL_TREE), + nreverse ($3), chainon ($1, $5)); + } + | enum_head identifier '{' + { $$ = start_enum ($2); } + enumlist maybecomma_warn '}' maybe_attribute + { $$ = finish_enum ($4, nreverse ($5), + chainon ($1, $8)); } + | enum_head '{' + { $$ = start_enum (NULL_TREE); } + enumlist maybecomma_warn '}' maybe_attribute + { $$ = finish_enum ($3, nreverse ($4), + chainon ($1, $7)); } + ; + +structsp_nonattr: + struct_head identifier + { $$ = xref_tag (RECORD_TYPE, $2); } + | union_head identifier + { $$ = xref_tag (UNION_TYPE, $2); } + | enum_head identifier + { $$ = xref_tag (ENUMERAL_TYPE, $2); + /* In ISO C, enumerated types can be referred to + only if already defined. */ + if (pedantic && !COMPLETE_TYPE_P ($$)) + pedwarn ("ISO C forbids forward references to `enum' types"); } + ; + +maybecomma: + /* empty */ + | ',' + ; + +maybecomma_warn: + /* empty */ + | ',' + { if (pedantic && ! flag_isoc99) + pedwarn ("comma at end of enumerator list"); } + ; + +/* We chain the components in reverse order. They are put in forward + order in structsp_attr. + + Note that component_declarator returns single decls, so components + and components_notype can use TREE_CHAIN directly, wheras components + and components_notype return lists (of comma separated decls), so + component_decl_list and component_decl_list2 must use chainon. + + The theory behind all this is that there will be more semicolon + separated fields than comma separated fields, and so we'll be + minimizing the number of node traversals required by chainon. */ + +component_decl_list: + component_decl_list2 + { $$ = $1; } + | component_decl_list2 component_decl + { $$ = chainon ($2, $1); + pedwarn ("no semicolon at end of struct or union"); } + ; + +component_decl_list2: /* empty */ + { $$ = NULL_TREE; } + | component_decl_list2 component_decl ';' + { $$ = chainon ($2, $1); } + | component_decl_list2 ';' + { if (pedantic) + pedwarn ("extra semicolon in struct or union specified"); } + ; + +component_decl: + declspecs_nosc_ts setspecs components + { $$ = $3; + POP_DECLSPEC_STACK; } + | declspecs_nosc_ts setspecs + { + /* Support for unnamed structs or unions as members of + structs or unions (which is [a] useful and [b] supports + MS P-SDK). */ + if (pedantic) + pedwarn ("ISO C doesn't support unnamed structs/unions"); + + $$ = grokfield(NULL, current_declspecs, NULL_TREE); + POP_DECLSPEC_STACK; } + | declspecs_nosc_nots setspecs components_notype + { $$ = $3; + POP_DECLSPEC_STACK; } + | declspecs_nosc_nots + { if (pedantic) + pedwarn ("ISO C forbids member declarations with no members"); + shadow_tag_warned ($1, pedantic); + $$ = NULL_TREE; } + | error + { $$ = NULL_TREE; } + | extension component_decl + { $$ = $2; + RESTORE_EXT_FLAGS ($1); } + ; + +components: + component_declarator + | components ',' maybe_resetattrs component_declarator + { TREE_CHAIN ($4) = $1; $$ = $4; } + ; + +components_notype: + component_notype_declarator + | components_notype ',' maybe_resetattrs component_notype_declarator + { TREE_CHAIN ($4) = $1; $$ = $4; } + ; + +component_declarator: + declarator maybe_attribute + { $$ = grokfield ($1, current_declspecs, NULL_TREE); + decl_attributes (&$$, + chainon ($2, all_prefix_attributes), 0); } + | declarator ':' expr_no_commas maybe_attribute + { $$ = grokfield ($1, current_declspecs, $3); + decl_attributes (&$$, + chainon ($4, all_prefix_attributes), 0); } + | ':' expr_no_commas maybe_attribute + { $$ = grokfield (NULL_TREE, current_declspecs, $2); + decl_attributes (&$$, + chainon ($3, all_prefix_attributes), 0); } + ; + +component_notype_declarator: + notype_declarator maybe_attribute + { $$ = grokfield ($1, current_declspecs, NULL_TREE); + decl_attributes (&$$, + chainon ($2, all_prefix_attributes), 0); } + | notype_declarator ':' expr_no_commas maybe_attribute + { $$ = grokfield ($1, current_declspecs, $3); + decl_attributes (&$$, + chainon ($4, all_prefix_attributes), 0); } + | ':' expr_no_commas maybe_attribute + { $$ = grokfield (NULL_TREE, current_declspecs, $2); + decl_attributes (&$$, + chainon ($3, all_prefix_attributes), 0); } + ; + +/* We chain the enumerators in reverse order. + They are put in forward order in structsp_attr. */ + +enumlist: + enumerator + | enumlist ',' enumerator + { if ($1 == error_mark_node) + $$ = $1; + else + TREE_CHAIN ($3) = $1, $$ = $3; } + | error + { $$ = error_mark_node; } + ; + + +enumerator: + identifier + { $$ = build_enumerator ($1, NULL_TREE); } + | identifier '=' expr_no_commas + { $$ = build_enumerator ($1, $3); } + ; + +typename: + declspecs_nosc + { pending_xref_error (); + $$ = $1; } + absdcl + { $$ = build_tree_list ($2, $3); } + ; + +absdcl: /* an absolute declarator */ + /* empty */ + { $$ = NULL_TREE; } + | absdcl1 + ; + +absdcl_maybe_attribute: /* absdcl maybe_attribute, but not just attributes */ + /* empty */ + { $$ = build_tree_list (build_tree_list (current_declspecs, + NULL_TREE), + all_prefix_attributes); } + | absdcl1 + { $$ = build_tree_list (build_tree_list (current_declspecs, + $1), + all_prefix_attributes); } + | absdcl1_noea attributes + { $$ = build_tree_list (build_tree_list (current_declspecs, + $1), + chainon ($2, all_prefix_attributes)); } + ; + +absdcl1: /* a nonempty absolute declarator */ + absdcl1_ea + | absdcl1_noea + ; + +absdcl1_noea: + direct_absdcl1 + | '*' maybe_type_quals_attrs absdcl1_noea + { $$ = make_pointer_declarator ($2, $3); } + ; + +absdcl1_ea: + '*' maybe_type_quals_attrs + { $$ = make_pointer_declarator ($2, NULL_TREE); } + | '*' maybe_type_quals_attrs absdcl1_ea + { $$ = make_pointer_declarator ($2, $3); } + ; + +direct_absdcl1: + '(' maybe_attribute absdcl1 ')' + { $$ = $2 ? tree_cons ($2, $3, NULL_TREE) : $3; } + | direct_absdcl1 '(' parmlist + { $$ = build_nt (CALL_EXPR, $1, $3, NULL_TREE); } + | direct_absdcl1 array_declarator + { $$ = set_array_declarator_type ($2, $1, 1); } + | '(' parmlist + { $$ = build_nt (CALL_EXPR, NULL_TREE, $2, NULL_TREE); } + | array_declarator + { $$ = set_array_declarator_type ($1, NULL_TREE, 1); } + ; + +/* The [...] part of a declarator for an array type. */ + +array_declarator: + '[' maybe_type_quals_attrs expr_no_commas ']' + { $$ = build_array_declarator ($3, $2, 0, 0); } + | '[' maybe_type_quals_attrs ']' + { $$ = build_array_declarator (NULL_TREE, $2, 0, 0); } + | '[' maybe_type_quals_attrs '*' ']' + { $$ = build_array_declarator (NULL_TREE, $2, 0, 1); } + | '[' STATIC maybe_type_quals_attrs expr_no_commas ']' + { $$ = build_array_declarator ($4, $3, 1, 0); } + /* declspecs_nosc_nots is a synonym for type_quals_attrs. */ + | '[' declspecs_nosc_nots STATIC expr_no_commas ']' + { $$ = build_array_declarator ($4, $2, 1, 0); } + ; + +/* A nonempty series of declarations and statements (possibly followed by + some labels) that can form the body of a compound statement. + NOTE: we don't allow labels on declarations; this might seem like a + natural extension, but there would be a conflict between attributes + on the label and prefix attributes on the declaration. */ + +stmts_and_decls: + lineno_stmt_decl_or_labels_ending_stmt + | lineno_stmt_decl_or_labels_ending_decl + | lineno_stmt_decl_or_labels_ending_label + { + error ("label at end of compound statement"); + } + | lineno_stmt_decl_or_labels_ending_error + ; + +lineno_stmt_decl_or_labels_ending_stmt: + lineno_stmt + | lineno_stmt_decl_or_labels_ending_stmt lineno_stmt + | lineno_stmt_decl_or_labels_ending_decl lineno_stmt + | lineno_stmt_decl_or_labels_ending_label lineno_stmt + | lineno_stmt_decl_or_labels_ending_error lineno_stmt + ; + +lineno_stmt_decl_or_labels_ending_decl: + lineno_decl + | lineno_stmt_decl_or_labels_ending_stmt lineno_decl + { + if ((pedantic && !flag_isoc99) + || warn_declaration_after_statement) + pedwarn_c90 ("ISO C90 forbids mixed declarations and code"); + } + | lineno_stmt_decl_or_labels_ending_decl lineno_decl + | lineno_stmt_decl_or_labels_ending_error lineno_decl + ; + +lineno_stmt_decl_or_labels_ending_label: + lineno_label + | lineno_stmt_decl_or_labels_ending_stmt lineno_label + | lineno_stmt_decl_or_labels_ending_decl lineno_label + | lineno_stmt_decl_or_labels_ending_label lineno_label + | lineno_stmt_decl_or_labels_ending_error lineno_label + ; + +lineno_stmt_decl_or_labels_ending_error: + errstmt + | lineno_stmt_decl_or_labels errstmt + ; + +lineno_stmt_decl_or_labels: + lineno_stmt_decl_or_labels_ending_stmt + | lineno_stmt_decl_or_labels_ending_decl + | lineno_stmt_decl_or_labels_ending_label + | lineno_stmt_decl_or_labels_ending_error + ; + +errstmt: error ';' + ; + +pushlevel: /* empty */ + { pushlevel (0); + clear_last_expr (); + add_scope_stmt (/*begin_p=*/1, /*partial_p=*/0); + } + ; + +poplevel: /* empty */ + { + $$ = add_scope_stmt (/*begin_p=*/0, /*partial_p=*/0); + } + ; + +maybe_label_decls: + /* empty */ + | label_decls + { if (pedantic) + pedwarn ("ISO C forbids label declarations"); } + ; + +label_decls: + label_decl + | label_decls label_decl + ; + +label_decl: + LABEL identifiers_or_typenames ';' + { tree link; + for (link = $2; link; link = TREE_CHAIN (link)) + { + tree label = declare_label (TREE_VALUE (link)); + C_DECLARED_LABEL_FLAG (label) = 1; + add_decl_stmt (label); + } + } + ; + +/* This is the body of a function definition. + It causes syntax errors to ignore to the next openbrace. */ +compstmt_or_error: + compstmt + {} + | error compstmt + ; + +compstmt_start: '{' { compstmt_count++; + $$ = c_begin_compound_stmt (); } + ; + +compstmt_nostart: '}' + { $$ = convert (void_type_node, integer_zero_node); } + | pushlevel maybe_label_decls compstmt_contents_nonempty '}' poplevel + { $$ = poplevel (KEEP_MAYBE, 0, 0); + SCOPE_STMT_BLOCK (TREE_PURPOSE ($5)) + = SCOPE_STMT_BLOCK (TREE_VALUE ($5)) + = $$; } + ; + +compstmt_contents_nonempty: + stmts_and_decls + | error + ; + +compstmt_primary_start: + '(' '{' + { if (last_tree == NULL) + { + error ("braced-group within expression allowed only inside a function"); + YYERROR; + } + /* We must force a BLOCK for this level + so that, if it is not expanded later, + there is a way to turn off the entire subtree of blocks + that are contained in it. */ + keep_next_level (); + compstmt_count++; + $$ = add_stmt (build_stmt (COMPOUND_STMT, last_tree)); + last_expr_type = NULL_TREE; + } + ; + +compstmt: compstmt_start compstmt_nostart + { RECHAIN_STMTS ($1, COMPOUND_BODY ($1)); + last_expr_type = NULL_TREE; + $$ = $1; } + ; + +/* Value is number of statements counted as of the closeparen. */ +simple_if: + if_prefix c99_block_lineno_labeled_stmt + { c_finish_then (); } +/* Make sure c_expand_end_cond is run once + for each call to c_expand_start_cond. + Otherwise a crash is likely. */ + | if_prefix error + ; + +if_prefix: + /* We must build the IF_STMT node before parsing its + condition so that STMT_LINENO refers to the line + containing the "if", and not the line containing + the close-parenthesis. + + c_begin_if_stmt returns the IF_STMT node, which + we later pass to c_expand_start_cond to fill + in the condition and other tidbits. */ + IF + { $$ = c_begin_if_stmt (); } + '(' expr ')' + { c_expand_start_cond (c_common_truthvalue_conversion ($4), + compstmt_count,$2); + $$ = stmt_count; + if_stmt_locus = $-1; } + ; + +/* This is a subroutine of stmt. + It is used twice, once for valid DO statements + and once for catching errors in parsing the end test. */ +do_stmt_start: + DO + { stmt_count++; + compstmt_count++; + c_in_iteration_stmt++; + $$ + = add_stmt (build_stmt (DO_STMT, NULL_TREE, + NULL_TREE)); + /* In the event that a parse error prevents + parsing the complete do-statement, set the + condition now. Otherwise, we can get crashes at + RTL-generation time. */ + DO_COND ($$) = error_mark_node; } + c99_block_lineno_labeled_stmt WHILE + { $$ = $2; + RECHAIN_STMTS ($$, DO_BODY ($$)); + c_in_iteration_stmt--; } + ; + +/* The forced readahead in here is because we might be at the end of a + line, and the line and file won't be bumped until yylex absorbs the + first token on the next line. */ + +save_location: + { if (yychar == YYEMPTY) + yychar = YYLEX; + $$ = input_location; } + ; + +lineno_labeled_stmt: + lineno_stmt + | lineno_label lineno_labeled_stmt + ; + +/* Like lineno_labeled_stmt, but a block in C99. */ +c99_block_lineno_labeled_stmt: + lineno_labeled_stmt + { if (flag_isoc99) + RECHAIN_STMTS ($1, COMPOUND_BODY ($1)); } + ; + +lineno_stmt: + save_location stmt + { if ($2) + { + STMT_LINENO ($2) = $1.line; + /* ??? We currently have no way of recording + the filename for a statement. This probably + matters little in practice at the moment, + but I suspect that problems will occur when + doing inlining at the tree level. */ + } + } + ; + +lineno_label: + save_location label + { if ($2) + { + STMT_LINENO ($2) = $1.line; + } + } + ; + +select_or_iter_stmt: + simple_if ELSE + { c_expand_start_else (); + $1 = stmt_count; } + c99_block_lineno_labeled_stmt + { c_finish_else (); + c_expand_end_cond (); + if (extra_warnings && stmt_count == $1) + warning ("empty body in an else-statement"); } + | simple_if %prec IF + { c_expand_end_cond (); + /* This warning is here instead of in simple_if, because we + do not want a warning if an empty if is followed by an + else statement. Increment stmt_count so we don't + give a second error if this is a nested `if'. */ + if (extra_warnings && stmt_count++ == $1) + warning ("%Hempty body in an if-statement", + &if_stmt_locus); } +/* Make sure c_expand_end_cond is run once + for each call to c_expand_start_cond. + Otherwise a crash is likely. */ + | simple_if ELSE error + { c_expand_end_cond (); } + /* We must build the WHILE_STMT node before parsing its + condition so that STMT_LINENO refers to the line + containing the "while", and not the line containing + the close-parenthesis. + + c_begin_while_stmt returns the WHILE_STMT node, which + we later pass to c_finish_while_stmt_cond to fill + in the condition and other tidbits. */ + | WHILE + { stmt_count++; + $$ = c_begin_while_stmt (); } + '(' expr ')' + { c_in_iteration_stmt++; + $4 = c_common_truthvalue_conversion ($4); + c_finish_while_stmt_cond + (c_common_truthvalue_conversion ($4), $2); + $$ = add_stmt ($2); } + c99_block_lineno_labeled_stmt + { c_in_iteration_stmt--; + RECHAIN_STMTS ($6, WHILE_BODY ($6)); } + | do_stmt_start + '(' expr ')' ';' + { DO_COND ($1) = c_common_truthvalue_conversion ($3); } + | do_stmt_start error + { } + | FOR + { $$ = build_stmt (FOR_STMT, NULL_TREE, NULL_TREE, + NULL_TREE, NULL_TREE); + add_stmt ($$); } + '(' for_init_stmt + { stmt_count++; + RECHAIN_STMTS ($2, FOR_INIT_STMT ($2)); } + xexpr ';' + { if ($6) + FOR_COND ($2) + = c_common_truthvalue_conversion ($6); } + xexpr ')' + { c_in_iteration_stmt++; + FOR_EXPR ($2) = $9; } + c99_block_lineno_labeled_stmt + { RECHAIN_STMTS ($2, FOR_BODY ($2)); + c_in_iteration_stmt--;} + | SWITCH '(' expr ')' + { stmt_count++; + $$ = c_start_case ($3); + c_in_case_stmt++; } + c99_block_lineno_labeled_stmt + { c_finish_case (); + c_in_case_stmt--; } + ; + +for_init_stmt: + xexpr ';' + { add_stmt (build_stmt (EXPR_STMT, $1)); } + | decl + { check_for_loop_decls (); } + ; + +/* Parse a single real statement, not including any labels. */ +stmt: + compstmt + { stmt_count++; $$ = $1; } + { stmt_count++; + $$ = c_expand_expr_stmt ($1); } + | select_or_iter_stmt + { if (flag_isoc99) + RECHAIN_STMTS ($1, COMPOUND_BODY ($1)); + $$ = NULL_TREE; } + | BREAK ';' + { stmt_count++; + if (!(c_in_iteration_stmt || c_in_case_stmt)) + { + error ("break statement not within loop or switch"); + $$ = NULL_TREE; + } + else + $$ = add_stmt (build_break_stmt ()); } + | CONTINUE ';' + { stmt_count++; + if (!c_in_iteration_stmt) + { + error ("continue statement not within a loop"); + $$ = NULL_TREE; + } + else + $$ = add_stmt (build_continue_stmt ()); } + | RETURN ';' + { stmt_count++; + $$ = c_expand_return (NULL_TREE); } + | RETURN expr ';' + { stmt_count++; + $$ = c_expand_return ($2); } + | ASM_KEYWORD maybe_type_qual '(' expr ')' ';' + { stmt_count++; + $$ = simple_asm_stmt ($4); } + /* This is the case with just output operands. */ + | ASM_KEYWORD maybe_type_qual '(' expr ':' asm_operands ')' ';' + { stmt_count++; + $$ = build_asm_stmt ($2, $4, $6, NULL_TREE, NULL_TREE); } + /* This is the case with input operands as well. */ + | ASM_KEYWORD maybe_type_qual '(' expr ':' asm_operands ':' + asm_operands ')' ';' + { stmt_count++; + $$ = build_asm_stmt ($2, $4, $6, $8, NULL_TREE); } + /* This is the case with clobbered registers as well. */ + | ASM_KEYWORD maybe_type_qual '(' expr ':' asm_operands ':' + asm_operands ':' asm_clobbers ')' ';' + { stmt_count++; + $$ = build_asm_stmt ($2, $4, $6, $8, $10); } + | GOTO identifier ';' + { tree decl; + stmt_count++; + decl = lookup_label ($2); + if (decl != 0) + { + TREE_USED (decl) = 1; + $$ = add_stmt (build_stmt (GOTO_STMT, decl)); + } + else + $$ = NULL_TREE; + } + | GOTO '*' expr ';' + { if (pedantic) + pedwarn ("ISO C forbids `goto *expr;'"); + stmt_count++; + $3 = convert (ptr_type_node, $3); + $$ = add_stmt (build_stmt (GOTO_STMT, $3)); } + | ';' + { $$ = NULL_TREE; } + | exprstmt + ; + +exprstmt: expr ';' + | primary '(' exprlist ')' stmt + ; + + +/* Any kind of label, including jump labels and case labels. + ANSI C accepts labels only before statements, but we allow them + also at the end of a compound statement. */ + +label: CASE expr_no_commas ':' + { stmt_count++; + $$ = do_case ($2, NULL_TREE); } + | CASE expr_no_commas ELLIPSIS expr_no_commas ':' + { stmt_count++; + $$ = do_case ($2, $4); } + | DEFAULT ':' + { stmt_count++; + $$ = do_case (NULL_TREE, NULL_TREE); } + | identifier save_location ':' maybe_attribute + { tree label = define_label ($2, $1); + stmt_count++; + if (label) + { + decl_attributes (&label, $4, 0); + $$ = add_stmt (build_stmt (LABEL_STMT, label)); + } + else + $$ = NULL_TREE; + } + ; + +/* Either a type-qualifier or nothing. First thing in an `asm' statement. */ + +maybe_type_qual: + /* empty */ + { $$ = NULL_TREE; } + | TYPE_QUAL + { } + ; + +xexpr: + /* empty */ + { $$ = NULL_TREE; } + | expr + ; + +/* These are the operands other than the first string and colon + in asm ("addextend %2,%1": "=dm" (x), "0" (y), "g" (*x)) */ +asm_operands: /* empty */ + { $$ = NULL_TREE; } + | nonnull_asm_operands + ; + +nonnull_asm_operands: + asm_operand + | nonnull_asm_operands ',' asm_operand + { $$ = chainon ($1, $3); } + ; + +asm_operand: + STRING '(' expr ')' + { $$ = build_tree_list (build_tree_list (NULL_TREE, $1), $3); } + | '[' identifier ']' STRING '(' expr ')' + { $2 = build_string (IDENTIFIER_LENGTH ($2), + IDENTIFIER_POINTER ($2)); + $$ = build_tree_list (build_tree_list ($2, $4), $6); } + ; + +asm_clobbers: + STRING + { $$ = tree_cons (NULL_TREE, $1, NULL_TREE); } + | asm_clobbers ',' STRING + { $$ = tree_cons (NULL_TREE, $3, $1); } + ; + +/* This is what appears inside the parens in a function declarator. + Its value is a list of ..._TYPE nodes. Attributes must appear here + to avoid a conflict with their appearance after an open parenthesis + in an abstract declarator, as in + "void bar (int (__attribute__((__mode__(SI))) int foo));". */ +parmlist: + maybe_attribute + { pushlevel (0); + declare_parm_level (); } + parmlist_1 + { $$ = $3; + poplevel (0, 0, 0); } + ; + +parmlist_1: + parmlist_2 ')' + | parms ';' + { mark_forward_parm_decls (); } + maybe_attribute + { /* Dummy action so attributes are in known place + on parser stack. */ } + parmlist_1 + { $$ = $6; } + | error ')' + { $$ = tree_cons (NULL_TREE, NULL_TREE, NULL_TREE); } + ; + +/* This is what appears inside the parens in a function declarator. + Is value is represented in the format that grokdeclarator expects. */ +parmlist_2: /* empty */ + { $$ = get_parm_info (0); } + | ELLIPSIS + { $$ = get_parm_info (0); + /* Gcc used to allow this as an extension. However, it does + not work for all targets, and thus has been disabled. + Also, since func (...) and func () are indistinguishable, + it caused problems with the code in expand_builtin which + tries to verify that BUILT_IN_NEXT_ARG is being used + correctly. */ + error ("ISO C requires a named argument before `...'"); + parsing_iso_function_signature = true; + } + | parms + { $$ = get_parm_info (1); + parsing_iso_function_signature = true; + } + | parms ',' ELLIPSIS + { $$ = get_parm_info (0); + parsing_iso_function_signature = true; + } + ; + +parms: + firstparm + { push_parm_decl ($1); } + | parms ',' parm + { push_parm_decl ($3); } + ; + +/* A single parameter declaration or parameter type name, + as found in a parmlist. */ +parm: + declspecs_ts setspecs parm_declarator maybe_attribute + { $$ = build_tree_list (build_tree_list (current_declspecs, + $3), + chainon ($4, all_prefix_attributes)); + POP_DECLSPEC_STACK; } + | declspecs_ts setspecs notype_declarator maybe_attribute + { $$ = build_tree_list (build_tree_list (current_declspecs, + $3), + chainon ($4, all_prefix_attributes)); + POP_DECLSPEC_STACK; } + | declspecs_ts setspecs absdcl_maybe_attribute + { $$ = $3; + POP_DECLSPEC_STACK; } + | declspecs_nots setspecs notype_declarator maybe_attribute + { $$ = build_tree_list (build_tree_list (current_declspecs, + $3), + chainon ($4, all_prefix_attributes)); + POP_DECLSPEC_STACK; } + + | declspecs_nots setspecs absdcl_maybe_attribute + { $$ = $3; + POP_DECLSPEC_STACK; } + ; + +/* The first parm, which must suck attributes from off the top of the parser + stack. */ +firstparm: + declspecs_ts_nosa setspecs_fp parm_declarator maybe_attribute + { $$ = build_tree_list (build_tree_list (current_declspecs, + $3), + chainon ($4, all_prefix_attributes)); + POP_DECLSPEC_STACK; } + | declspecs_ts_nosa setspecs_fp notype_declarator maybe_attribute + { $$ = build_tree_list (build_tree_list (current_declspecs, + $3), + chainon ($4, all_prefix_attributes)); + POP_DECLSPEC_STACK; } + | declspecs_ts_nosa setspecs_fp absdcl_maybe_attribute + { $$ = $3; + POP_DECLSPEC_STACK; } + | declspecs_nots_nosa setspecs_fp notype_declarator maybe_attribute + { $$ = build_tree_list (build_tree_list (current_declspecs, + $3), + chainon ($4, all_prefix_attributes)); + POP_DECLSPEC_STACK; } + + | declspecs_nots_nosa setspecs_fp absdcl_maybe_attribute + { $$ = $3; + POP_DECLSPEC_STACK; } + ; + +setspecs_fp: + setspecs + { prefix_attributes = chainon (prefix_attributes, $-2); + all_prefix_attributes = prefix_attributes; } + ; + +/* This is used in a function definition + where either a parmlist or an identifier list is ok. + Its value is a list of ..._TYPE nodes or a list of identifiers. */ +parmlist_or_identifiers: + maybe_attribute + { pushlevel (0); + declare_parm_level (); } + parmlist_or_identifiers_1 + { $$ = $3; + poplevel (0, 0, 0); } + ; + +parmlist_or_identifiers_1: + parmlist_1 + | identifiers ')' + { tree t; + for (t = $1; t; t = TREE_CHAIN (t)) + if (TREE_VALUE (t) == NULL_TREE) + error ("`...' in old-style identifier list"); + $$ = tree_cons (NULL_TREE, NULL_TREE, $1); + + /* Make sure we have a parmlist after attributes. */ + if ($-1 != 0 + && (TREE_CODE ($$) != TREE_LIST + || TREE_PURPOSE ($$) == 0 + || TREE_CODE (TREE_PURPOSE ($$)) != PARM_DECL)) + YYERROR1; + } + ; + +/* A nonempty list of identifiers. */ +identifiers: + IDENTIFIER + { $$ = build_tree_list (NULL_TREE, $1); } + | identifiers ',' IDENTIFIER + { $$ = chainon ($1, build_tree_list (NULL_TREE, $3)); } + ; + +/* A nonempty list of identifiers, including typenames. */ +identifiers_or_typenames: + identifier + { $$ = build_tree_list (NULL_TREE, $1); } + | identifiers_or_typenames ',' identifier + { $$ = chainon ($1, build_tree_list (NULL_TREE, $3)); } + ; + +extension: + EXTENSION + { $$ = SAVE_EXT_FLAGS(); + pedantic = 0; + warn_pointer_arith = 0; + warn_traditional = 0; + flag_iso = 0; } + ; diff --git a/src/ptgen/gcc/c.y.foot b/src/ptgen/gcc/c.y.foot new file mode 100755 index 0000000..a9ca213 --- /dev/null +++ b/src/ptgen/gcc/c.y.foot @@ -0,0 +1,12 @@ +#include + +extern char yytext[]; +extern int column; +extern int line; + +void yyerror( char *s) +{ +fflush(stdout); +fprintf(stderr,"%s: %d.%d\n",s,line,column); +} + diff --git a/src/ptgen/gcc/c.y.head b/src/ptgen/gcc/c.y.head new file mode 100755 index 0000000..0ca6d1b --- /dev/null +++ b/src/ptgen/gcc/c.y.head @@ -0,0 +1,97 @@ +%{ +/* +#include "config.h" +#include "system.h" +#include "coretypes.h" +#include "tm.h" +#include "tree.h" +#include "input.h" +#include "cpplib.h" +#include "intl.h" +#include "timevar.h" +#include "c-pragma.h" +#include "c-tree.h" +#include "flags.h" +#include "varray.h" +#include "output.h" +#include "toplev.h" +#include "ggc.h" +*/ + + +%} + + +%start program + +%token IDENTIFIER +%token TYPENAME +%token SCSPEC /* Storage class other than static. */ +%token STATIC /* Static storage class. */ +%token TYPESPEC +%token TYPE_QUAL +%token CONSTANT +%token STRING +%token ELLIPSIS +%token SIZEOF ENUM STRUCT UNION IF ELSE WHILE DO FOR SWITCH CASE DEFAULT +%token BREAK CONTINUE RETURN GOTO ASM_KEYWORD TYPEOF ALIGNOF +%token ATTRIBUTE EXTENSION LABEL +%token REALPART IMAGPART VA_ARG CHOOSE_EXPR TYPES_COMPATIBLE_P +%token PTR_VALUE PTR_BASE PTR_EXTENT +%token FUNC_NAME + +%type IDENTIFIER +%type TYPENAME +%type SCSPEC /* Storage class other than static. */ +%type STATIC /* Static storage class. */ +%type TYPESPEC +%type TYPE_QUAL +%type CONSTANT +%type STRING +%type ELLIPSIS +%type SIZEOF ENUM STRUCT UNION IF ELSE WHILE DO FOR SWITCH CASE DEFAULT +%type BREAK CONTINUE RETURN GOTO ASM_KEYWORD TYPEOF ALIGNOF +%type ATTRIBUTE EXTENSION LABEL +%type REALPART IMAGPART VA_ARG CHOOSE_EXPR TYPES_COMPATIBLE_P +%type PTR_VALUE PTR_BASE PTR_EXTENT +%type FUNC_NAME + +%type ',' ')' ']' ';' '}' '{' '~' '!' + +%nonassoc IF +%nonassoc ELSE + +%right ASSIGN '=' +%right '?' ':' +%left OROR +%left ANDAND +%left '|' +%left '^' +%left '&' +%left EQCOMPARE +%left ARITHCOMPARE +%left LSHIFT RSHIFT +%left '+' '-' +%left '*' '/' '%' +%right UNARY PLUSPLUS MINUSMINUS +%left HYPERUNARY +%left POINTSAT '.' '(' '[' + + +%token INTERFACE IMPLEMENTATION END SELECTOR DEFS ENCODE +%token CLASSNAME PUBLIC PRIVATE PROTECTED PROTOCOL OBJECTNAME CLASS ALIAS +%token AT_THROW AT_TRY AT_CATCH AT_FINALLY AT_SYNCHRONIZED +%token OBJC_STRING + +/* +%type INTERFACE IMPLEMENTATION END SELECTOR DEFS ENCODE +%type CLASSNAME PUBLIC PRIVATE PROTECTED PROTOCOL OBJECTNAME CLASS ALIAS +%type AT_THROW AT_TRY AT_CATCH AT_FINALLY AT_SYNCHRONIZED +%type OBJC_STRING + +%type ENUM STRUCT UNION IF ELSE WHILE DO FOR SWITCH CASE DEFAULT +%type BREAK CONTINUE RETURN GOTO ASM_KEYWORD SIZEOF TYPEOF ALIGNOF +%type IDENTIFIER TYPENAME CONSTANT STRING +%type SCSPEC STATIC TYPESPEC TYPE_QUAL +*/ + diff --git a/src/ptgen/gcc/catomicNodes.h b/src/ptgen/gcc/catomicNodes.h new file mode 100644 index 0000000..16bda58 --- /dev/null +++ b/src/ptgen/gcc/catomicNodes.h @@ -0,0 +1,34 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"exprstmt", "expr", "decl", "datadef", "datadecl", +NULL diff --git a/src/ptgen/gcc/ccontextualNodes.h b/src/ptgen/gcc/ccontextualNodes.h new file mode 100644 index 0000000..7303c52 --- /dev/null +++ b/src/ptgen/gcc/ccontextualNodes.h @@ -0,0 +1,42 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"program", // "extdefs", "extdef", "extdef_1", + "fndef", + // "datadef", + "select_or_iter_stmt", /* need to check the node kind of its first child to decide the context. */ + "simple_if", + "do_stmt_start", + "WHILE", + "FOR", + "SWITCH", +NULL diff --git a/src/ptgen/gcc/cparentNodes.h b/src/ptgen/gcc/cparentNodes.h new file mode 100644 index 0000000..b1c25c4 --- /dev/null +++ b/src/ptgen/gcc/cparentNodes.h @@ -0,0 +1,37 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"stmt", "decl", "extdef", +/* However, these nodes will still not be outputted if they contain + less than mergeTokens (default 30) tokens. +*/ +NULL diff --git a/src/ptgen/gcc/crelevantNodes.h b/src/ptgen/gcc/crelevantNodes.h new file mode 100644 index 0000000..d5f99e3 --- /dev/null +++ b/src/ptgen/gcc/crelevantNodes.h @@ -0,0 +1,129 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"extdef", + "datadef", + "fndef", + "identifier", + "unop", + "expr", +"unary_expr", + "sizeof", + "alignof", + "typeof", + "cast_expr", +"expr_no_commas", + "primary", + "datadecl", + "decl", +"typespec_nonreserved_nonattr", + "maybeasm", + "initdcl", +"notype_initdcl", + "attribute", + "attrib", + "initelt", + "initval", +"designator", + "declarator", + "struct_head", + "union_head", + "enum_head", +"enumerator", + "typename", + "array_declarator", +"simple_if", + "do_stmt_start", + "select_or_iter_stmt", +"for_init_stmt", + "stmt", + "exprstmt", + "label", + "asm_operand", +"parm", + "firstparm", + "extension", +"OROR", +"RSHIFT", +"POINTSAT", +"DO", +"DEFAULT", +"FUNC_NAME", +"CHOOSE_EXPR", +"LABEL", +"VA_ARG", +"WHILE", +"LSHIFT", +"CONTINUE", +"CONSTANT", +"SIZEOF", +"ARITHCOMPARE", +"GOTO", +"CASE", +"BREAK", +"ASM_KEYWORD", +"EXTENSION", +"UNION", +"SWITCH", +"IDENTIFIER", +"ASSIGN", +"ENUM", +"STRING", +"FOR", +"PLUSPLUS", +"MINUSMINUS", +"ELSE", +"ELLIPSIS", +"ANDAND", +"EQCOMPARE", +"STRUCT", +"TYPENAME", +"TYPEOF", +"RETURN", +"TYPESPEC", + "':'", + "'.'", + "'~'", + "'%'", + "'*'", + "'+'", + "'-'", + "'/'", + "'?'", + "'&'", + "'!'", + "'|'", + "'^'", + "'='", + "'['", + "']'", +NULL diff --git a/src/ptgen/gcc/main.cc b/src/ptgen/gcc/main.cc new file mode 100755 index 0000000..7375d3f --- /dev/null +++ b/src/ptgen/gcc/main.cc @@ -0,0 +1,69 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include + +using namespace std; + +map name2id; +map id2name; + +int yyparse(); + +extern Tree *root; + +void id_init(); + +int last_line= 0; + + +int main() +{ + id_init(); + yyparse(); + if (!root) { + cerr << "failed to parse file" << endl; + return 1; + } + + root->printTok(); + //root->print(); + +// int c= root->countTerminals(); + +// cout << c << endl; + + return 0; +} + diff --git a/src/ptgen/gcc/mainc.py b/src/ptgen/gcc/mainc.py new file mode 100644 index 0000000..440542a --- /dev/null +++ b/src/ptgen/gcc/mainc.py @@ -0,0 +1,184 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Given a normal yacc .y file, simply split it into three files for our yacc.g to work: *.y (which contains and only contains all the rules), *.y.head, *.y.foot + +# Pi: To make the parse reentrant. http://www.gnu.org/software/bison/manual/html_mono/bison.html#Pure-Decl + +import sys +sys.path.append('..') + +import YaccParser,YaccLexer + +if len(sys.argv) != 2: + print >> sys.stderr, "usage %s grammar"%sys.argv[0] + sys.exit(1) + + +grammar= open(sys.argv[1],'r') +y= YaccParser.Parser(YaccLexer.Lexer(grammar)) +y.grammar() +grammar.close() + + +outg= open('pt_'+sys.argv[1],'w') + +print >> outg, """ +%pure-parser + +%{ +#include + +using namespace std; +%} + +%union{ +Tree *t; +} + +%{ +void yyerror(char*s); +int yylex(YYSTYPE *yylvalp); + +Tree *root; +%} + +""" + +for nt in y.NonTerminals: + print >> outg, '%type ', nt + + +header= open(sys.argv[1]+'.head','r') +outg.write( header.read() ) +header.close() + +print >> outg, """ + +%% + +""" + +name2id= {} +current= 0 +for nt in y.NonTerminals: + name2id[nt]=current + current+=1 +for t in y.Terminals: + name2id[t]=current + current+=1 + +#outh= open('tokid.h','w') +#for nt in y.NonTerminals: +# print >> outh, '#define ID_'+nt, name2id[nt] +#outh.close() + + +for rule in y.Rules: + print >> outg, rule[0], ':', + for production in rule[1]: + print >> outg, production[1], + """ + leave= False + for production in rule[1]: + if production[0]=="error": + leave= True + break + if leave: + print >> outg, ';\n' + continue + """ + print >> outg, """ + { + $$= new NonTerminal(""", name2id[rule[0]], ");" + nodelist= [ production[0] for production \ + in zip(range(1,len(rule[1])+1),rule[1]) \ + if production[1][0]=="node"] + + for i in range(len(nodelist)): + print >> outg, "\n $$->addChild($%d);"%nodelist[i] + print >> outg, "\n $%d->parent= $$;"%nodelist[i] + if i + 1 < len(nodelist): + print >> outg,\ + "\n $%d->nextSibbling= $%d;"%\ + (nodelist[i],nodelist[i+1]) + if rule[0]=="program": + print >> outg, "root= $$;" + print >> outg, """ + } + ; +""" + +print >> outg, """ + +%% + +""" + +footer= open(sys.argv[1]+'.foot','r') +print >> outg, footer.read() +footer.close() + + +outg.close() + + +head = open('head.cc','w') + +print >> head, """ +#include +#include + +using namespace std; + +extern map name2id; +extern map id2name; + +void id_init() +{ +""" +for nt in y.NonTerminals: + id= name2id[nt] + print >> head, 'name2id["%s"]= %d;'%(nt,id) + print >> head, 'id2name[%d]= "%s";'%(id,nt) +for t in y.Terminals: + id= name2id[t] + print >> head, 'name2id["%s"]= %d;'%(t,id) + print >> head, 'id2name[%d]= "%s";'%(id,t) +print >> head, '}' +print >> head + +head.close() + diff --git a/src/ptgen/java/Makefile b/src/ptgen/java/Makefile new file mode 100755 index 0000000..114a93a --- /dev/null +++ b/src/ptgen/java/Makefile @@ -0,0 +1,60 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +CXX= g++ -I../../include +OBJS= lex.yy.o pt_j.tab.o head.o +#TARGET=j_ptgen # this is just a test driver +TARGET=javaptgen.a + +all: $(TARGET) + +javaptgen.a: $(OBJS) + ar -qcs $@ $(OBJS) + +j_ptgen: $(OBJS) main.o + $(CXX) -o $@ $(OBJS) main.o + +#${TARGET}: ${OBJS} main.o +# $(CXX) -o ${TARGET} ${OBJS} main.o + +lex.yy.cc: j.l pt_j.tab.cc + flex -olex.yy.cc j.l + +pt_j.tab.cc: pt_j.y + bison -d pt_j.y -o pt_j.tab.cc + +head.cc pt_j.y: j.y j.y.head j.y.foot + ./mainj.py j.y + +.PHONY: clean +clean: + rm -f *.o lex.yy.cc pt_j.tab* pt_j.y head.cc $(TARGET) diff --git a/src/ptgen/java/j.l b/src/ptgen/java/j.l new file mode 100755 index 0000000..5ea8992 --- /dev/null +++ b/src/ptgen/java/j.l @@ -0,0 +1,225 @@ +D [0-9] +L [a-zA-Z_] +H [a-fA-F0-9] +E [Ee][+-]?{D}+ +FS (f|F|l|L) +IS (u|U|l|L)* + +%{ +#include +#include +#include "pt_j.tab.hh" +#include +#include +using namespace std; + +extern map name2id; +void count(); +void comment(); +void cpp_comment(); +void macro(); + +// TODO: potential troublesome global variables +int column = 0; +int line = 1; + +#define YY_DECL int yylex(YYSTYPE *yylvalp) + +%} + +%x COMMENT + +%% + +"/*" { count(); comment();} +"//" { count(); cpp_comment(); } +^"#" {count(); macro();} +"public" {count();yylvalp->t=new Terminal(name2id["PUBLIC_TK"],yytext,line);return (PUBLIC_TK);} +"private" {count();yylvalp->t=new Terminal(name2id["PRIVATE_TK"],yytext,line);return (PRIVATE_TK);} +"protected" {count();yylvalp->t=new Terminal(name2id["PROTECTED_TK"],yytext,line);return (PROTECTED_TK);} +"static" {count();yylvalp->t=new Terminal(name2id["STATIC_TK"],yytext,line);return (STATIC_TK);} +"final" {count();yylvalp->t=new Terminal(name2id["FINAL_TK"],yytext,line);return (FINAL_TK);} +"synchronized" {count();yylvalp->t=new Terminal(name2id["SYNCHRONIZED_TK"],yytext,line);return (SYNCHRONIZED_TK);} +"volatile" {count();yylvalp->t=new Terminal(name2id["VOLATILE_TK"],yytext,line);return (VOLATILE_TK);} +"transient" {count();yylvalp->t=new Terminal(name2id["TRANSIENT_TK"],yytext,line);return (TRANSIENT_TK);} +"native" {count();yylvalp->t=new Terminal(name2id["NATIVE_TK"],yytext,line);return (NATIVE_TK);} +"abstract" {count();yylvalp->t=new Terminal(name2id["ABSTRACT_TK"],yytext,line);return (ABSTRACT_TK);} +"strictfp" {count();yylvalp->t=new Terminal(name2id["STRICT_TK"],yytext,line);return (STRICT_TK);} +"default" {count();yylvalp->t=new Terminal(name2id["DEFAULT_TK"],yytext,line);return (DEFAULT_TK);} +"if" {count();yylvalp->t=new Terminal(name2id["IF_TK"],yytext,line);return (IF_TK);} +"throw" {count();yylvalp->t=new Terminal(name2id["THROW_TK"],yytext,line);return (THROW_TK);} +"boolean" {count();yylvalp->t=new Terminal(name2id["BOOLEAN_TK"],yytext,line);return (BOOLEAN_TK);} +"do" {count();yylvalp->t=new Terminal(name2id["DO_TK"],yytext,line);return (DO_TK);} +"implements" {count();yylvalp->t=new Terminal(name2id["IMPLEMENTS_TK"],yytext,line);return (IMPLEMENTS_TK);} +"throws" {count();yylvalp->t=new Terminal(name2id["THROWS_TK"],yytext,line);return (THROWS_TK);} +"break" {count();yylvalp->t=new Terminal(name2id["BREAK_TK"],yytext,line);return (BREAK_TK);} +"import" {count();yylvalp->t=new Terminal(name2id["IMPORT_TK"],yytext,line);return (IMPORT_TK);} +"else" {count();yylvalp->t=new Terminal(name2id["ELSE_TK"],yytext,line);return (ELSE_TK);} +"instanceof" {count();yylvalp->t=new Terminal(name2id["INSTANCEOF_TK"],yytext,line);return (INSTANCEOF_TK);} +"return" {count();yylvalp->t=new Terminal(name2id["RETURN_TK"],yytext,line);return (RETURN_TK);} +"void" {count();yylvalp->t=new Terminal(name2id["VOID_TK"],yytext,line);return (VOID_TK);} +"catch" {count();yylvalp->t=new Terminal(name2id["CATCH_TK"],yytext,line);return (CATCH_TK);} +"interface" {count();yylvalp->t=new Terminal(name2id["INTERFACE_TK"],yytext,line);return (INTERFACE_TK);} +"case" {count();yylvalp->t=new Terminal(name2id["CASE_TK"],yytext,line);return (CASE_TK);} +"extends" {count();yylvalp->t=new Terminal(name2id["EXTENDS_TK"],yytext,line);return (EXTENDS_TK);} +"finally" {count();yylvalp->t=new Terminal(name2id["FINALLY_TK"],yytext,line);return (FINALLY_TK);} +"super" {count();yylvalp->t=new Terminal(name2id["SUPER_TK"],yytext,line);return (SUPER_TK);} +"while" {count();yylvalp->t=new Terminal(name2id["WHILE_TK"],yytext,line);return (WHILE_TK);} +"class" {count();yylvalp->t=new Terminal(name2id["CLASS_TK"],yytext,line);return (CLASS_TK);} +"switch" {count();yylvalp->t=new Terminal(name2id["SWITCH_TK"],yytext,line);return (SWITCH_TK);} +"const" {count();yylvalp->t=new Terminal(name2id["CONST_TK"],yytext,line);return (CONST_TK);} +"try" {count();yylvalp->t=new Terminal(name2id["TRY_TK"],yytext,line);return (TRY_TK);} +"for" {count();yylvalp->t=new Terminal(name2id["FOR_TK"],yytext,line);return (FOR_TK);} +"new" {count();yylvalp->t=new Terminal(name2id["NEW_TK"],yytext,line);return (NEW_TK);} +"continue" {count();yylvalp->t=new Terminal(name2id["CONTINUE_TK"],yytext,line);return (CONTINUE_TK);} +"goto" {count();yylvalp->t=new Terminal(name2id["GOTO_TK"],yytext,line);return (GOTO_TK);} +"package" {count();yylvalp->t=new Terminal(name2id["PACKAGE_TK"],yytext,line);return (PACKAGE_TK);} +"this" {count();yylvalp->t=new Terminal(name2id["THIS_TK"],yytext,line);return (THIS_TK);} +"assert" {count();yylvalp->t=new Terminal(name2id["ASSERT_TK"],yytext,line);return (ASSERT_TK);} +"byte" {count();yylvalp->t=new Terminal(name2id["BYTE_TK"],yytext,line);return (BYTE_TK);} +"short" {count();yylvalp->t=new Terminal(name2id["SHORT_TK"],yytext,line);return (SHORT_TK);} +"int" {count();yylvalp->t=new Terminal(name2id["INT_TK"],yytext,line);return (INT_TK);} +"long" {count();yylvalp->t=new Terminal(name2id["LONG_TK"],yytext,line);return (LONG_TK);} +"char" {count();yylvalp->t=new Terminal(name2id["CHAR_TK"],yytext,line);return (CHAR_TK);} +"integral" {count();yylvalp->t=new Terminal(name2id["INTEGRAL_TK"],yytext,line);return (INTEGRAL_TK);} +"float" {count();yylvalp->t=new Terminal(name2id["FLOAT_TK"],yytext,line);return (FLOAT_TK);} +"double" {count();yylvalp->t=new Terminal(name2id["DOUBLE_TK"],yytext,line);return (DOUBLE_TK);} +"true" {count();yylvalp->t=new Terminal(name2id["BOOL_LIT_TK"],yytext,line);return (BOOL_LIT_TK);} +"false" {count();yylvalp->t=new Terminal(name2id["BOOL_LIT_TK"],yytext,line);return (BOOL_LIT_TK);} +"null" {count();yylvalp->t=new Terminal(name2id["NULL_TK"],yytext,line);return (NULL_TK);} +{L}({L}|{D})* {count();yylvalp->t=new Terminal(name2id["ID_TK"],yytext,line);return(ID_TK); } +(0[xX]){H}+([Ll]?) {count();yylvalp->t=new Terminal(name2id["INT_LIT_TK"],yytext,line);return(INT_LIT_TK); } +{D}+([Ll]?) {count();yylvalp->t=new Terminal(name2id["INT_LIT_TK"],yytext,line);return(INT_LIT_TK); } +{D}+[FfDd] {count();yylvalp->t=new Terminal(name2id["FP_LIT_TK"],yytext,line);return(FP_LIT_TK); } +{D}+"."{D}*("E"({D})+)?[fFdD]? {count();yylvalp->t=new Terminal(name2id["FP_LIT_TK"],yytext,line);return(FP_LIT_TK); } +{D}*"."{D}+("E"({D})+)?[fFdD]? {count();yylvalp->t=new Terminal(name2id["FP_LIT_TK"],yytext,line);return(FP_LIT_TK); } +"\"".*"\"" {count();yylvalp->t=new Terminal(name2id["STRING_LIT_TK"],yytext,line);return(STRING_LIT_TK); } +"'"[^ \r\n\t]*"'" {count();yylvalp->t=new Terminal(name2id["CHAR_LIT_TK"],yytext,line);return(CHAR_LIT_TK); } +{L}?\"(\\.|[^\\"])*\"k {count();yylvalp->t=new Terminal(name2id["STRING_LIT_TK"],yytext,line);return(STRING_LIT_TK); } +">>=" {count();yylvalp->t=new Terminal(name2id["SRS_ASSIGN_TK"],yytext,line);return(SRS_ASSIGN_TK); } +"<<=" {count();yylvalp->t=new Terminal(name2id["LS_ASSIGN_TK"],yytext,line);return(LS_ASSIGN_TK); } +">>>=" {count();yylvalp->t=new Terminal(name2id["ZRS_ASSIGN_TK"],yytext,line);return(ZRS_ASSIGN_TK); } +"+=" {count();yylvalp->t=new Terminal(name2id["PLUS_ASSIGN_TK"],yytext,line);return(PLUS_ASSIGN_TK); } +"-=" {count();yylvalp->t=new Terminal(name2id["MINUS_ASSIGN_TK"],yytext,line);return(MINUS_ASSIGN_TK); } +"*=" {count();yylvalp->t=new Terminal(name2id["MULT_ASSIGN_TK"],yytext,line);return(MULT_ASSIGN_TK); } +"/=" {count();yylvalp->t=new Terminal(name2id["DIV_ASSIGN_TK"],yytext,line);return(DIV_ASSIGN_TK); } +"%=" {count();yylvalp->t=new Terminal(name2id["REM_ASSIGN_TK"],yytext,line);return(REM_ASSIGN_TK); } +"&=" {count();yylvalp->t=new Terminal(name2id["AND_ASSIGN_TK"],yytext,line);return(AND_ASSIGN_TK); } +"^=" {count();yylvalp->t=new Terminal(name2id["XOR_ASSIGN_TK"],yytext,line);return(XOR_ASSIGN_TK); } +"|=" {count();yylvalp->t=new Terminal(name2id["OR_ASSIGN_TK"],yytext,line);return(OR_ASSIGN_TK); } +">>" {count();yylvalp->t=new Terminal(name2id["SRS_TK"],yytext,line);return(SRS_TK); } +"<<" {count();yylvalp->t=new Terminal(name2id["LS_TK"],yytext,line);return(LS_TK); } +">>>" {count();yylvalp->t=new Terminal(name2id["ZRS_TK"],yytext,line);return(ZRS_TK); } +"++" {count();yylvalp->t=new Terminal(name2id["INCR_TK"],yytext,line);return(INCR_TK); } +"--" {count();yylvalp->t=new Terminal(name2id["DECR_TK"],yytext,line);return(DECR_TK); } +"&&" {count();yylvalp->t=new Terminal(name2id["BOOL_AND_TK"],yytext,line);return(BOOL_AND_TK); } +"||" {count();yylvalp->t=new Terminal(name2id["BOOL_OR_TK"],yytext,line);return(BOOL_OR_TK); } +"<=" {count();yylvalp->t=new Terminal(name2id["LTE_TK"],yytext,line);return(LTE_TK); } +">=" {count();yylvalp->t=new Terminal(name2id["GTE_TK"],yytext,line);return(GTE_TK); } +"==" {count();yylvalp->t=new Terminal(name2id["EQ_TK"],yytext,line);return(EQ_TK); } +"!=" {count();yylvalp->t=new Terminal(name2id["NEQ_TK"],yytext,line);return(NEQ_TK); } +";" {count();yylvalp->t=new Terminal(name2id["SC_TK"],yytext,line);return(SC_TK); } +"{" {count();yylvalp->t=new Terminal(name2id["OCB_TK"],yytext,line);return(OCB_TK); } +"}" {count();yylvalp->t=new Terminal(name2id["CCB_TK"],yytext,line);return(CCB_TK); } +"," {count();yylvalp->t=new Terminal(name2id["C_TK"],yytext,line);return(C_TK); } +"=" {count();yylvalp->t=new Terminal(name2id["ASSIGN_TK"],yytext,line);return(ASSIGN_TK); } +"(" {count();yylvalp->t=new Terminal(name2id["OP_TK"],yytext,line);return(OP_TK); } +")" {count();yylvalp->t=new Terminal(name2id["CP_TK"],yytext,line);return(CP_TK); } +"[" {count();yylvalp->t=new Terminal(name2id["OSB_TK"],yytext,line);return(OSB_TK); } +"]" {count();yylvalp->t=new Terminal(name2id["CSB_TK"],yytext,line);return(CSB_TK); } +"\." {count();yylvalp->t=new Terminal(name2id["DOT_TK"],yytext,line);return(DOT_TK); } +"&" {count();yylvalp->t=new Terminal(name2id["AND_TK"],yytext,line);return(AND_TK); } +"!" {count();yylvalp->t=new Terminal(name2id["NOT_TK"],yytext,line);return(NOT_TK); } +"~" {count();yylvalp->t=new Terminal(name2id["NEG_TK"],yytext,line);return(NEG_TK); } +"-" {count();yylvalp->t=new Terminal(name2id["MINUS_TK"],yytext,line);return(MINUS_TK); } +"+" {count();yylvalp->t=new Terminal(name2id["PLUS_TK"],yytext,line);return(PLUS_TK); } +"*" {count();yylvalp->t=new Terminal(name2id["MULT_TK"],yytext,line);return(MULT_TK); } +"/" {count();yylvalp->t=new Terminal(name2id["DIV_TK"],yytext,line);return(DIV_TK); } +"%" {count();yylvalp->t=new Terminal(name2id["REM_TK"],yytext,line);return(REM_TK); } +"<" {count();yylvalp->t=new Terminal(name2id["LT_TK"],yytext,line);return(LT_TK); } +">" {count();yylvalp->t=new Terminal(name2id["GT_TK"],yytext,line);return(GT_TK); } +"^" {count();yylvalp->t=new Terminal(name2id["XOR_TK"],yytext,line);return(XOR_TK); } +"|" {count();yylvalp->t=new Terminal(name2id["OR_TK"],yytext,line);return(OR_TK); } +"?" {count();yylvalp->t=new Terminal(name2id["REL_QM_TK"],yytext,line);return(REL_QM_TK); } +":" {count();yylvalp->t=new Terminal(name2id["REL_CL_TK"],yytext,line);return(REL_CL_TK); } +([ \t\v\n\r])+ {count();} +. {count();} + +%% + +int yywrap() +{ + return(1); +} + + + +void comment() +{ + int c; + + for (;;) { + while ( (c = yyinput()) != '*' && c != EOF ) { + if (c=='\n') { + line++;column=0; + } else { + column++; + } + } + + if ( c == '*' ) { + while ( (c = yyinput()) == '*' ) + column++; + column++; + if (c =='\n') {line++;column=0;} + if ( c == '/' ) + break; + } + + if ( c == EOF ) { + break; + } + } +} + +void cpp_comment() +{ + int c; + while ((c = yyinput()) != '\n' && c != 0 && c!=EOF) + column++; + line++; + column= 0; +} + +void macro() +{ + int c,last=0; + again: + while ((c = yyinput()) != '\n' && c != 0 && c!=EOF) { + if (c == '\n' && last == '\\') { + line++; + goto again; + } + last= c; + } + + line++; + column= 0; +} + +void count() +{ + int i; + + for (i = 0; yytext[i] != '\0'; i++) + if (yytext[i] == '\n') { + column = 0; + line++; + } else if (yytext[i] == '\t') + column += 4; + else + column++; + + //ECHO; +} + diff --git a/src/ptgen/java/j.y b/src/ptgen/java/j.y new file mode 100755 index 0000000..ba19a0d --- /dev/null +++ b/src/ptgen/java/j.y @@ -0,0 +1,2076 @@ +goal: compilation_unit + {} +; + +/* 19.3 Productions from 3: Lexical structure */ +literal: + INT_LIT_TK +| FP_LIT_TK +| BOOL_LIT_TK +| CHAR_LIT_TK +| STRING_LIT_TK +| NULL_TK +; + +/* 19.4 Productions from 4: Types, Values and Variables */ +type: + primitive_type +| reference_type +; + +primitive_type: + integral +| float +| BOOLEAN_TK +; + +integral: + BYTE_TK +| SHORT_TK +| INT_TK +| LONG_TK +| CHAR_TK +| INTEGRAL_TK +; + +float: + FLOAT_TK +| DOUBLE_TK +| FP_TK +; + +reference_type: + class_or_interface_type +| array_type +; + +class_or_interface_type: + name +; + +class_type: + class_or_interface_type /* Default rule */ +; + +interface_type: + class_or_interface_type +; + +array_type: + primitive_type dims + { + int osb = pop_current_osb (ctxp); + tree t = build_java_array_type (($1), -1); + while (--osb) + t = build_unresolved_array_type (t); + $$ = t; + } +| name dims + { + int osb = pop_current_osb (ctxp); + tree t = $1; + while (osb--) + t = build_unresolved_array_type (t); + $$ = t; + } +; + +/* 19.5 Productions from 6: Names */ +name: + simple_name /* Default rule */ +| qualified_name /* Default rule */ +; + +simple_name: + identifier /* Default rule */ +; + +qualified_name: + name DOT_TK identifier + { $$ = make_qualified_name ($1, $3, $2.location); } +; + +identifier: + ID_TK +; + +/* 19.6: Production from 7: Packages */ +compilation_unit: + {$$ = NULL;} +| package_declaration +| import_declarations +| type_declarations +| package_declaration import_declarations +| package_declaration type_declarations +| import_declarations type_declarations +| package_declaration import_declarations type_declarations +; + +import_declarations: + import_declaration + { + $$ = NULL; + } +| import_declarations import_declaration + { + $$ = NULL; + } +; + +type_declarations: + type_declaration +| type_declarations type_declaration +; + +package_declaration: + PACKAGE_TK name SC_TK + { + ctxp->package = EXPR_WFL_NODE ($2); + register_package (ctxp->package); + } +| PACKAGE_TK error + {yyerror ("Missing name"); RECOVER;} +| PACKAGE_TK name error + {yyerror ("';' expected"); RECOVER;} +; + +import_declaration: + single_type_import_declaration +| type_import_on_demand_declaration +; + +single_type_import_declaration: + IMPORT_TK name SC_TK + { + tree name = EXPR_WFL_NODE ($2), last_name; + int i = IDENTIFIER_LENGTH (name)-1; + const char *last = &IDENTIFIER_POINTER (name)[i]; + while (last != IDENTIFIER_POINTER (name)) + { + if (last [0] == '.') + break; + last--; + } + last_name = get_identifier (++last); + if (IS_A_SINGLE_IMPORT_CLASSFILE_NAME_P (last_name)) + { + tree err = find_name_in_single_imports (last_name); + if (err && err != name) + parse_error_context + ($2, "Ambiguous class: `%s' and `%s'", + IDENTIFIER_POINTER (name), + IDENTIFIER_POINTER (err)); + else + REGISTER_IMPORT ($2, last_name); + } + else + REGISTER_IMPORT ($2, last_name); + } +| IMPORT_TK error + {yyerror ("Missing name"); RECOVER;} +| IMPORT_TK name error + {yyerror ("';' expected"); RECOVER;} +; + +type_import_on_demand_declaration: + IMPORT_TK name DOT_TK MULT_TK SC_TK + { + tree name = EXPR_WFL_NODE ($2); + tree it; + /* Search for duplicates. */ + for (it = ctxp->import_demand_list; it; it = TREE_CHAIN (it)) + if (EXPR_WFL_NODE (TREE_PURPOSE (it)) == name) + break; + /* Don't import the same thing more than once, just ignore + duplicates (7.5.2) */ + if (! it) + { + read_import_dir ($2); + ctxp->import_demand_list = + chainon (ctxp->import_demand_list, + build_tree_list ($2, NULL_TREE)); + } + } +| IMPORT_TK name DOT_TK error + {yyerror ("'*' expected"); RECOVER;} +| IMPORT_TK name DOT_TK MULT_TK error + {yyerror ("';' expected"); RECOVER;} +; + +type_declaration: + class_declaration + { end_class_declaration (0); } +| interface_declaration + { end_class_declaration (0); } +| empty_statement +| error + { + YYERROR_NOW; + yyerror ("Class or interface declaration expected"); + } +; + +/* 19.7 Shortened from the original: + modifiers: modifier | modifiers modifier + modifier: any of public... */ +modifiers: + modifier + { + $$ = (1 << $1); + } +| modifiers modifier + { + int acc = (1 << $2); + if ($$ & acc) + parse_error_context + (ctxp->modifier_ctx [$2], "Modifier `%s' declared twice", + java_accstring_lookup (acc)); + else + { + $$ |= acc; + } + } +; +modifier: + PUBLIC_TK +| PRIVATE_TK +| PROTECTED_TK +| STATIC_TK +| FINAL_TK +| SYNCHRONIZED_TK +| VOLATILE_TK +| TRANSIENT_TK +| NATIVE_TK +| PAD_TK +| ABSTRACT_TK +| STRICT_TK +| CONST_TK +| MODIFIER_TK +; + +/* 19.8.1 Production from $8.1: Class Declaration */ +class_declaration: + modifiers CLASS_TK identifier super interfaces + { create_class ($1, $3, $4, $5); } + class_body + {;} +| CLASS_TK identifier super interfaces + { create_class (0, $2, $3, $4); } + class_body + {;} +| modifiers CLASS_TK error + { yyerror ("Missing class name"); RECOVER; } +| CLASS_TK error + { yyerror ("Missing class name"); RECOVER; } +| CLASS_TK identifier error + { + if (!ctxp->class_err) yyerror (" expected"); + DRECOVER(class1); + } +| modifiers CLASS_TK identifier error + { if (!ctxp->class_err) yyerror (" expected"); RECOVER; } +; + +super: + { $$ = NULL; } +| EXTENDS_TK class_type + { $$ = $2; } +| EXTENDS_TK class_type error + {yyerror (" expected"); ctxp->class_err=1;} +| EXTENDS_TK error + {yyerror ("Missing super class name"); ctxp->class_err=1;} +; + +interfaces: + { $$ = NULL_TREE; } +| IMPLEMENTS_TK interface_type_list + { $$ = $2; } +| IMPLEMENTS_TK error + { + ctxp->class_err=1; + yyerror ("Missing interface name"); + } +; + +interface_type_list: + interface_type + { + ctxp->interface_number = 1; + $$ = build_tree_list ($1, NULL_TREE); + } +| interface_type_list C_TK interface_type + { + ctxp->interface_number++; + $$ = chainon ($1, build_tree_list ($3, NULL_TREE)); + } +| interface_type_list C_TK error + {yyerror ("Missing interface name"); RECOVER;} +; + +class_body: + OCB_TK CCB_TK + { + /* Store the location of the when doing xrefs */ + if (flag_emit_xref) + DECL_END_SOURCE_LINE (GET_CPC ()) = + EXPR_WFL_ADD_COL ($2.location, 1); + $$ = GET_CPC (); + } +| OCB_TK class_body_declarations CCB_TK + { + /* Store the location of the when doing xrefs */ + if (flag_emit_xref) + DECL_END_SOURCE_LINE (GET_CPC ()) = + EXPR_WFL_ADD_COL ($3.location, 1); + $$ = GET_CPC (); + } +; + +class_body_declarations: + class_body_declaration +| class_body_declarations class_body_declaration +; + +class_body_declaration: + class_member_declaration +| static_initializer +| constructor_declaration +| block /* Added, JDK1.1, instance initializer */ + { + if ($1 != empty_stmt_node) + { + TREE_CHAIN ($1) = CPC_INSTANCE_INITIALIZER_STMT (ctxp); + SET_CPC_INSTANCE_INITIALIZER_STMT (ctxp, $1); + } + } +; + +class_member_declaration: + field_declaration +| method_declaration +| class_declaration /* Added, JDK1.1 inner classes */ + { end_class_declaration (1); } +| interface_declaration /* Added, JDK1.1 inner interfaces */ + { end_class_declaration (1); } +| empty_statement +; + +/* 19.8.2 Productions from 8.3: Field Declarations */ +field_declaration: + type variable_declarators SC_TK + { register_fields (0, $1, $2); } +| modifiers type variable_declarators SC_TK + { + check_modifiers + ("Illegal modifier `%s' for field declaration", + $1, FIELD_MODIFIERS); + check_modifiers_consistency ($1); + register_fields ($1, $2, $3); + } +; + +variable_declarators: + /* Should we use build_decl_list () instead ? FIXME */ + variable_declarator /* Default rule */ +| variable_declarators C_TK variable_declarator + { $$ = chainon ($1, $3); } +| variable_declarators C_TK error + {yyerror ("Missing term"); RECOVER;} +; + +variable_declarator: + variable_declarator_id + { $$ = build_tree_list ($1, NULL_TREE); } +| variable_declarator_id ASSIGN_TK variable_initializer + { + if (java_error_count) + $3 = NULL_TREE; + $$ = build_tree_list + ($1, build_assignment ($2.token, $2.location, $1, $3)); + } +| variable_declarator_id ASSIGN_TK error + { + yyerror ("Missing variable initializer"); + $$ = build_tree_list ($1, NULL_TREE); + RECOVER; + } +| variable_declarator_id ASSIGN_TK variable_initializer error + { + yyerror ("';' expected"); + $$ = build_tree_list ($1, NULL_TREE); + RECOVER; + } +; + +variable_declarator_id: + identifier +| variable_declarator_id OSB_TK CSB_TK + { $$ = build_unresolved_array_type ($1); } +| identifier error + {yyerror ("Invalid declaration"); DRECOVER(vdi);} +| variable_declarator_id OSB_TK error + { + yyerror ("']' expected"); + DRECOVER(vdi); + } +| variable_declarator_id CSB_TK error + {yyerror ("Unbalanced ']'"); DRECOVER(vdi);} +; + +variable_initializer: + expression +| array_initializer +; + +/* 19.8.3 Productions from 8.4: Method Declarations */ +method_declaration: + method_header + { + current_function_decl = $1; + if (current_function_decl + && TREE_CODE (current_function_decl) == FUNCTION_DECL) + source_start_java_method (current_function_decl); + else + current_function_decl = NULL_TREE; + } + method_body + { finish_method_declaration ($3); } +| method_header error + {YYNOT_TWICE yyerror (" expected"); RECOVER;} +; + +method_header: + type method_declarator throws + { $$ = method_header (0, $1, $2, $3); } +| VOID_TK method_declarator throws + { $$ = method_header (0, void_type_node, $2, $3); } +| modifiers type method_declarator throws + { $$ = method_header ($1, $2, $3, $4); } +| modifiers VOID_TK method_declarator throws + { $$ = method_header ($1, void_type_node, $3, $4); } +| type error + { + yyerror ("Invalid method declaration, method name required"); + RECOVER; + } +| modifiers type error + { + yyerror ("Identifier expected"); + RECOVER; + } +| VOID_TK error + { + yyerror ("Identifier expected"); + RECOVER; + } +| modifiers VOID_TK error + { + yyerror ("Identifier expected"); + RECOVER; + } +| modifiers error + { + yyerror ("Invalid method declaration, return type required"); + RECOVER; + } +; + +method_declarator: + identifier OP_TK CP_TK + { + ctxp->formal_parameter_number = 0; + $$ = method_declarator ($1, NULL_TREE); + } +| identifier OP_TK formal_parameter_list CP_TK + { $$ = method_declarator ($1, $3); } +| method_declarator OSB_TK CSB_TK + { + EXPR_WFL_LINECOL (wfl_operator) = $2.location; + TREE_PURPOSE ($1) = + build_unresolved_array_type (TREE_PURPOSE ($1)); + parse_warning_context + (wfl_operator, + "Discouraged form of returned type specification"); + } +| identifier OP_TK error + {yyerror ("')' expected"); DRECOVER(method_declarator);} +| method_declarator OSB_TK error + {yyerror ("']' expected"); RECOVER;} +; + +formal_parameter_list: + formal_parameter + { + ctxp->formal_parameter_number = 1; + } +| formal_parameter_list C_TK formal_parameter + { + ctxp->formal_parameter_number += 1; + $$ = chainon ($1, $3); + } +| formal_parameter_list C_TK error + { yyerror ("Missing formal parameter term"); RECOVER; } +; + +formal_parameter: + type variable_declarator_id + { + $$ = build_tree_list ($2, $1); + } +| final type variable_declarator_id /* Added, JDK1.1 final parms */ + { + $$ = build_tree_list ($3, $2); + ARG_FINAL_P ($$) = 1; + } +| type error + { + yyerror ("Missing identifier"); RECOVER; + $$ = NULL_TREE; + } +| final type error + { + yyerror ("Missing identifier"); RECOVER; + $$ = NULL_TREE; + } +; + +final: + modifiers + { + check_modifiers ("Illegal modifier `%s'. Only `final' was expected here", + $1, ACC_FINAL); + if ($1 != ACC_FINAL) + MODIFIER_WFL (FINAL_TK) = build_wfl_node (NULL_TREE); + } +; + +throws: + { $$ = NULL_TREE; } +| THROWS_TK class_type_list + { $$ = $2; } +| THROWS_TK error + {yyerror ("Missing class type term"); RECOVER;} +; + +class_type_list: + class_type + { $$ = build_tree_list ($1, $1); } +| class_type_list C_TK class_type + { $$ = tree_cons ($3, $3, $1); } +| class_type_list C_TK error + {yyerror ("Missing class type term"); RECOVER;} +; + +method_body: + block +| SC_TK { $$ = NULL_TREE; } +; + +/* 19.8.4 Productions from 8.5: Static Initializers */ +static_initializer: + static block + { + TREE_CHAIN ($2) = CPC_STATIC_INITIALIZER_STMT (ctxp); + SET_CPC_STATIC_INITIALIZER_STMT (ctxp, $2); + current_static_block = NULL_TREE; + } +; + +static: /* Test lval.sub_token here */ + modifiers + { + check_modifiers ("Illegal modifier `%s' for static initializer", $1, ACC_STATIC); + /* Can't have a static initializer in an innerclass */ + if ($1 | ACC_STATIC && + GET_CPC_LIST () && !TOPLEVEL_CLASS_DECL_P (GET_CPC ())) + parse_error_context + (MODIFIER_WFL (STATIC_TK), + "Can't define static initializer in class `%s'. Static initializer can only be defined in top-level classes", + IDENTIFIER_POINTER (DECL_NAME (GET_CPC ()))); + SOURCE_FRONTEND_DEBUG (("Modifiers: %d", $1)); + } +; + +/* 19.8.5 Productions from 8.6: Constructor Declarations */ +constructor_declaration: + constructor_header + { + current_function_decl = $1; + source_start_java_method (current_function_decl); + } + constructor_body + { finish_method_declaration ($3); } +; + +constructor_header: + constructor_declarator throws + { $$ = method_header (0, NULL_TREE, $1, $2); } +| modifiers constructor_declarator throws + { $$ = method_header ($1, NULL_TREE, $2, $3); } +; + +constructor_declarator: + simple_name OP_TK CP_TK + { + ctxp->formal_parameter_number = 0; + $$ = method_declarator ($1, NULL_TREE); + } +| simple_name OP_TK formal_parameter_list CP_TK + { $$ = method_declarator ($1, $3); } +; + +constructor_body: + /* Unlike regular method, we always need a complete (empty) + body so we can safely perform all the required code + addition (super invocation and field initialization) */ + block_begin constructor_block_end + { + BLOCK_EXPR_BODY ($2) = empty_stmt_node; + $$ = $2; + } +| block_begin explicit_constructor_invocation constructor_block_end + { $$ = $3; } +| block_begin block_statements constructor_block_end + { $$ = $3; } +| block_begin explicit_constructor_invocation block_statements constructor_block_end + { $$ = $4; } +; + +constructor_block_end: + block_end +; + +/* Error recovery for that rule moved down expression_statement: rule. */ +explicit_constructor_invocation: + this_or_super OP_TK CP_TK SC_TK + { + $$ = build_method_invocation ($1, NULL_TREE); + $$ = build_debugable_stmt (EXPR_WFL_LINECOL ($1), $$); + $$ = java_method_add_stmt (current_function_decl, $$); + } +| this_or_super OP_TK argument_list CP_TK SC_TK + { + $$ = build_method_invocation ($1, $3); + $$ = build_debugable_stmt (EXPR_WFL_LINECOL ($1), $$); + $$ = java_method_add_stmt (current_function_decl, $$); + } + /* Added, JDK1.1 inner classes. Modified because the rule + 'primary' couldn't work. */ +| name DOT_TK SUPER_TK OP_TK argument_list CP_TK SC_TK + {$$ = parse_jdk1_1_error ("explicit constructor invocation"); } +| name DOT_TK SUPER_TK OP_TK CP_TK SC_TK + {$$ = parse_jdk1_1_error ("explicit constructor invocation"); } +; + +this_or_super: /* Added, simplifies error diagnostics */ + THIS_TK + { + tree wfl = build_wfl_node (this_identifier_node); + EXPR_WFL_LINECOL (wfl) = $1.location; + $$ = wfl; + } +| SUPER_TK + { + tree wfl = build_wfl_node (super_identifier_node); + EXPR_WFL_LINECOL (wfl) = $1.location; + $$ = wfl; + } +; + +/* 19.9 Productions from 9: Interfaces */ +/* 19.9.1 Productions from 9.1: Interfaces Declarations */ +interface_declaration: + INTERFACE_TK identifier + { create_interface (0, $2, NULL_TREE); } + interface_body + { ; } +| modifiers INTERFACE_TK identifier + { create_interface ($1, $3, NULL_TREE); } + interface_body + { ; } +| INTERFACE_TK identifier extends_interfaces + { create_interface (0, $2, $3); } + interface_body + { ; } +| modifiers INTERFACE_TK identifier extends_interfaces + { create_interface ($1, $3, $4); } + interface_body + { ; } +| INTERFACE_TK identifier error + { yyerror (" expected"); RECOVER; } +| modifiers INTERFACE_TK identifier error + { yyerror (" expected"); RECOVER; } +; + +extends_interfaces: + EXTENDS_TK interface_type + { + ctxp->interface_number = 1; + $$ = build_tree_list ($2, NULL_TREE); + } +| extends_interfaces C_TK interface_type + { + ctxp->interface_number++; + $$ = chainon ($1, build_tree_list ($3, NULL_TREE)); + } +| EXTENDS_TK error + {yyerror ("Invalid interface type"); RECOVER;} +| extends_interfaces C_TK error + {yyerror ("Missing term"); RECOVER;} +; + +interface_body: + OCB_TK CCB_TK + { $$ = NULL_TREE; } +| OCB_TK interface_member_declarations CCB_TK + { $$ = NULL_TREE; } +; + +interface_member_declarations: + interface_member_declaration +| interface_member_declarations interface_member_declaration +; + +interface_member_declaration: + constant_declaration +| abstract_method_declaration +| class_declaration /* Added, JDK1.1 inner classes */ + { end_class_declaration (1); } +| interface_declaration /* Added, JDK1.1 inner interfaces */ + { end_class_declaration (1); } +; + +constant_declaration: + field_declaration +; + +abstract_method_declaration: + method_header SC_TK + { + check_abstract_method_header ($1); + current_function_decl = NULL_TREE; /* FIXME ? */ + } +| method_header error + {yyerror ("';' expected"); RECOVER;} +; + +/* 19.10 Productions from 10: Arrays */ +array_initializer: + OCB_TK CCB_TK + { $$ = build_new_array_init ($1.location, NULL_TREE); } +| OCB_TK C_TK CCB_TK + { $$ = build_new_array_init ($1.location, NULL_TREE); } +| OCB_TK variable_initializers CCB_TK + { $$ = build_new_array_init ($1.location, $2); } +| OCB_TK variable_initializers C_TK CCB_TK + { $$ = build_new_array_init ($1.location, $2); } +; + +variable_initializers: + variable_initializer + { + $$ = tree_cons (maybe_build_array_element_wfl ($1), + $1, NULL_TREE); + } +| variable_initializers C_TK variable_initializer + { + $$ = tree_cons (maybe_build_array_element_wfl ($3), $3, $1); + } +| variable_initializers C_TK error + {yyerror ("Missing term"); RECOVER;} +; + +/* 19.11 Production from 14: Blocks and Statements */ +block: + block_begin block_end + { $$ = $2; } +| block_begin block_statements block_end + { $$ = $3; } +; + +block_begin: + OCB_TK + { enter_block (); } +; + +block_end: + CCB_TK + { + maybe_absorb_scoping_blocks (); + /* Store the location of the when doing xrefs */ + if (current_function_decl && flag_emit_xref) + DECL_END_SOURCE_LINE (current_function_decl) = + EXPR_WFL_ADD_COL ($1.location, 1); + $$ = exit_block (); + if (!BLOCK_SUBBLOCKS ($$)) + BLOCK_SUBBLOCKS ($$) = empty_stmt_node; + } +; + +block_statements: + block_statement +| block_statements block_statement +; + +block_statement: + local_variable_declaration_statement +| statement + { java_method_add_stmt (current_function_decl, $1); } +| class_declaration /* Added, JDK1.1 local classes */ + { + LOCAL_CLASS_P (TREE_TYPE (GET_CPC ())) = 1; + end_class_declaration (1); + } +; + +local_variable_declaration_statement: + local_variable_declaration SC_TK /* Can't catch missing ';' here */ +; + +local_variable_declaration: + type variable_declarators + { declare_local_variables (0, $1, $2); } +| final type variable_declarators /* Added, JDK1.1 final locals */ + { declare_local_variables ($1, $2, $3); } +; + +statement: + statement_without_trailing_substatement +| labeled_statement +| if_then_statement +| if_then_else_statement +| while_statement +| for_statement + { $$ = exit_block (); } +; + +statement_nsi: + statement_without_trailing_substatement +| labeled_statement_nsi +| if_then_else_statement_nsi +| while_statement_nsi +| for_statement_nsi + { $$ = exit_block (); } +; + +statement_without_trailing_substatement: + block +| empty_statement +| expression_statement +| switch_statement +| do_statement +| break_statement +| continue_statement +| return_statement +| synchronized_statement +| throw_statement +| try_statement +| assert_statement +; + +empty_statement: + SC_TK + { + if (flag_extraneous_semicolon + && ! current_static_block + && (! current_function_decl || + /* Verify we're not in a inner class declaration */ + (GET_CPC () != TYPE_NAME + (DECL_CONTEXT (current_function_decl))))) + + { + EXPR_WFL_SET_LINECOL (wfl_operator, input_line, -1); + parse_warning_context (wfl_operator, "An empty declaration is a deprecated feature that should not be used"); + } + $$ = empty_stmt_node; + } +; + +label_decl: + identifier REL_CL_TK + { + $$ = build_labeled_block (EXPR_WFL_LINECOL ($1), + EXPR_WFL_NODE ($1)); + pushlevel (2); + push_labeled_block ($$); + PUSH_LABELED_BLOCK ($$); + } +; + +labeled_statement: + label_decl statement + { $$ = finish_labeled_statement ($1, $2); } +| identifier error + {yyerror ("':' expected"); RECOVER;} +; + +labeled_statement_nsi: + label_decl statement_nsi + { $$ = finish_labeled_statement ($1, $2); } +; + +/* We concentrate here a bunch of error handling rules that we couldn't write + earlier, because expression_statement catches a missing ';'. */ +expression_statement: + statement_expression SC_TK + { + /* We have a statement. Generate a WFL around it so + we can debug it */ + $$ = build_expr_wfl ($1, input_filename, input_line, 0); + /* We know we have a statement, so set the debug + info to be eventually generate here. */ + $$ = JAVA_MAYBE_GENERATE_DEBUG_INFO ($$); + } +| error SC_TK + { + YYNOT_TWICE yyerror ("Invalid expression statement"); + DRECOVER (expr_stmt); + } +| error OCB_TK + { + YYNOT_TWICE yyerror ("Invalid expression statement"); + DRECOVER (expr_stmt); + } +| error CCB_TK + { + YYNOT_TWICE yyerror ("Invalid expression statement"); + DRECOVER (expr_stmt); + } +| this_or_super OP_TK error + {yyerror ("')' expected"); RECOVER;} +| this_or_super OP_TK CP_TK error + { + parse_ctor_invocation_error (); + RECOVER; + } +| this_or_super OP_TK argument_list error + {yyerror ("')' expected"); RECOVER;} +| this_or_super OP_TK argument_list CP_TK error + { + parse_ctor_invocation_error (); + RECOVER; + } +| name DOT_TK SUPER_TK error + {yyerror ("'(' expected"); RECOVER;} +| name DOT_TK SUPER_TK OP_TK error + {yyerror ("')' expected"); RECOVER;} +| name DOT_TK SUPER_TK OP_TK argument_list error + {yyerror ("')' expected"); RECOVER;} +| name DOT_TK SUPER_TK OP_TK argument_list CP_TK error + {yyerror ("';' expected"); RECOVER;} +| name DOT_TK SUPER_TK OP_TK CP_TK error + {yyerror ("';' expected"); RECOVER;} +; + +statement_expression: + assignment +| pre_increment_expression +| pre_decrement_expression +| post_increment_expression +| post_decrement_expression +| method_invocation +| class_instance_creation_expression +; + +if_then_statement: + IF_TK OP_TK expression CP_TK statement + { + $$ = build_if_else_statement ($2.location, $3, + $5, NULL_TREE); + } +| IF_TK error + {yyerror ("'(' expected"); RECOVER;} +| IF_TK OP_TK error + {yyerror ("Missing term"); RECOVER;} +| IF_TK OP_TK expression error + {yyerror ("')' expected"); RECOVER;} +; + +if_then_else_statement: + IF_TK OP_TK expression CP_TK statement_nsi ELSE_TK statement + { $$ = build_if_else_statement ($2.location, $3, $5, $7); } +; + +if_then_else_statement_nsi: + IF_TK OP_TK expression CP_TK statement_nsi ELSE_TK statement_nsi + { $$ = build_if_else_statement ($2.location, $3, $5, $7); } +; + +switch_statement: + switch_expression + { + enter_block (); + } + switch_block + { + /* Make into "proper list" of COMPOUND_EXPRs. + I.e. make the last statement also have its own + COMPOUND_EXPR. */ + maybe_absorb_scoping_blocks (); + TREE_OPERAND ($1, 1) = exit_block (); + $$ = build_debugable_stmt (EXPR_WFL_LINECOL ($1), $1); + } +; + +switch_expression: + SWITCH_TK OP_TK expression CP_TK + { + $$ = build (SWITCH_EXPR, NULL_TREE, $3, NULL_TREE); + EXPR_WFL_LINECOL ($$) = $2.location; + } +| SWITCH_TK error + {yyerror ("'(' expected"); RECOVER;} +| SWITCH_TK OP_TK error + {yyerror ("Missing term or ')'"); DRECOVER(switch_statement);} +| SWITCH_TK OP_TK expression CP_TK error + {yyerror (" expected"); RECOVER;} +; + +/* Default assignment is there to avoid type node on switch_block + node. */ + +switch_block: + OCB_TK CCB_TK + { $$ = NULL_TREE; } +| OCB_TK switch_labels CCB_TK + { $$ = NULL_TREE; } +| OCB_TK switch_block_statement_groups CCB_TK + { $$ = NULL_TREE; } +| OCB_TK switch_block_statement_groups switch_labels CCB_TK + { $$ = NULL_TREE; } +; + +switch_block_statement_groups: + switch_block_statement_group +| switch_block_statement_groups switch_block_statement_group +; + +switch_block_statement_group: + switch_labels block_statements +; + +switch_labels: + switch_label +| switch_labels switch_label +; + +switch_label: + CASE_TK constant_expression REL_CL_TK + { + tree lab = build1 (CASE_EXPR, NULL_TREE, $2); + EXPR_WFL_LINECOL (lab) = $1.location; + java_method_add_stmt (current_function_decl, lab); + } +| DEFAULT_TK REL_CL_TK + { + tree lab = build (DEFAULT_EXPR, NULL_TREE, NULL_TREE); + EXPR_WFL_LINECOL (lab) = $1.location; + java_method_add_stmt (current_function_decl, lab); + } +| CASE_TK error + {yyerror ("Missing or invalid constant expression"); RECOVER;} +| CASE_TK constant_expression error + {yyerror ("':' expected"); RECOVER;} +| DEFAULT_TK error + {yyerror ("':' expected"); RECOVER;} +; + +while_expression: + WHILE_TK OP_TK expression CP_TK + { + tree body = build_loop_body ($2.location, $3, 0); + $$ = build_new_loop (body); + } +; + +while_statement: + while_expression statement + { $$ = finish_loop_body (0, NULL_TREE, $2, 0); } +| WHILE_TK error + {YYERROR_NOW; yyerror ("'(' expected"); RECOVER;} +| WHILE_TK OP_TK error + {yyerror ("Missing term and ')' expected"); RECOVER;} +| WHILE_TK OP_TK expression error + {yyerror ("')' expected"); RECOVER;} +; + +while_statement_nsi: + while_expression statement_nsi + { $$ = finish_loop_body (0, NULL_TREE, $2, 0); } +; + +do_statement_begin: + DO_TK + { + tree body = build_loop_body (0, NULL_TREE, 1); + $$ = build_new_loop (body); + } + /* Need error handing here. FIXME */ +; + +do_statement: + do_statement_begin statement WHILE_TK OP_TK expression CP_TK SC_TK + { $$ = finish_loop_body ($4.location, $5, $2, 1); } +; + +for_statement: + for_begin SC_TK expression SC_TK for_update CP_TK statement + { + if (TREE_CODE_CLASS (TREE_CODE ($3)) == 'c') + $3 = build_wfl_node ($3); + $$ = finish_for_loop (EXPR_WFL_LINECOL ($3), $3, $5, $7); + } +| for_begin SC_TK SC_TK for_update CP_TK statement + { + $$ = finish_for_loop (0, NULL_TREE, $4, $6); + /* We have not condition, so we get rid of the EXIT_EXPR */ + LOOP_EXPR_BODY_CONDITION_EXPR (LOOP_EXPR_BODY ($$), 0) = + empty_stmt_node; + } +| for_begin SC_TK error + {yyerror ("Invalid control expression"); RECOVER;} +| for_begin SC_TK expression SC_TK error + {yyerror ("Invalid update expression"); RECOVER;} +| for_begin SC_TK SC_TK error + {yyerror ("Invalid update expression"); RECOVER;} +; + +for_statement_nsi: + for_begin SC_TK expression SC_TK for_update CP_TK statement_nsi + { $$ = finish_for_loop (EXPR_WFL_LINECOL ($3), $3, $5, $7);} +| for_begin SC_TK SC_TK for_update CP_TK statement_nsi + { + $$ = finish_for_loop (0, NULL_TREE, $4, $6); + /* We have not condition, so we get rid of the EXIT_EXPR */ + LOOP_EXPR_BODY_CONDITION_EXPR (LOOP_EXPR_BODY ($$), 0) = + empty_stmt_node; + } +; + +for_header: + FOR_TK OP_TK + { + /* This scope defined for local variable that may be + defined within the scope of the for loop */ + enter_block (); + } +| FOR_TK error + {yyerror ("'(' expected"); DRECOVER(for_1);} +| FOR_TK OP_TK error + {yyerror ("Invalid init statement"); RECOVER;} +; + +for_begin: + for_header for_init + { + /* We now declare the loop body. The loop is + declared as a for loop. */ + tree body = build_loop_body (0, NULL_TREE, 0); + $$ = build_new_loop (body); + FOR_LOOP_P ($$) = 1; + /* The loop is added to the current block the for + statement is defined within */ + java_method_add_stmt (current_function_decl, $$); + } +; +for_init: /* Can be empty */ + { $$ = empty_stmt_node; } +| statement_expression_list + { + /* Init statement recorded within the previously + defined block scope */ + $$ = java_method_add_stmt (current_function_decl, $1); + } +| local_variable_declaration + { + /* Local variable are recorded within the previously + defined block scope */ + $$ = NULL_TREE; + } +| statement_expression_list error + {yyerror ("';' expected"); DRECOVER(for_init_1);} +; + +for_update: /* Can be empty */ + {$$ = empty_stmt_node;} +| statement_expression_list + { $$ = build_debugable_stmt (BUILD_LOCATION (), $1); } +; + +statement_expression_list: + statement_expression + { $$ = add_stmt_to_compound (NULL_TREE, NULL_TREE, $1); } +| statement_expression_list C_TK statement_expression + { $$ = add_stmt_to_compound ($1, NULL_TREE, $3); } +| statement_expression_list C_TK error + {yyerror ("Missing term"); RECOVER;} +; + +break_statement: + BREAK_TK SC_TK + { $$ = build_bc_statement ($1.location, 1, NULL_TREE); } +| BREAK_TK identifier SC_TK + { $$ = build_bc_statement ($1.location, 1, $2); } +| BREAK_TK error + {yyerror ("Missing term"); RECOVER;} +| BREAK_TK identifier error + {yyerror ("';' expected"); RECOVER;} +; + +continue_statement: + CONTINUE_TK SC_TK + { $$ = build_bc_statement ($1.location, 0, NULL_TREE); } +| CONTINUE_TK identifier SC_TK + { $$ = build_bc_statement ($1.location, 0, $2); } +| CONTINUE_TK error + {yyerror ("Missing term"); RECOVER;} +| CONTINUE_TK identifier error + {yyerror ("';' expected"); RECOVER;} +; + +return_statement: + RETURN_TK SC_TK + { $$ = build_return ($1.location, NULL_TREE); } +| RETURN_TK expression SC_TK + { $$ = build_return ($1.location, $2); } +| RETURN_TK error + {yyerror ("Missing term"); RECOVER;} +| RETURN_TK expression error + {yyerror ("';' expected"); RECOVER;} +; + +throw_statement: + THROW_TK expression SC_TK + { + $$ = build1 (THROW_EXPR, NULL_TREE, $2); + EXPR_WFL_LINECOL ($$) = $1.location; + } +| THROW_TK error + {yyerror ("Missing term"); RECOVER;} +| THROW_TK expression error + {yyerror ("';' expected"); RECOVER;} +; + +assert_statement: + ASSERT_TK expression REL_CL_TK expression SC_TK + { + $$ = build_assertion ($1.location, $2, $4); + } +| ASSERT_TK expression SC_TK + { + $$ = build_assertion ($1.location, $2, NULL_TREE); + } +| ASSERT_TK error + {yyerror ("Missing term"); RECOVER;} +| ASSERT_TK expression error + {yyerror ("';' expected"); RECOVER;} +; + +synchronized_statement: + synchronized OP_TK expression CP_TK block + { + $$ = build (SYNCHRONIZED_EXPR, NULL_TREE, $3, $5); + EXPR_WFL_LINECOL ($$) = + EXPR_WFL_LINECOL (MODIFIER_WFL (SYNCHRONIZED_TK)); + } +| synchronized OP_TK expression CP_TK error + {yyerror (" expected"); RECOVER;} +| synchronized error + {yyerror ("'(' expected"); RECOVER;} +| synchronized OP_TK error CP_TK + {yyerror ("Missing term"); RECOVER;} +| synchronized OP_TK error + {yyerror ("Missing term"); RECOVER;} +; + +synchronized: + modifiers + { + check_modifiers ( + "Illegal modifier `%s'. Only `synchronized' was expected here", + $1, ACC_SYNCHRONIZED); + if ($1 != ACC_SYNCHRONIZED) + MODIFIER_WFL (SYNCHRONIZED_TK) = + build_wfl_node (NULL_TREE); + } +; + +try_statement: + TRY_TK block catches + { $$ = build_try_statement ($1.location, $2, $3); } +| TRY_TK block finally + { $$ = build_try_finally_statement ($1.location, $2, $3); } +| TRY_TK block catches finally + { $$ = build_try_finally_statement + ($1.location, build_try_statement ($1.location, + $2, $3), $4); + } +| TRY_TK error + {yyerror (" expected"); DRECOVER (try_statement);} +; + +catches: + catch_clause +| catches catch_clause + { + TREE_CHAIN ($2) = $1; + $$ = $2; + } +; + +catch_clause: + catch_clause_parameter block + { + java_method_add_stmt (current_function_decl, $2); + exit_block (); + $$ = $1; + } +; + +catch_clause_parameter: + CATCH_TK OP_TK formal_parameter CP_TK + { + /* We add a block to define a scope for + formal_parameter (CCBP). The formal parameter is + declared initialized by the appropriate function + call */ + tree ccpb; + tree init; + if ($3) + { + ccpb = enter_block (); + init = build_assignment + (ASSIGN_TK, $2.location, TREE_PURPOSE ($3), + build (JAVA_EXC_OBJ_EXPR, ptr_type_node)); + declare_local_variables (0, TREE_VALUE ($3), + build_tree_list + (TREE_PURPOSE ($3), init)); + $$ = build1 (CATCH_EXPR, NULL_TREE, ccpb); + EXPR_WFL_LINECOL ($$) = $1.location; + } + else + { + $$ = error_mark_node; + } + } +| CATCH_TK error + {yyerror ("'(' expected"); RECOVER; $$ = NULL_TREE;} +| CATCH_TK OP_TK error + { + yyerror ("Missing term or ')' expected"); + RECOVER; $$ = NULL_TREE; + } +| CATCH_TK OP_TK error CP_TK /* That's for () */ + {yyerror ("Missing term"); RECOVER; $$ = NULL_TREE;} +; + +finally: + FINALLY_TK block + { $$ = $2; } +| FINALLY_TK error + {yyerror (" expected"); RECOVER; } +; + +/* 19.12 Production from 15: Expressions */ +primary: + primary_no_new_array +| array_creation_expression +; + +primary_no_new_array: + literal +| THIS_TK + { $$ = build_this ($1.location); } +| OP_TK expression CP_TK + {$$ = $2;} +| class_instance_creation_expression +| field_access +| method_invocation +| array_access +| type_literals + /* Added, JDK1.1 inner classes. Documentation is wrong + refering to a 'ClassName' (class_name) rule that doesn't + exist. Used name: instead. */ +| name DOT_TK THIS_TK + { + tree wfl = build_wfl_node (this_identifier_node); + $$ = make_qualified_primary ($1, wfl, EXPR_WFL_LINECOL ($1)); + } +| OP_TK expression error + {yyerror ("')' expected"); RECOVER;} +| name DOT_TK error + {yyerror ("'class' or 'this' expected" ); RECOVER;} +| primitive_type DOT_TK error + {yyerror ("'class' expected" ); RECOVER;} +| VOID_TK DOT_TK error + {yyerror ("'class' expected" ); RECOVER;} +; + +type_literals: + name DOT_TK CLASS_TK + { $$ = build_incomplete_class_ref ($2.location, $1); } +| array_type DOT_TK CLASS_TK + { $$ = build_incomplete_class_ref ($2.location, $1); } +| primitive_type DOT_TK CLASS_TK + { $$ = build_incomplete_class_ref ($2.location, $1); } +| VOID_TK DOT_TK CLASS_TK + { + $$ = build_incomplete_class_ref ($2.location, + void_type_node); + } +; + +class_instance_creation_expression: + NEW_TK class_type OP_TK argument_list CP_TK + { $$ = build_new_invocation ($2, $4); } +| NEW_TK class_type OP_TK CP_TK + { $$ = build_new_invocation ($2, NULL_TREE); } +| anonymous_class_creation + /* Added, JDK1.1 inner classes, modified to use name or + primary instead of primary solely which couldn't work in + all situations. */ +| something_dot_new identifier OP_TK CP_TK + { + tree ctor = build_new_invocation ($2, NULL_TREE); + $$ = make_qualified_primary ($1, ctor, + EXPR_WFL_LINECOL ($1)); + } +| something_dot_new identifier OP_TK CP_TK class_body +| something_dot_new identifier OP_TK argument_list CP_TK + { + tree ctor = build_new_invocation ($2, $4); + $$ = make_qualified_primary ($1, ctor, + EXPR_WFL_LINECOL ($1)); + } +| something_dot_new identifier OP_TK argument_list CP_TK class_body +| NEW_TK error SC_TK + {yyerror ("'(' expected"); DRECOVER(new_1);} +| NEW_TK class_type error + {yyerror ("'(' expected"); RECOVER;} +| NEW_TK class_type OP_TK error + {yyerror ("')' or term expected"); RECOVER;} +| NEW_TK class_type OP_TK argument_list error + {yyerror ("')' expected"); RECOVER;} +| something_dot_new error + {YYERROR_NOW; yyerror ("Identifier expected"); RECOVER;} +| something_dot_new identifier error + {yyerror ("'(' expected"); RECOVER;} +; + +/* Created after JDK1.1 rules originally added to + class_instance_creation_expression, but modified to use + 'class_type' instead of 'TypeName' (type_name) which is mentioned + in the documentation but doesn't exist. */ + +anonymous_class_creation: + NEW_TK class_type OP_TK argument_list CP_TK + { create_anonymous_class ($1.location, $2); } + class_body + { + tree id = build_wfl_node (DECL_NAME (GET_CPC ())); + EXPR_WFL_LINECOL (id) = EXPR_WFL_LINECOL ($2); + + end_class_declaration (1); + + /* Now we can craft the new expression */ + $$ = build_new_invocation (id, $4); + + /* Note that we can't possibly be here if + `class_type' is an interface (in which case the + anonymous class extends Object and implements + `class_type', hence its constructor can't have + arguments.) */ + + /* Otherwise, the innerclass must feature a + constructor matching `argument_list'. Anonymous + classes are a bit special: it's impossible to + define constructor for them, hence constructors + must be generated following the hints provided by + the `new' expression. Whether a super constructor + of that nature exists or not is to be verified + later on in verify_constructor_super. + + It's during the expansion of a `new' statement + refering to an anonymous class that a ctor will + be generated for the anonymous class, with the + right arguments. */ + + } +| NEW_TK class_type OP_TK CP_TK + { create_anonymous_class ($1.location, $2); } + class_body + { + tree id = build_wfl_node (DECL_NAME (GET_CPC ())); + EXPR_WFL_LINECOL (id) = EXPR_WFL_LINECOL ($2); + + end_class_declaration (1); + + /* Now we can craft the new expression. The + statement doesn't need to be remember so that a + constructor can be generated, since its signature + is already known. */ + $$ = build_new_invocation (id, NULL_TREE); + } +; + +something_dot_new: /* Added, not part of the specs. */ + name DOT_TK NEW_TK + { $$ = $1; } +| primary DOT_TK NEW_TK + { $$ = $1; } +; + +argument_list: + expression + { + $$ = tree_cons (NULL_TREE, $1, NULL_TREE); + ctxp->formal_parameter_number = 1; + } +| argument_list C_TK expression + { + ctxp->formal_parameter_number += 1; + $$ = tree_cons (NULL_TREE, $3, $1); + } +| argument_list C_TK error + {yyerror ("Missing term"); RECOVER;} +; + +array_creation_expression: + NEW_TK primitive_type dim_exprs + { $$ = build_newarray_node ($2, $3, 0); } +| NEW_TK class_or_interface_type dim_exprs + { $$ = build_newarray_node ($2, $3, 0); } +| NEW_TK primitive_type dim_exprs dims + { $$ = build_newarray_node ($2, $3, pop_current_osb (ctxp));} +| NEW_TK class_or_interface_type dim_exprs dims + { $$ = build_newarray_node ($2, $3, pop_current_osb (ctxp));} + /* Added, JDK1.1 anonymous array. Initial documentation rule + modified */ +| NEW_TK class_or_interface_type dims array_initializer + { + char *sig; + int osb = pop_current_osb (ctxp); + while (osb--) + obstack_grow (&temporary_obstack, "[]", 2); + obstack_1grow (&temporary_obstack, '\0'); + sig = obstack_finish (&temporary_obstack); + $$ = build (NEW_ANONYMOUS_ARRAY_EXPR, NULL_TREE, + $2, get_identifier (sig), $4); + } +| NEW_TK primitive_type dims array_initializer + { + int osb = pop_current_osb (ctxp); + tree type = $2; + while (osb--) + type = build_java_array_type (type, -1); + $$ = build (NEW_ANONYMOUS_ARRAY_EXPR, NULL_TREE, + build_pointer_type (type), NULL_TREE, $4); + } +| NEW_TK error CSB_TK + {yyerror ("'[' expected"); DRECOVER ("]");} +| NEW_TK error OSB_TK + {yyerror ("']' expected"); RECOVER;} +; + +dim_exprs: + dim_expr + { $$ = build_tree_list (NULL_TREE, $1); } +| dim_exprs dim_expr + { $$ = tree_cons (NULL_TREE, $2, $$); } +; + +dim_expr: + OSB_TK expression CSB_TK + { + if (JNUMERIC_TYPE_P (TREE_TYPE ($2))) + { + $2 = build_wfl_node ($2); + TREE_TYPE ($2) = NULL_TREE; + } + EXPR_WFL_LINECOL ($2) = $1.location; + $$ = $2; + } +| OSB_TK expression error + {yyerror ("']' expected"); RECOVER;} +| OSB_TK error + { + yyerror ("Missing term"); + yyerror ("']' expected"); + RECOVER; + } +; + +dims: + OSB_TK CSB_TK + { + int allocate = 0; + /* If not initialized, allocate memory for the osb + numbers stack */ + if (!ctxp->osb_limit) + { + allocate = ctxp->osb_limit = 32; + ctxp->osb_depth = -1; + } + /* If capacity overflown, reallocate a bigger chunk */ + else if (ctxp->osb_depth+1 == ctxp->osb_limit) + allocate = ctxp->osb_limit << 1; + + if (allocate) + { + allocate *= sizeof (int); + if (ctxp->osb_number) + ctxp->osb_number = xrealloc (ctxp->osb_number, + allocate); + else + ctxp->osb_number = xmalloc (allocate); + } + ctxp->osb_depth++; + CURRENT_OSB (ctxp) = 1; + } +| dims OSB_TK CSB_TK + { CURRENT_OSB (ctxp)++; } +| dims OSB_TK error + { yyerror ("']' expected"); RECOVER;} +; + +field_access: + primary DOT_TK identifier + { $$ = make_qualified_primary ($1, $3, $2.location); } + /* FIXME - REWRITE TO: + { $$ = build_binop (COMPONENT_REF, $2.location, $1, $3); } */ +| SUPER_TK DOT_TK identifier + { + tree super_wfl = build_wfl_node (super_identifier_node); + EXPR_WFL_LINECOL (super_wfl) = $1.location; + $$ = make_qualified_name (super_wfl, $3, $2.location); + } +| SUPER_TK error + {yyerror ("Field expected"); DRECOVER (super_field_acces);} +; + +method_invocation: + name OP_TK CP_TK + { $$ = build_method_invocation ($1, NULL_TREE); } +| name OP_TK argument_list CP_TK + { $$ = build_method_invocation ($1, $3); } +| primary DOT_TK identifier OP_TK CP_TK + { + if (TREE_CODE ($1) == THIS_EXPR) + $$ = build_this_super_qualified_invocation + (1, $3, NULL_TREE, 0, $2.location); + else + { + tree invok = build_method_invocation ($3, NULL_TREE); + $$ = make_qualified_primary ($1, invok, $2.location); + } + } +| primary DOT_TK identifier OP_TK argument_list CP_TK + { + if (TREE_CODE ($1) == THIS_EXPR) + $$ = build_this_super_qualified_invocation + (1, $3, $5, 0, $2.location); + else + { + tree invok = build_method_invocation ($3, $5); + $$ = make_qualified_primary ($1, invok, $2.location); + } + } +| SUPER_TK DOT_TK identifier OP_TK CP_TK + { + $$ = build_this_super_qualified_invocation + (0, $3, NULL_TREE, $1.location, $2.location); + } +| SUPER_TK DOT_TK identifier OP_TK argument_list CP_TK + { + $$ = build_this_super_qualified_invocation + (0, $3, $5, $1.location, $2.location); + } + /* Screws up thing. I let it here until I'm convinced it can + be removed. FIXME +| primary DOT_TK error + {yyerror ("'(' expected"); DRECOVER(bad);} */ +| SUPER_TK DOT_TK error CP_TK + { yyerror ("'(' expected"); DRECOVER (method_invocation); } +| SUPER_TK DOT_TK error DOT_TK + { yyerror ("'(' expected"); DRECOVER (method_invocation); } +; + +array_access: + name OSB_TK expression CSB_TK + { $$ = build_array_ref ($2.location, $1, $3); } +| primary_no_new_array OSB_TK expression CSB_TK + { $$ = build_array_ref ($2.location, $1, $3); } +| name OSB_TK error + { + yyerror ("Missing term and ']' expected"); + DRECOVER(array_access); + } +| name OSB_TK expression error + { + yyerror ("']' expected"); + DRECOVER(array_access); + } +| primary_no_new_array OSB_TK error + { + yyerror ("Missing term and ']' expected"); + DRECOVER(array_access); + } +| primary_no_new_array OSB_TK expression error + { + yyerror ("']' expected"); + DRECOVER(array_access); + } +; + +postfix_expression: + primary +| name +| post_increment_expression +| post_decrement_expression +; + +post_increment_expression: + postfix_expression INCR_TK + { $$ = build_incdec ($2.token, $2.location, $1, 1); } +; + +post_decrement_expression: + postfix_expression DECR_TK + { $$ = build_incdec ($2.token, $2.location, $1, 1); } +; + +trap_overflow_corner_case: + pre_increment_expression +| pre_decrement_expression +| PLUS_TK unary_expression + {$$ = build_unaryop ($1.token, $1.location, $2); } +| unary_expression_not_plus_minus +| PLUS_TK error + {yyerror ("Missing term"); RECOVER} +; + +unary_expression: + trap_overflow_corner_case + { + error_if_numeric_overflow ($1); + $$ = $1; + } +| MINUS_TK trap_overflow_corner_case + {$$ = build_unaryop ($1.token, $1.location, $2); } +| MINUS_TK error + {yyerror ("Missing term"); RECOVER} +; + +pre_increment_expression: + INCR_TK unary_expression + {$$ = build_incdec ($1.token, $1.location, $2, 0); } +| INCR_TK error + {yyerror ("Missing term"); RECOVER} +; + +pre_decrement_expression: + DECR_TK unary_expression + {$$ = build_incdec ($1.token, $1.location, $2, 0); } +| DECR_TK error + {yyerror ("Missing term"); RECOVER} +; + +unary_expression_not_plus_minus: + postfix_expression +| NOT_TK unary_expression + {$$ = build_unaryop ($1.token, $1.location, $2); } +| NEG_TK unary_expression + {$$ = build_unaryop ($1.token, $1.location, $2); } +| cast_expression +| NOT_TK error + {yyerror ("Missing term"); RECOVER} +| NEG_TK error + {yyerror ("Missing term"); RECOVER} +; + +cast_expression: /* Error handling here is potentially weak */ + OP_TK primitive_type dims CP_TK unary_expression + { + tree type = $2; + int osb = pop_current_osb (ctxp); + while (osb--) + type = build_java_array_type (type, -1); + $$ = build_cast ($1.location, type, $5); + } +| OP_TK primitive_type CP_TK unary_expression + { $$ = build_cast ($1.location, $2, $4); } +| OP_TK expression CP_TK unary_expression_not_plus_minus + { $$ = build_cast ($1.location, $2, $4); } +| OP_TK name dims CP_TK unary_expression_not_plus_minus + { + const char *ptr; + int osb = pop_current_osb (ctxp); + obstack_grow (&temporary_obstack, + IDENTIFIER_POINTER (EXPR_WFL_NODE ($2)), + IDENTIFIER_LENGTH (EXPR_WFL_NODE ($2))); + while (osb--) + obstack_grow (&temporary_obstack, "[]", 2); + obstack_1grow (&temporary_obstack, '\0'); + ptr = obstack_finish (&temporary_obstack); + EXPR_WFL_NODE ($2) = get_identifier (ptr); + $$ = build_cast ($1.location, $2, $5); + } +| OP_TK primitive_type OSB_TK error + {yyerror ("']' expected, invalid type expression");} +| OP_TK error + { + YYNOT_TWICE yyerror ("Invalid type expression"); RECOVER; + RECOVER; + } +| OP_TK primitive_type dims CP_TK error + {yyerror ("Missing term"); RECOVER;} +| OP_TK primitive_type CP_TK error + {yyerror ("Missing term"); RECOVER;} +| OP_TK name dims CP_TK error + {yyerror ("Missing term"); RECOVER;} +; + +multiplicative_expression: + unary_expression +| multiplicative_expression MULT_TK unary_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), + $2.location, $1, $3); + } +| multiplicative_expression DIV_TK unary_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| multiplicative_expression REM_TK unary_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| multiplicative_expression MULT_TK error + {yyerror ("Missing term"); RECOVER;} +| multiplicative_expression DIV_TK error + {yyerror ("Missing term"); RECOVER;} +| multiplicative_expression REM_TK error + {yyerror ("Missing term"); RECOVER;} +; + +additive_expression: + multiplicative_expression +| additive_expression PLUS_TK multiplicative_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| additive_expression MINUS_TK multiplicative_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| additive_expression PLUS_TK error + {yyerror ("Missing term"); RECOVER;} +| additive_expression MINUS_TK error + {yyerror ("Missing term"); RECOVER;} +; + +shift_expression: + additive_expression +| shift_expression LS_TK additive_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| shift_expression SRS_TK additive_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| shift_expression ZRS_TK additive_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| shift_expression LS_TK error + {yyerror ("Missing term"); RECOVER;} +| shift_expression SRS_TK error + {yyerror ("Missing term"); RECOVER;} +| shift_expression ZRS_TK error + {yyerror ("Missing term"); RECOVER;} +; + +relational_expression: + shift_expression +| relational_expression LT_TK shift_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| relational_expression GT_TK shift_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| relational_expression LTE_TK shift_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| relational_expression GTE_TK shift_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| relational_expression INSTANCEOF_TK reference_type + { $$ = build_binop (INSTANCEOF_EXPR, $2.location, $1, $3); } +| relational_expression LT_TK error + {yyerror ("Missing term"); RECOVER;} +| relational_expression GT_TK error + {yyerror ("Missing term"); RECOVER;} +| relational_expression LTE_TK error + {yyerror ("Missing term"); RECOVER;} +| relational_expression GTE_TK error + {yyerror ("Missing term"); RECOVER;} +| relational_expression INSTANCEOF_TK error + {yyerror ("Invalid reference type"); RECOVER;} +; + +equality_expression: + relational_expression +| equality_expression EQ_TK relational_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| equality_expression NEQ_TK relational_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| equality_expression EQ_TK error + {yyerror ("Missing term"); RECOVER;} +| equality_expression NEQ_TK error + {yyerror ("Missing term"); RECOVER;} +; + +and_expression: + equality_expression +| and_expression AND_TK equality_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| and_expression AND_TK error + {yyerror ("Missing term"); RECOVER;} +; + +exclusive_or_expression: + and_expression +| exclusive_or_expression XOR_TK and_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| exclusive_or_expression XOR_TK error + {yyerror ("Missing term"); RECOVER;} +; + +inclusive_or_expression: + exclusive_or_expression +| inclusive_or_expression OR_TK exclusive_or_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| inclusive_or_expression OR_TK error + {yyerror ("Missing term"); RECOVER;} +; + +conditional_and_expression: + inclusive_or_expression +| conditional_and_expression BOOL_AND_TK inclusive_or_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| conditional_and_expression BOOL_AND_TK error + {yyerror ("Missing term"); RECOVER;} +; + +conditional_or_expression: + conditional_and_expression +| conditional_or_expression BOOL_OR_TK conditional_and_expression + { + $$ = build_binop (BINOP_LOOKUP ($2.token), $2.location, + $1, $3); + } +| conditional_or_expression BOOL_OR_TK error + {yyerror ("Missing term"); RECOVER;} +; + +conditional_expression: /* Error handling here is weak */ + conditional_or_expression +| conditional_or_expression REL_QM_TK expression REL_CL_TK conditional_expression + { + $$ = build (CONDITIONAL_EXPR, NULL_TREE, $1, $3, $5); + EXPR_WFL_LINECOL ($$) = $2.location; + } +| conditional_or_expression REL_QM_TK REL_CL_TK error + { + YYERROR_NOW; + yyerror ("Missing term"); + DRECOVER (1); + } +| conditional_or_expression REL_QM_TK error + {yyerror ("Missing term"); DRECOVER (2);} +| conditional_or_expression REL_QM_TK expression REL_CL_TK error + {yyerror ("Missing term"); DRECOVER (3);} +; + +assignment_expression: + conditional_expression +| assignment +; + +assignment: + left_hand_side assignment_operator assignment_expression + { $$ = build_assignment ($2.token, $2.location, $1, $3); } +| left_hand_side assignment_operator error + { + YYNOT_TWICE yyerror ("Missing term"); + DRECOVER (assign); + } +; + +left_hand_side: + name +| field_access +| array_access +; + +assignment_operator: + assign_any +| ASSIGN_TK +; + +assign_any: + PLUS_ASSIGN_TK +| MINUS_ASSIGN_TK +| MULT_ASSIGN_TK +| DIV_ASSIGN_TK +| REM_ASSIGN_TK +| LS_ASSIGN_TK +| SRS_ASSIGN_TK +| ZRS_ASSIGN_TK +| AND_ASSIGN_TK +| XOR_ASSIGN_TK +| OR_ASSIGN_TK +; + + +expression: + assignment_expression +; + +constant_expression: + expression +; diff --git a/src/ptgen/java/j.y.foot b/src/ptgen/java/j.y.foot new file mode 100755 index 0000000..a9ca213 --- /dev/null +++ b/src/ptgen/java/j.y.foot @@ -0,0 +1,12 @@ +#include + +extern char yytext[]; +extern int column; +extern int line; + +void yyerror( char *s) +{ +fflush(stdout); +fprintf(stderr,"%s: %d.%d\n",s,line,column); +} + diff --git a/src/ptgen/java/j.y.head b/src/ptgen/java/j.y.head new file mode 100755 index 0000000..92b24e3 --- /dev/null +++ b/src/ptgen/java/j.y.head @@ -0,0 +1,54 @@ +%token PLUS_TK MINUS_TK MULT_TK DIV_TK REM_TK +%token LS_TK SRS_TK ZRS_TK +%token AND_TK XOR_TK OR_TK +%token BOOL_AND_TK BOOL_OR_TK +%token EQ_TK NEQ_TK GT_TK GTE_TK LT_TK LTE_TK + +/* This maps to the same binop_lookup entry than the token above */ + +%token PLUS_ASSIGN_TK MINUS_ASSIGN_TK MULT_ASSIGN_TK DIV_ASSIGN_TK +%token REM_ASSIGN_TK +%token LS_ASSIGN_TK SRS_ASSIGN_TK ZRS_ASSIGN_TK +%token AND_ASSIGN_TK XOR_ASSIGN_TK OR_ASSIGN_TK + + +/* Modifier TOKEN have to be kept in this order. Don't scramble it */ + +%token PUBLIC_TK PRIVATE_TK PROTECTED_TK +%token STATIC_TK FINAL_TK SYNCHRONIZED_TK +%token VOLATILE_TK TRANSIENT_TK NATIVE_TK +%token PAD_TK ABSTRACT_TK STRICT_TK +%token MODIFIER_TK + +/* Keep those two in order, too */ +%token DECR_TK INCR_TK + +/* From now one, things can be in any order */ + +%token DEFAULT_TK IF_TK THROW_TK +%token BOOLEAN_TK DO_TK IMPLEMENTS_TK +%token THROWS_TK BREAK_TK IMPORT_TK +%token ELSE_TK INSTANCEOF_TK RETURN_TK +%token VOID_TK CATCH_TK INTERFACE_TK +%token CASE_TK EXTENDS_TK FINALLY_TK +%token SUPER_TK WHILE_TK CLASS_TK +%token SWITCH_TK CONST_TK TRY_TK +%token FOR_TK NEW_TK CONTINUE_TK +%token GOTO_TK PACKAGE_TK THIS_TK +%token ASSERT_TK + +%token BYTE_TK SHORT_TK INT_TK LONG_TK +%token CHAR_TK INTEGRAL_TK + +%token FLOAT_TK DOUBLE_TK FP_TK + +%token ID_TK + +%token REL_QM_TK REL_CL_TK NOT_TK NEG_TK + +%token ASSIGN_ANY_TK ASSIGN_TK +%token OP_TK CP_TK OCB_TK CCB_TK OSB_TK CSB_TK SC_TK C_TK DOT_TK + +%token STRING_LIT_TK CHAR_LIT_TK INT_LIT_TK FP_LIT_TK +%token TRUE_TK FALSE_TK BOOL_LIT_TK NULL_TK + diff --git a/src/ptgen/java/jatomicNodes.h b/src/ptgen/java/jatomicNodes.h new file mode 100644 index 0000000..ed6f76c --- /dev/null +++ b/src/ptgen/java/jatomicNodes.h @@ -0,0 +1,34 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"expression_statement","expression","field_declaration","method_header","local_variable_declaration", +NULL diff --git a/src/ptgen/java/jcontextualNodes.h b/src/ptgen/java/jcontextualNodes.h new file mode 100644 index 0000000..27884ba --- /dev/null +++ b/src/ptgen/java/jcontextualNodes.h @@ -0,0 +1,55 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"goal", // "compilation_unit", "type_declarations", + "type_declaration", + "class_declaration", /* recursive */ + "class_body", + "class_body_declaration", + "constructor_declaration", + "method_declaration", +// "block_statement", "statement", "statement_nsi", ?? what's the diff between "statement" and "statement_nsi" +/* consider loops, branches, exceptions */ + "if_then_statement", "if_then_else_statement", "if_then_else_statement_nsi", + "while_statement", "while_statement_nsi", + "for_statement", "for_statement_nsi", + "switch_statement", + "do_statement", + "synchronized_statement", + "throw_statement", + "try_statement", + "catch_clause", "finally", + "interface_declaration", /* recursive */ + "interface_body", + "interface_member_declaration", + "abstract_method_declaration", +NULL diff --git a/src/ptgen/java/jparentNodes.h b/src/ptgen/java/jparentNodes.h new file mode 100644 index 0000000..e5bea2a --- /dev/null +++ b/src/ptgen/java/jparentNodes.h @@ -0,0 +1,34 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"statement","type_declaration", +NULL diff --git a/src/ptgen/java/jrelevantNodes.h b/src/ptgen/java/jrelevantNodes.h new file mode 100644 index 0000000..4fc0e93 --- /dev/null +++ b/src/ptgen/java/jrelevantNodes.h @@ -0,0 +1,175 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"type_declaration", +"identifier", +"package_declaration", +//"import_declaration", +"modifier", +"class_declaration", +"super", +"interfaces", +"class_member_declaration", +"field_declaration", +"method_declaration", +"variable_declarator", +"block", +"this_or_super", +"extends_interfaces", +"local_variable_declaration", +"label_decl", +"statement", +"expression_statement", +"if_then_statement", +"if_then_else_statement", +"switch_statement", +"while_statement", +"do_statement", +"for_statement", +"finally", +"primary", +"dims", +"method_invocation", +"field_access", +"array_access", +"post_increment_expression", +"pre_increment_expression", +"post_decrement_expression", +"pre_increment_expression", +"unary_expression", +"cast_expression", +"multiplicative_expression", +"shift_expression", +"additive_expression", +"relational_expression", +"equality_expression", +"conditional_expression", +"assignment_expression", +"assignment", +"expression", +"PLUS_TK", +"MINUS_TK", +"MULT_TK", +"DIV_TK", +"REM_TK", +"LS_TK", +"SRS_TK", +"ZRS_TK", +"AND_TK", +"XOR_TK", +"OR_TK", +"BOOL_AND_TK", +"BOOL_OR_TK", +"EQ_TK", +"NEQ_TK", +"GT_TK", +"GTE_TK", +"LT_TK", +"LTE_TK", +"PLUS_ASSIGN_TK", +"MINUS_ASSIGN_TK", +"MULT_ASSIGN_TK", +"DIV_ASSIGN_TK", +"REM_ASSIGN_TK", +"LS_ASSIGN_TK", +"SRS_ASSIGN_TK", +"ZRS_ASSIGN_TK", +"AND_ASSIGN_TK", +"XOR_ASSIGN_TK", +"OR_ASSIGN_TK", +"PUBLIC_TK", +"PRIVATE_TK", +"PROTECTED_TK", +"STATIC_TK", +"FINAL_TK", +"SYNCHRONIZED_TK", +"VOLATILE_TK", +"TRANSIENT_TK", +"NATIVE_TK", +"PAD_TK", +"ABSTRACT_TK", +"STRICT_TK", +"DECR_TK", +"INCR_TK", +"DEFAULT_TK", +"IF_TK", +"THROW_TK", +"BOOLEAN_TK", +"DO_TK", +"IMPLEMENTS_TK", +"THROWS_TK", +"BREAK_TK", +//"IMPORT_TK", +"ELSE_TK", +"INSTANCEOF_TK", +"RETURN_TK", +"VOID_TK", +"CATCH_TK", +"INTERFACE_TK", +"CASE_TK", +"EXTENDS_TK", +"FINALLY_TK", +"SUPER_TK", +"WHILE_TK", +"CLASS_TK", +"SWITCH_TK", +"CONST_TK", +"TRY_TK", +"FOR_TK", +"NEW_TK", +"CONTINUE_TK", +"PACKAGE_TK", +"THIS_TK", +"ASSERT_TK", +"BYTE_TK", +"SHORT_TK", +"INT_TK", +"LONG_TK", +"CHAR_TK", +"FLOAT_TK", +"DOUBLE_TK", +"ID_TK", +"REL_QM_TK", +"REL_CL_TK", +"NOT_TK", +"NEG_TK", +"ASSIGN_TK", +"OSB_TK", +"CSB_TK", +"DOT_TK", +"STRING_LIT_TK", +"CHAR_LIT_TK", +"INT_LIT_TK", +"FP_LIT_TK", +"BOOL_LIT_TK", +"NULL_TK", +NULL diff --git a/src/ptgen/java/main.cc b/src/ptgen/java/main.cc new file mode 100755 index 0000000..967c97f --- /dev/null +++ b/src/ptgen/java/main.cc @@ -0,0 +1,66 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include + +using namespace std; + +map name2id; +map id2name; + +int yyparse(); + +extern Tree *root; + +void id_init(); + +int main() +{ + id_init(); + yyparse(); + if (!root) { + cerr << "failed to parse file" << endl; + return 1; + } + + root->printTok(); + //root->print(); + +// int c= root->countTerminals(); + +// cout << c << endl; + + return 0; +} + diff --git a/src/ptgen/java/mainj.py b/src/ptgen/java/mainj.py new file mode 100644 index 0000000..b03cd7f --- /dev/null +++ b/src/ptgen/java/mainj.py @@ -0,0 +1,187 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Given a normal yacc .y file, simply split it into three files for our yacc.g to work: *.y (which contains and only contains all the rules), *.y.head, *.y.foot + +# Pi: To make the parse reentrant. http://www.gnu.org/software/bison/manual/html_mono/bison.html#Pure-Decl + +import sys +sys.path.append('..') + +import YaccParser,YaccLexer + +if len(sys.argv) != 2: + print >> sys.stderr, "usage %s grammar"%sys.argv[0] + sys.exit(1) + + +grammar= open(sys.argv[1],'r') +y= YaccParser.Parser(YaccLexer.Lexer(grammar)) +y.grammar() +grammar.close() + + +outg= open('pt_'+sys.argv[1],'w') + +print >> outg, """ +%pure-parser + +%{ +#include + +using namespace std; +%} + +%union{ +Tree *t; +} + +%{ +void yyerror(char*s); +int yylex(YYSTYPE *yylvalp); + +Tree *root; +%} + +""" + +for nt in y.NonTerminals: + print >> outg, '%type ', nt + +for t in y.Terminals: + print >> outg, '%type ', t + + + +header= open(sys.argv[1]+'.head','r') +outg.write( header.read() ) +header.close() + +print >> outg, """ + +%% + +""" + +name2id= {} +current= 0 +for nt in y.NonTerminals: + name2id[nt]=current + current+=1 +for t in y.Terminals: + name2id[t]=current + current+=1 + +#outh= open('tokid.h','w') +#for nt in y.NonTerminals: +# print >> outh, '#define ID_'+nt, name2id[nt] +#outh.close() + + +for rule in y.Rules: + print >> outg, rule[0], ':', + for production in rule[1]: + print >> outg, production[1], + """ + leave= False + for production in rule[1]: + if production[0]=="error": + leave= True + break + if leave: + print >> outg, ';\n' + continue + """ + print >> outg, """ + { + $$= new NonTerminal(""", name2id[rule[0]], ");" + nodelist= [ production[0] for production \ + in zip(range(1,len(rule[1])+1),rule[1]) \ + if production[1][0]=="node"] + for i in range(len(nodelist)): + print >> outg, "\n $$->addChild($%d);"%nodelist[i] + print >> outg, "\n $%d->parent= $$;"%nodelist[i] + if i + 1 < len(nodelist): + print >> outg,\ + "\n $%d->nextSibbling= $%d;"%\ + (nodelist[i],nodelist[i+1]) + if rule[0]=="goal": + print >> outg, "root= $$;" + print >> outg, """ + } + ; +""" + +print >> outg, """ + +%% + +""" + +footer= open(sys.argv[1]+'.foot','r') +print >> outg, footer.read() +footer.close() + + +outg.close() + + +head = open('head.cc','w') + +print >> head, """ +#include +#include + +using namespace std; + +extern map name2id; +extern map id2name; + +void id_init() +{ +""" +for nt in y.NonTerminals: + id= name2id[nt] + print >> head, 'name2id["%s"]= %d;'%(nt,id) + print >> head, 'id2name[%d]= "%s";'%(id,nt) +for t in y.Terminals: + id= name2id[t] + print >> head, 'name2id["%s"]= %d;'%(t,id) + print >> head, 'id2name[%d]= "%s";'%(id,t) +print >> head, '}' +print >> head + +head.close() + diff --git a/src/ptgen/php5/Makefile b/src/ptgen/php5/Makefile new file mode 100755 index 0000000..77396da --- /dev/null +++ b/src/ptgen/php5/Makefile @@ -0,0 +1,61 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +CXX= g++ -I../../include +OBJS= lex.yy.o pt_zend_language_parser.tab.o head.o +#TARGET=php_ptgen # this is just a test driver +TARGET=phpptgen.a + +all: $(TARGET) + +phpptgen.a: $(OBJS) + ar -qcs $@ $(OBJS) + +php_ptgen: $(OBJS) main.o + $(CXX) -o $@ $(OBJS) main.o + +#${TARGET}: ${OBJS} main.o +# $(CXX) -o ${TARGET} ${OBJS} main.o + +pt_zend_language_parser.tab.cc: pt_zend_language_parser.y + bison -d pt_zend_language_parser.y -o pt_zend_language_parser.tab.cc + +lex.yy.cc: zend_language_scanner.l pt_zend_language_parser.tab.cc + flex -i -olex.yy.cc zend_language_scanner.l + +head.cc pt_zend_language_parser.y: zend_language_parser.y zend_language_parser.y.head zend_language_parser.y.foot + ./mainphp.py zend_language_parser.y + sed -i "s/'\"'/'\\\\\"'/" head.cc + +.PHONY: clean +clean: + rm -f *.o lex.yy.cc pt_zend_language_parser.tab* pt_zend_language_parser.y head.cc $(TARGET) diff --git a/src/ptgen/php5/mainphp.py b/src/ptgen/php5/mainphp.py new file mode 100644 index 0000000..ab167fc --- /dev/null +++ b/src/ptgen/php5/mainphp.py @@ -0,0 +1,187 @@ +#! /usr/bin/env python + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Given a normal yacc .y file, simply split it into three files for our yacc.g to work: *.y (which contains and only contains all the rules), *.y.head, *.y.foot + +# Pi: To make the parse reentrant. http://www.gnu.org/software/bison/manual/html_mono/bison.html#Pure-Decl + +import sys +sys.path.append('..') + +import YaccParser,YaccLexer + +if len(sys.argv) != 2: + print >> sys.stderr, "usage %s grammar"%sys.argv[0] + sys.exit(1) + + +grammar= open(sys.argv[1],'r') +y= YaccParser.Parser(YaccLexer.Lexer(grammar)) +y.grammar() +grammar.close() + + +outg= open('pt_'+sys.argv[1],'w') + +print >> outg, """ +%pure-parser + +%{ +#include + +using namespace std; +%} + +%union{ +Tree *t; +} + +%{ +void yyerror(char*s); +int yylex(YYSTYPE *yylvalp); + +Tree *root; +%} + +""" + +for nt in y.NonTerminals: + print >> outg, '%type ', nt + +for t in y.Terminals: + print >> outg, '%type ', t + + + +header= open(sys.argv[1]+'.head','r') +outg.write( header.read() ) +header.close() + +print >> outg, """ + +%% + +""" + +name2id= {} +current= 0 +for nt in y.NonTerminals: + name2id[nt]=current + current+=1 +for t in y.Terminals: + name2id[t]=current + current+=1 + +#outh= open('tokid.h','w') +#for nt in y.NonTerminals: +# print >> outh, '#define ID_'+nt, name2id[nt] +#outh.close() + + +for rule in y.Rules: + print >> outg, rule[0], ':', + for production in rule[1]: + print >> outg, production[1], + """ + leave= False + for production in rule[1]: + if production[0]=="error": + leave= True + break + if leave: + print >> outg, ';\n' + continue + """ + print >> outg, """ + { + $$= new NonTerminal(""", name2id[rule[0]], ");" + nodelist= [ production[0] for production \ + in zip(range(1,len(rule[1])+1),rule[1]) \ + if production[1][0]=="node"] + for i in range(len(nodelist)): + print >> outg, "\n $$->addChild($%d);"%nodelist[i] + print >> outg, "\n $%d->parent= $$;"%nodelist[i] + if i + 1 < len(nodelist): + print >> outg,\ + "\n $%d->nextSibbling= $%d;"%\ + (nodelist[i],nodelist[i+1]) + if rule[0]=="start": + print >> outg, "root= $$;" + print >> outg, """ + } + ; +""" + +print >> outg, """ + +%% + +""" + +footer= open(sys.argv[1]+'.foot','r') +print >> outg, footer.read() +footer.close() + + +outg.close() + + +head = open('head.cc','w') + +print >> head, """ +#include +#include + +using namespace std; + +extern map name2id; +extern map id2name; + +void id_init() +{ +""" +for nt in y.NonTerminals: + id= name2id[nt] + print >> head, 'name2id["%s"]= %d;'%(nt,id) + print >> head, 'id2name[%d]= "%s";'%(id,nt) +for t in y.Terminals: + id= name2id[t] + print >> head, 'name2id["%s"]= %d;'%(t,id) + print >> head, 'id2name[%d]= "%s";'%(id,t) +print >> head, '}' +print >> head + +head.close() + diff --git a/src/ptgen/php5/phpatomicNodes.h b/src/ptgen/php5/phpatomicNodes.h new file mode 100755 index 0000000..e951fdb --- /dev/null +++ b/src/ptgen/php5/phpatomicNodes.h @@ -0,0 +1,97 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"expr", +"r_variable", +"variable", +"base_variable_with_function_calls", +"object_property", +"method_or_not", +"variable_properties", +"function_call_parameter_list", +"non_empty_function_call_parameter_list", +"expr_without_variable", +"assignment_list", +"class_name_reference", +"dynamic_class_name_reference", +"base_variable", +"reference_variable", +"dim_offset", +"compound_variable", +"simple_indirect_reference", +"dynamic_class_name_variable_properties", +"static_member", +"object_dim_list", +"variable_without_objects", +"ctor_arguments", +"rw_variable", +"w_variable", +"internal_functions_in_yacc", +"exit_expr", +"scalar", +"class_constant", +"array_pair_list", +"non_empty_array_pair_list", +"encaps_list", +"encaps_var", +"function_call", +"for_expr", +"non_empty_for_expr", +"global_var_list", +"static_var_list", +"static_scalar", +"static_class_constant", +"echo_expr_list", +"use_filename", +"unset_variables", +"foreach_variable", +"foreach_optional_arg", +"declare_list", +"parameter_list", +"non_empty_parameter_list", +"assignment_list_element", +"class_variable_declaration", +"method_modifiers", +"member_modifier", +"static_array_pair_list", +"non_empty_static_array_pair_list", +"class_entry_type", +"extends_from", +"implements_list", +"interface_extends_list", +"interface_list", +"variable_modifiers", +"non_empty_member_modifiers", +"class_constant_declaration", +"isset_variables", +"variable_name", +NULL diff --git a/src/ptgen/php5/phpcontextualNodes.h b/src/ptgen/php5/phpcontextualNodes.h new file mode 100755 index 0000000..102a754 --- /dev/null +++ b/src/ptgen/php5/phpcontextualNodes.h @@ -0,0 +1,45 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"start", +"unticked_statement", //<--partially...need special handlers... +"unticked_function_declaration_statement", +//"T_FUNCTION", +"unticked_class_declaration_statement", +//"class_entry_type", "interface_entry", +//"method_body", +//"for_statement", "foreach_statement", "declare_statement", "switch_case_list", "while_statement", +"elseif_list", "new_elseif_list", +//"else_single", "new_else_single", +"additional_catch", +//"T_IF", "T_WHILE", "T_DO", "T_FOR", "T_SWITCH", "T_FOREACH", "T_TRY", "T_CATCH", +NULL diff --git a/src/ptgen/php5/phpparentNodes.h b/src/ptgen/php5/phpparentNodes.h new file mode 100755 index 0000000..ec0a957 --- /dev/null +++ b/src/ptgen/php5/phpparentNodes.h @@ -0,0 +1,40 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +"top_statement", "inner_statement", +//"statement", "function_declaration_statement", "class_declaration_statement", "T_HALT_COMPILER", +//"unticked_statement", +"elseif_list", "else_single", "new_elseif_list", "new_else_single", +"switch_case_list", "case_list", +"while_statement", "for_statement", "foreach_statement", "declare_statement", "additional_catch", +"class_statement", +NULL diff --git a/src/ptgen/php5/phprelevantNodes.h b/src/ptgen/php5/phprelevantNodes.h new file mode 100755 index 0000000..a1cd16d --- /dev/null +++ b/src/ptgen/php5/phprelevantNodes.h @@ -0,0 +1,242 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +//"top_statement", +//"inner_statement", +//"expr", +//"r_variable", +"expr_without_variable", +"assignment_list_element", +"variable", +//"base_variable_with_function_calls", +"function_call", +"method_or_not", +"function_call_parameter_list", +"variable_property", +"class_name_reference", +"ctor_arguments", +"dynamic_class_name_reference", +"base_variable", +"static_member", +"object_property", +"dynamic_class_name_variable_property", +"object_dim_list", +"variable_name", +"variable_without_objects", +"reference_variable", +"dim_offset", +"compound_variable", +"simple_indirect_reference", +//"rw_variable", +"scalar", +"class_constant", +"array_pair_list", +//"w_variable", +//"encaps_list", +"encaps_var", +"encaps_var_offset", +"internal_functions_in_yacc", +"exit_expr", +//"statement", +"unticked_statement", +"global_var", +"use_filename", +//"unset_variable", +"foreach_variable", +"foreach_optional_arg", +"foreach_statement", +//"declare_list", +"declare_statement", +"additional_catches", +"additional_catch", +"elseif_list", +"else_single", +"new_elseif_list", +"new_else_single", +"while_statement", +"for_expr", +"for_statement", +"switch_case_list", +"case_list", +"case_separator", +"unticked_function_declaration_statement", +"unticked_class_declaration_statement", +"class_entry_type", +"extends_from", +//"implements_list", +"class_statement", +//"class_variable_declaration", +//"class_constant_declaration", +"member_modifier", +"method_body", +//"interface_entry", +//"interface_extends_list", +"fully_qualified_class_name", +//"is_reference", +"static_scalar", +"common_scalar", +"static_class_constant", +"optional_class_type", +"T_ENDIF", +"':'", +"T_IF", +"T_CLASS", +"T_IMPLEMENTS", +"T_VAR", +"T_CONST", +"T_FINAL", +"T_ABSTRACT", +"T_STATIC", +"T_PRIVATE", +"T_PROTECTED", +"T_PUBLIC", +"T_INTERFACE", +"T_EXTENDS", +"T_FUNC_C", +"T_METHOD_C", +"T_CLASS_C", +"T_FILE", +"T_LINE", +"T_CONSTANT_ENCAPSED_STRING", +"T_DNUMBER", +"T_LNUMBER", +"T_DOUBLE_ARROW", +"T_PAAMAYIM_NEKUDOTAYIM", +"T_FUNCTION", +"T_STRING", +"T_VARIABLE", +"'&'", +"'='", +"'-'", +"'+'", +"T_ARRAY", +"T_HALT_COMPILER", +"T_DEC", +"T_OBJECT_CAST", +"T_ENDFOREACH", +"'~'", +//"T_BAD_CHARACTER", +"T_DIV_EQUAL", +"T_THROW", +"T_INCLUDE_ONCE", +"T_MUL_EQUAL", +"T_ELSEIF", +"T_SL", +"T_IS_EQUAL", +"T_IS_NOT_IDENTICAL", +"T_CONTINUE", +"T_ARRAY_CAST", +"T_SR", +"T_STRING_CAST", +"'.'", +"T_REQUIRE_ONCE", +"T_INCLUDE", +//"T_CHARACTER", +"T_ELSE", +//"T_CURLY_OPEN", +"T_INT_CAST", +"T_IS_GREATER_OR_EQUAL", +"T_LOGICAL_XOR", +"T_EMPTY", +"T_BOOLEAN_OR", +"T_DECLARE", +"T_BREAK", +"'%'", +"'`'", +//"T_ENCAPSED_AND_WHITESPACE", +"T_LOGICAL_AND", +"T_ENDWHILE", +"T_DEFAULT", +"'*'", +"T_DOUBLE_CAST", +"T_ISSET", +"T_DOLLAR_OPEN_CURLY_BRACES", +"T_LOGICAL_OR", +"T_AND_EQUAL", +"'$'", +"T_BOOLEAN_AND", +"T_ECHO", +"T_SL_EQUAL", +"T_CATCH", +"T_PLUS_EQUAL", +"T_INSTANCEOF", +"T_OR_EQUAL", +"T_INLINE_HTML", +"T_GLOBAL", +"'<'", +"T_INC", +"T_UNSET_CAST", +"T_SR_EQUAL", +"T_ENDDECLARE", +"T_MINUS_EQUAL", +"T_IS_IDENTICAL", +"T_TRY", +"T_IS_NOT_EQUAL", +"T_CASE", +"'?'", +"'@'", +"T_CONCAT_EQUAL", +"'>'", +"T_START_HEREDOC", +"T_DO", +"T_CLONE", +"T_RETURN", +"T_ENDFOR", +"'/'", +"T_PRINT", +"'['", +"T_EVAL", +"T_XOR_EQUAL", +"T_FOREACH", +"T_NUM_STRING", +"T_SWITCH", +"T_BOOL_CAST", +"T_STRING_VARNAME", +"T_UNSET", +"T_ENDSWITCH", +"T_FOR", +"'!'", +"']'", +"T_EXIT", +"T_WHILE", +"T_END_HEREDOC", +"'|'", +"T_NEW", +"T_REQUIRE", +"T_OBJECT_OPERATOR", +"'^'", +"T_MOD_EQUAL", +"T_IS_SMALLER_OR_EQUAL", +"T_AS", +"T_LIST", +"T_USE", +NULL diff --git a/src/ptgen/php5/zend_language_parser.y b/src/ptgen/php5/zend_language_parser.y new file mode 100755 index 0000000..351c272 --- /dev/null +++ b/src/ptgen/php5/zend_language_parser.y @@ -0,0 +1,778 @@ +start: + top_statement_list +; + +top_statement_list: + top_statement_list { zend_do_extended_info(TSRMLS_C); } top_statement { HANDLE_INTERACTIVE(); } + | /* empty */ +; + + +top_statement: + statement + | function_declaration_statement { zend_do_early_binding(TSRMLS_C); } + | class_declaration_statement { zend_do_early_binding(TSRMLS_C); } + | T_HALT_COMPILER '(' ')' stmt_end { zval c; if (zend_get_constant("__COMPILER_HALT_OFFSET__", sizeof("__COMPILER_HALT_OFFSET__") - 1, &c TSRMLS_CC)) { zval_dtor(&c); zend_error(E_COMPILE_ERROR, "__HALT_COMPILER() can only be used once per request"); } else { REGISTER_MAIN_LONG_CONSTANT("__COMPILER_HALT_OFFSET__", zend_get_scanned_file_offset(TSRMLS_C), CONST_CS); } YYACCEPT; } +; + + +inner_statement_list: + inner_statement_list { zend_do_extended_info(TSRMLS_C); } inner_statement { HANDLE_INTERACTIVE(); } + | /* empty */ +; + + +inner_statement: + statement + | function_declaration_statement + | class_declaration_statement + | T_HALT_COMPILER '(' ')' stmt_end { zend_error(E_COMPILE_ERROR, "__HALT_COMPILER() can only be used from the outermost scope"); } +; + + +stmt_end: + T_CLOSE_TAG + | ';' +; + +statement: + unticked_statement { zend_do_ticks(TSRMLS_C); } +; + +unticked_statement: + '{' inner_statement_list '}' + | T_IF '(' expr ')' { zend_do_if_cond(&$3, &$4 TSRMLS_CC); } statement { zend_do_if_after_statement(&$4, 1 TSRMLS_CC); } elseif_list else_single { zend_do_if_end(TSRMLS_C); } + | T_IF '(' expr ')' ':' { zend_do_if_cond(&$3, &$4 TSRMLS_CC); } inner_statement_list { zend_do_if_after_statement(&$4, 1 TSRMLS_CC); } new_elseif_list new_else_single T_ENDIF stmt_end { zend_do_if_end(TSRMLS_C); } + | T_WHILE '(' { $1.u.opline_num = get_next_op_number(CG(active_op_array)); } expr ')' { zend_do_while_cond(&$4, &$5 TSRMLS_CC); } while_statement { zend_do_while_end(&$1, &$5 TSRMLS_CC); } + | T_DO { $1.u.opline_num = get_next_op_number(CG(active_op_array)); zend_do_do_while_begin(TSRMLS_C); } statement T_WHILE '(' { $5.u.opline_num = get_next_op_number(CG(active_op_array)); } expr ')' stmt_end { zend_do_do_while_end(&$1, &$5, &$7 TSRMLS_CC); } + | T_FOR + '(' + for_expr + ';' { zend_do_free(&$3 TSRMLS_CC); $4.u.opline_num = get_next_op_number(CG(active_op_array)); } + for_expr + ';' { zend_do_extended_info(TSRMLS_C); zend_do_for_cond(&$6, &$7 TSRMLS_CC); } + for_expr + ')' { zend_do_free(&$9 TSRMLS_CC); zend_do_for_before_statement(&$4, &$7 TSRMLS_CC); } + for_statement { zend_do_for_end(&$7 TSRMLS_CC); } + | T_SWITCH '(' expr ')' { zend_do_switch_cond(&$3 TSRMLS_CC); } switch_case_list { zend_do_switch_end(&$6 TSRMLS_CC); } + | T_BREAK stmt_end { zend_do_brk_cont(ZEND_BRK, NULL TSRMLS_CC); } + | T_BREAK expr stmt_end { zend_do_brk_cont(ZEND_BRK, &$2 TSRMLS_CC); } + | T_CONTINUE stmt_end { zend_do_brk_cont(ZEND_CONT, NULL TSRMLS_CC); } + | T_CONTINUE expr stmt_end { zend_do_brk_cont(ZEND_CONT, &$2 TSRMLS_CC); } + | T_RETURN stmt_end { zend_do_return(NULL, 0 TSRMLS_CC); } + | T_RETURN expr_without_variable stmt_end { zend_do_return(&$2, 0 TSRMLS_CC); } + | T_RETURN variable stmt_end { zend_do_return(&$2, 1 TSRMLS_CC); } + | T_GLOBAL global_var_list stmt_end + | T_STATIC static_var_list stmt_end + | T_ECHO echo_expr_list stmt_end + | T_INLINE_HTML { zend_do_echo(&$1 TSRMLS_CC); } + | expr stmt_end { zend_do_free(&$1 TSRMLS_CC); } + | T_USE use_filename stmt_end { zend_error(E_COMPILE_ERROR,"use: Not yet supported. Please use include_once() or require_once()"); zval_dtor(&$2.u.constant); } + | T_UNSET '(' unset_variables ')' stmt_end + | T_FOREACH '(' variable T_AS + { zend_do_foreach_begin(&$1, &$2, &$3, &$4, 1 TSRMLS_CC); } + foreach_variable foreach_optional_arg ')' { zend_do_foreach_cont(&$1, &$2, &$4, &$6, &$7 TSRMLS_CC); } + foreach_statement { zend_do_foreach_end(&$1, &$4 TSRMLS_CC); } + | T_FOREACH '(' expr_without_variable T_AS + { zend_do_foreach_begin(&$1, &$2, &$3, &$4, 0 TSRMLS_CC); } + variable foreach_optional_arg ')' { zend_check_writable_variable(&$6); zend_do_foreach_cont(&$1, &$2, &$4, &$6, &$7 TSRMLS_CC); } + foreach_statement { zend_do_foreach_end(&$1, &$4 TSRMLS_CC); } + | T_DECLARE { $1.u.opline_num = get_next_op_number(CG(active_op_array)); zend_do_declare_begin(TSRMLS_C); } '(' declare_list ')' declare_statement { zend_do_declare_end(&$1 TSRMLS_CC); } + | stmt_end /* empty statement */ + | T_TRY { zend_do_try(&$1 TSRMLS_CC); } '{' inner_statement_list '}' + T_CATCH '(' { zend_initialize_try_catch_element(&$1 TSRMLS_CC); } + fully_qualified_class_name { zend_do_first_catch(&$7 TSRMLS_CC); } + T_VARIABLE ')' { zend_do_begin_catch(&$1, &$9, &$11, 1 TSRMLS_CC); } + '{' inner_statement_list '}' { zend_do_end_catch(&$1 TSRMLS_CC); } + additional_catches { zend_do_mark_last_catch(&$7, &$18 TSRMLS_CC); } + | T_THROW expr stmt_end { zend_do_throw(&$2 TSRMLS_CC); } +; + + +additional_catches: + non_empty_additional_catches { $$ = $1; } + | /* empty */ { $$.u.opline_num = -1; } +; + +non_empty_additional_catches: + additional_catch { $$ = $1; } + | non_empty_additional_catches additional_catch { $$ = $2; } +; + + +additional_catch: + T_CATCH '(' fully_qualified_class_name { $$.u.opline_num = get_next_op_number(CG(active_op_array)); } T_VARIABLE ')' { zend_do_begin_catch(&$1, &$3, &$5, 0 TSRMLS_CC); } '{' inner_statement_list '}' { zend_do_end_catch(&$1 TSRMLS_CC); } +; + + +unset_variables: + unset_variable + | unset_variables ',' unset_variable +; + +unset_variable: + variable { zend_do_end_variable_parse(BP_VAR_UNSET, 0 TSRMLS_CC); zend_do_unset(&$1 TSRMLS_CC); } +; + +use_filename: + T_CONSTANT_ENCAPSED_STRING { $$ = $1; } + | '(' T_CONSTANT_ENCAPSED_STRING ')' { $$ = $2; } +; + + +function_declaration_statement: + unticked_function_declaration_statement { zend_do_ticks(TSRMLS_C); } +; + +class_declaration_statement: + unticked_class_declaration_statement { zend_do_ticks(TSRMLS_C); } +; + + +is_reference: + /* empty */ { $$.op_type = ZEND_RETURN_VAL; } + | '&' { $$.op_type = ZEND_RETURN_REF; } +; + + +unticked_function_declaration_statement: + T_FUNCTION { $1.u.opline_num = CG(zend_lineno); } is_reference T_STRING { zend_do_begin_function_declaration(&$1, &$4, 0, $3.op_type, NULL TSRMLS_CC); } + '(' parameter_list ')' '{' inner_statement_list '}' { zend_do_end_function_declaration(&$1 TSRMLS_CC); } +; + +unticked_class_declaration_statement: + class_entry_type T_STRING extends_from + { zend_do_begin_class_declaration(&$1, &$2, &$3 TSRMLS_CC); } + implements_list + '{' + class_statement_list + '}' { zend_do_end_class_declaration(&$1, &$2 TSRMLS_CC); } + | interface_entry T_STRING + { zend_do_begin_class_declaration(&$1, &$2, NULL TSRMLS_CC); } + interface_extends_list + '{' + class_statement_list + '}' { zend_do_end_class_declaration(&$1, &$2 TSRMLS_CC); } +; + + +class_entry_type: + T_CLASS { $$.u.opline_num = CG(zend_lineno); $$.u.EA.type = 0; } + | T_ABSTRACT T_CLASS { $$.u.opline_num = CG(zend_lineno); $$.u.EA.type = ZEND_ACC_EXPLICIT_ABSTRACT_CLASS; } + | T_FINAL T_CLASS { $$.u.opline_num = CG(zend_lineno); $$.u.EA.type = ZEND_ACC_FINAL_CLASS; } +; + +extends_from: + /* empty */ { $$.op_type = IS_UNUSED; } + | T_EXTENDS fully_qualified_class_name { $$ = $2; } +; + +interface_entry: + T_INTERFACE { $$.u.opline_num = CG(zend_lineno); $$.u.EA.type = ZEND_ACC_INTERFACE; } +; + +interface_extends_list: + /* empty */ + | T_EXTENDS interface_list +; + +implements_list: + /* empty */ + | T_IMPLEMENTS interface_list +; + +interface_list: + fully_qualified_class_name { zend_do_implements_interface(&$1 TSRMLS_CC); } + | interface_list ',' fully_qualified_class_name { zend_do_implements_interface(&$3 TSRMLS_CC); } +; + +foreach_optional_arg: + /* empty */ { $$.op_type = IS_UNUSED; } + | T_DOUBLE_ARROW foreach_variable { $$ = $2; } +; + + +foreach_variable: + variable { zend_check_writable_variable(&$1); $$ = $1; } + | '&' variable { zend_check_writable_variable(&$2); $$ = $2; $$.u.EA.type |= ZEND_PARSED_REFERENCE_VARIABLE; } +; + +for_statement: + statement + | ':' inner_statement_list T_ENDFOR stmt_end +; + + +foreach_statement: + statement + | ':' inner_statement_list T_ENDFOREACH stmt_end +; + + +declare_statement: + statement + | ':' inner_statement_list T_ENDDECLARE stmt_end +; + + +declare_list: + T_STRING '=' static_scalar { zend_do_declare_stmt(&$1, &$3 TSRMLS_CC); } + | declare_list ',' T_STRING '=' static_scalar { zend_do_declare_stmt(&$3, &$5 TSRMLS_CC); } +; + + +switch_case_list: + '{' case_list '}' { $$ = $2; } + | '{' ';' case_list '}' { $$ = $3; } + | ':' case_list T_ENDSWITCH stmt_end { $$ = $2; } + | ':' ';' case_list T_ENDSWITCH stmt_end { $$ = $3; } +; + + +case_list: + /* empty */ { $$.op_type = IS_UNUSED; } + | case_list T_CASE expr case_separator { zend_do_extended_info(TSRMLS_C); zend_do_case_before_statement(&$1, &$2, &$3 TSRMLS_CC); } inner_statement_list { zend_do_case_after_statement(&$$, &$2 TSRMLS_CC); $$.op_type = IS_CONST; } + | case_list T_DEFAULT case_separator { zend_do_extended_info(TSRMLS_C); zend_do_default_before_statement(&$1, &$2 TSRMLS_CC); } inner_statement_list { zend_do_case_after_statement(&$$, &$2 TSRMLS_CC); $$.op_type = IS_CONST; } +; + + +case_separator: + ':' + | ';' +; + + +while_statement: + statement + | ':' inner_statement_list T_ENDWHILE stmt_end +; + + + +elseif_list: + /* empty */ + | elseif_list T_ELSEIF '(' expr ')' { zend_do_if_cond(&$4, &$5 TSRMLS_CC); } statement { zend_do_if_after_statement(&$5, 0 TSRMLS_CC); } +; + + +new_elseif_list: + /* empty */ + | new_elseif_list T_ELSEIF '(' expr ')' ':' { zend_do_if_cond(&$4, &$5 TSRMLS_CC); } inner_statement_list { zend_do_if_after_statement(&$5, 0 TSRMLS_CC); } +; + + +else_single: + /* empty */ + | T_ELSE statement +; + + +new_else_single: + /* empty */ + | T_ELSE ':' inner_statement_list +; + + +parameter_list: + non_empty_parameter_list + | /* empty */ +; + + +non_empty_parameter_list: + optional_class_type T_VARIABLE { znode tmp; fetch_simple_variable(&tmp, &$2, 0 TSRMLS_CC); $$.op_type = IS_CONST; Z_LVAL($$.u.constant)=1; Z_TYPE($$.u.constant)=IS_LONG; INIT_PZVAL(&$$.u.constant); zend_do_receive_arg(ZEND_RECV, &tmp, &$$, NULL, &$1, &$2, 0 TSRMLS_CC); } + | optional_class_type '&' T_VARIABLE { znode tmp; fetch_simple_variable(&tmp, &$3, 0 TSRMLS_CC); $$.op_type = IS_CONST; Z_LVAL($$.u.constant)=1; Z_TYPE($$.u.constant)=IS_LONG; INIT_PZVAL(&$$.u.constant); zend_do_receive_arg(ZEND_RECV, &tmp, &$$, NULL, &$1, &$3, 1 TSRMLS_CC); } + | optional_class_type '&' T_VARIABLE '=' static_scalar { znode tmp; fetch_simple_variable(&tmp, &$3, 0 TSRMLS_CC); $$.op_type = IS_CONST; Z_LVAL($$.u.constant)=1; Z_TYPE($$.u.constant)=IS_LONG; INIT_PZVAL(&$$.u.constant); zend_do_receive_arg(ZEND_RECV_INIT, &tmp, &$$, &$5, &$1, &$3, 1 TSRMLS_CC); } + | optional_class_type T_VARIABLE '=' static_scalar { znode tmp; fetch_simple_variable(&tmp, &$2, 0 TSRMLS_CC); $$.op_type = IS_CONST; Z_LVAL($$.u.constant)=1; Z_TYPE($$.u.constant)=IS_LONG; INIT_PZVAL(&$$.u.constant); zend_do_receive_arg(ZEND_RECV_INIT, &tmp, &$$, &$4, &$1, &$2, 0 TSRMLS_CC); } + | non_empty_parameter_list ',' optional_class_type T_VARIABLE { znode tmp; fetch_simple_variable(&tmp, &$4, 0 TSRMLS_CC); $$=$1; Z_LVAL($$.u.constant)++; zend_do_receive_arg(ZEND_RECV, &tmp, &$$, NULL, &$3, &$4, 0 TSRMLS_CC); } + | non_empty_parameter_list ',' optional_class_type '&' T_VARIABLE { znode tmp; fetch_simple_variable(&tmp, &$5, 0 TSRMLS_CC); $$=$1; Z_LVAL($$.u.constant)++; zend_do_receive_arg(ZEND_RECV, &tmp, &$$, NULL, &$3, &$5, 1 TSRMLS_CC); } + | non_empty_parameter_list ',' optional_class_type '&' T_VARIABLE '=' static_scalar { znode tmp; fetch_simple_variable(&tmp, &$5, 0 TSRMLS_CC); $$=$1; Z_LVAL($$.u.constant)++; zend_do_receive_arg(ZEND_RECV_INIT, &tmp, &$$, &$7, &$3, &$5, 1 TSRMLS_CC); } + | non_empty_parameter_list ',' optional_class_type T_VARIABLE '=' static_scalar { znode tmp; fetch_simple_variable(&tmp, &$4, 0 TSRMLS_CC); $$=$1; Z_LVAL($$.u.constant)++; zend_do_receive_arg(ZEND_RECV_INIT, &tmp, &$$, &$6, &$3, &$4, 0 TSRMLS_CC); } +; + + +optional_class_type: + /* empty */ { $$.op_type = IS_UNUSED; } + | T_STRING { $$ = $1; } + | T_ARRAY { $$.op_type = IS_CONST; Z_TYPE($$.u.constant)=IS_NULL;} +; + + +function_call_parameter_list: + non_empty_function_call_parameter_list { $$ = $1; } + | /* empty */ { Z_LVAL($$.u.constant) = 0; } +; + + +non_empty_function_call_parameter_list: + expr_without_variable { Z_LVAL($$.u.constant) = 1; zend_do_pass_param(&$1, ZEND_SEND_VAL, Z_LVAL($$.u.constant) TSRMLS_CC); } + | variable { Z_LVAL($$.u.constant) = 1; zend_do_pass_param(&$1, ZEND_SEND_VAR, Z_LVAL($$.u.constant) TSRMLS_CC); } + | '&' w_variable { Z_LVAL($$.u.constant) = 1; zend_do_pass_param(&$2, ZEND_SEND_REF, Z_LVAL($$.u.constant) TSRMLS_CC); } + | non_empty_function_call_parameter_list ',' expr_without_variable { Z_LVAL($$.u.constant)=Z_LVAL($1.u.constant)+1; zend_do_pass_param(&$3, ZEND_SEND_VAL, Z_LVAL($$.u.constant) TSRMLS_CC); } + | non_empty_function_call_parameter_list ',' variable { Z_LVAL($$.u.constant)=Z_LVAL($1.u.constant)+1; zend_do_pass_param(&$3, ZEND_SEND_VAR, Z_LVAL($$.u.constant) TSRMLS_CC); } + | non_empty_function_call_parameter_list ',' '&' w_variable { Z_LVAL($$.u.constant)=Z_LVAL($1.u.constant)+1; zend_do_pass_param(&$4, ZEND_SEND_REF, Z_LVAL($$.u.constant) TSRMLS_CC); } +; + +global_var_list: + global_var_list ',' global_var { zend_do_fetch_global_variable(&$3, NULL, ZEND_FETCH_GLOBAL_LOCK TSRMLS_CC); } + | global_var { zend_do_fetch_global_variable(&$1, NULL, ZEND_FETCH_GLOBAL_LOCK TSRMLS_CC); } +; + + +global_var: + T_VARIABLE { $$ = $1; } + | '$' r_variable { $$ = $2; } + | '$' '{' expr '}' { $$ = $3; } +; + + +static_var_list: + static_var_list ',' T_VARIABLE { zend_do_fetch_static_variable(&$3, NULL, ZEND_FETCH_STATIC TSRMLS_CC); } + | static_var_list ',' T_VARIABLE '=' static_scalar { zend_do_fetch_static_variable(&$3, &$5, ZEND_FETCH_STATIC TSRMLS_CC); } + | T_VARIABLE { zend_do_fetch_static_variable(&$1, NULL, ZEND_FETCH_STATIC TSRMLS_CC); } + | T_VARIABLE '=' static_scalar { zend_do_fetch_static_variable(&$1, &$3, ZEND_FETCH_STATIC TSRMLS_CC); } + +; + + +class_statement_list: + class_statement_list class_statement + | /* empty */ +; + + +class_statement: + variable_modifiers { CG(access_type) = Z_LVAL($1.u.constant); } class_variable_declaration stmt_end + | class_constant_declaration stmt_end + | method_modifiers T_FUNCTION { $2.u.opline_num = CG(zend_lineno); } is_reference T_STRING { zend_do_begin_function_declaration(&$2, &$5, 1, $4.op_type, &$1 TSRMLS_CC); } '(' + parameter_list ')' method_body { zend_do_abstract_method(&$5, &$1, &$10 TSRMLS_CC); zend_do_end_function_declaration(&$2 TSRMLS_CC); } +; + + +method_body: + stmt_end /* abstract method */ { Z_LVAL($$.u.constant) = ZEND_ACC_ABSTRACT; } + | '{' inner_statement_list '}' { Z_LVAL($$.u.constant) = 0; } +; + +variable_modifiers: + non_empty_member_modifiers { $$ = $1; } + | T_VAR { Z_LVAL($$.u.constant) = ZEND_ACC_PUBLIC; } +; + +method_modifiers: + /* empty */ { Z_LVAL($$.u.constant) = ZEND_ACC_PUBLIC; } + | non_empty_member_modifiers { $$ = $1; if (!(Z_LVAL($$.u.constant) & ZEND_ACC_PPP_MASK)) { Z_LVAL($$.u.constant) |= ZEND_ACC_PUBLIC; } } +; + +non_empty_member_modifiers: + member_modifier { $$ = $1; } + | non_empty_member_modifiers member_modifier { Z_LVAL($$.u.constant) = zend_do_verify_access_types(&$1, &$2); } +; + +member_modifier: + T_PUBLIC { Z_LVAL($$.u.constant) = ZEND_ACC_PUBLIC; } + | T_PROTECTED { Z_LVAL($$.u.constant) = ZEND_ACC_PROTECTED; } + | T_PRIVATE { Z_LVAL($$.u.constant) = ZEND_ACC_PRIVATE; } + | T_STATIC { Z_LVAL($$.u.constant) = ZEND_ACC_STATIC; } + | T_ABSTRACT { Z_LVAL($$.u.constant) = ZEND_ACC_ABSTRACT; } + | T_FINAL { Z_LVAL($$.u.constant) = ZEND_ACC_FINAL; } +; + +class_variable_declaration: + class_variable_declaration ',' T_VARIABLE { zend_do_declare_property(&$3, NULL, CG(access_type) TSRMLS_CC); } + | class_variable_declaration ',' T_VARIABLE '=' static_scalar { zend_do_declare_property(&$3, &$5, CG(access_type) TSRMLS_CC); } + | T_VARIABLE { zend_do_declare_property(&$1, NULL, CG(access_type) TSRMLS_CC); } + | T_VARIABLE '=' static_scalar { zend_do_declare_property(&$1, &$3, CG(access_type) TSRMLS_CC); } +; + +class_constant_declaration: + class_constant_declaration ',' T_STRING '=' static_scalar { zend_do_declare_class_constant(&$3, &$5 TSRMLS_CC); } + | T_CONST T_STRING '=' static_scalar { zend_do_declare_class_constant(&$2, &$4 TSRMLS_CC); } +; + +echo_expr_list: + echo_expr_list ',' expr { zend_do_echo(&$3 TSRMLS_CC); } + | expr { zend_do_echo(&$1 TSRMLS_CC); } +; + + +for_expr: + /* empty */ { $$.op_type = IS_CONST; Z_TYPE($$.u.constant) = IS_BOOL; Z_LVAL($$.u.constant) = 1; } + | non_empty_for_expr { $$ = $1; } +; + +non_empty_for_expr: + non_empty_for_expr ',' { zend_do_free(&$1 TSRMLS_CC); } expr { $$ = $4; } + | expr { $$ = $1; } +; + +expr_without_variable: + T_LIST '(' { zend_do_list_init(TSRMLS_C); } assignment_list ')' '=' expr { zend_do_list_end(&$$, &$7 TSRMLS_CC); } + | variable '=' expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_W, 0 TSRMLS_CC); zend_do_assign(&$$, &$1, &$3 TSRMLS_CC); } + | variable '=' '&' variable { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_W, 0 TSRMLS_CC); zend_do_end_variable_parse(BP_VAR_W, 0 TSRMLS_CC); zend_do_assign_ref(&$$, &$1, &$4 TSRMLS_CC); } + | variable '=' '&' T_NEW class_name_reference { zend_error(E_STRICT, "Assigning the return value of new by reference is deprecated"); zend_check_writable_variable(&$1); zend_do_extended_fcall_begin(TSRMLS_C); zend_do_begin_new_object(&$4, &$5 TSRMLS_CC); } ctor_arguments { zend_do_end_new_object(&$3, &$4, &$7 TSRMLS_CC); zend_do_extended_fcall_end(TSRMLS_C); zend_do_end_variable_parse(BP_VAR_W, 0 TSRMLS_CC); zend_do_assign_ref(&$$, &$1, &$3 TSRMLS_CC); } + | T_NEW class_name_reference { zend_do_extended_fcall_begin(TSRMLS_C); zend_do_begin_new_object(&$1, &$2 TSRMLS_CC); } ctor_arguments { zend_do_end_new_object(&$$, &$1, &$4 TSRMLS_CC); zend_do_extended_fcall_end(TSRMLS_C);} + | T_CLONE expr { zend_do_clone(&$$, &$2 TSRMLS_CC); } + | variable T_PLUS_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_ADD, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_MINUS_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_SUB, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_MUL_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_MUL, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_DIV_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_DIV, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_CONCAT_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_CONCAT, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_MOD_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_MOD, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_AND_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_BW_AND, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_OR_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_BW_OR, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_XOR_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_BW_XOR, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_SL_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_SL, &$$, &$1, &$3 TSRMLS_CC); } + | variable T_SR_EQUAL expr { zend_check_writable_variable(&$1); zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); zend_do_binary_assign_op(ZEND_ASSIGN_SR, &$$, &$1, &$3 TSRMLS_CC); } + | rw_variable T_INC { zend_do_post_incdec(&$$, &$1, ZEND_POST_INC TSRMLS_CC); } + | T_INC rw_variable { zend_do_pre_incdec(&$$, &$2, ZEND_PRE_INC TSRMLS_CC); } + | rw_variable T_DEC { zend_do_post_incdec(&$$, &$1, ZEND_POST_DEC TSRMLS_CC); } + | T_DEC rw_variable { zend_do_pre_incdec(&$$, &$2, ZEND_PRE_DEC TSRMLS_CC); } + | expr T_BOOLEAN_OR { zend_do_boolean_or_begin(&$1, &$2 TSRMLS_CC); } expr { zend_do_boolean_or_end(&$$, &$1, &$4, &$2 TSRMLS_CC); } + | expr T_BOOLEAN_AND { zend_do_boolean_and_begin(&$1, &$2 TSRMLS_CC); } expr { zend_do_boolean_and_end(&$$, &$1, &$4, &$2 TSRMLS_CC); } + | expr T_LOGICAL_OR { zend_do_boolean_or_begin(&$1, &$2 TSRMLS_CC); } expr { zend_do_boolean_or_end(&$$, &$1, &$4, &$2 TSRMLS_CC); } + | expr T_LOGICAL_AND { zend_do_boolean_and_begin(&$1, &$2 TSRMLS_CC); } expr { zend_do_boolean_and_end(&$$, &$1, &$4, &$2 TSRMLS_CC); } + | expr T_LOGICAL_XOR expr { zend_do_binary_op(ZEND_BOOL_XOR, &$$, &$1, &$3 TSRMLS_CC); } + | expr '|' expr { zend_do_binary_op(ZEND_BW_OR, &$$, &$1, &$3 TSRMLS_CC); } + | expr '&' expr { zend_do_binary_op(ZEND_BW_AND, &$$, &$1, &$3 TSRMLS_CC); } + | expr '^' expr { zend_do_binary_op(ZEND_BW_XOR, &$$, &$1, &$3 TSRMLS_CC); } + | expr '.' expr { zend_do_binary_op(ZEND_CONCAT, &$$, &$1, &$3 TSRMLS_CC); } + | expr '+' expr { zend_do_binary_op(ZEND_ADD, &$$, &$1, &$3 TSRMLS_CC); } + | expr '-' expr { zend_do_binary_op(ZEND_SUB, &$$, &$1, &$3 TSRMLS_CC); } + | expr '*' expr { zend_do_binary_op(ZEND_MUL, &$$, &$1, &$3 TSRMLS_CC); } + | expr '/' expr { zend_do_binary_op(ZEND_DIV, &$$, &$1, &$3 TSRMLS_CC); } + | expr '%' expr { zend_do_binary_op(ZEND_MOD, &$$, &$1, &$3 TSRMLS_CC); } + | expr T_SL expr { zend_do_binary_op(ZEND_SL, &$$, &$1, &$3 TSRMLS_CC); } + | expr T_SR expr { zend_do_binary_op(ZEND_SR, &$$, &$1, &$3 TSRMLS_CC); } + | '+' expr { Z_LVAL($1.u.constant)=0; Z_TYPE($1.u.constant)=IS_LONG; $1.op_type = IS_CONST; INIT_PZVAL(&$1.u.constant); zend_do_binary_op(ZEND_ADD, &$$, &$1, &$2 TSRMLS_CC); } + | '-' expr { Z_LVAL($1.u.constant)=0; Z_TYPE($1.u.constant)=IS_LONG; $1.op_type = IS_CONST; INIT_PZVAL(&$1.u.constant); zend_do_binary_op(ZEND_SUB, &$$, &$1, &$2 TSRMLS_CC); } + | '!' expr { zend_do_unary_op(ZEND_BOOL_NOT, &$$, &$2 TSRMLS_CC); } + | '~' expr { zend_do_unary_op(ZEND_BW_NOT, &$$, &$2 TSRMLS_CC); } + | expr T_IS_IDENTICAL expr { zend_do_binary_op(ZEND_IS_IDENTICAL, &$$, &$1, &$3 TSRMLS_CC); } + | expr T_IS_NOT_IDENTICAL expr { zend_do_binary_op(ZEND_IS_NOT_IDENTICAL, &$$, &$1, &$3 TSRMLS_CC); } + | expr T_IS_EQUAL expr { zend_do_binary_op(ZEND_IS_EQUAL, &$$, &$1, &$3 TSRMLS_CC); } + | expr T_IS_NOT_EQUAL expr { zend_do_binary_op(ZEND_IS_NOT_EQUAL, &$$, &$1, &$3 TSRMLS_CC); } + | expr '<' expr { zend_do_binary_op(ZEND_IS_SMALLER, &$$, &$1, &$3 TSRMLS_CC); } + | expr T_IS_SMALLER_OR_EQUAL expr { zend_do_binary_op(ZEND_IS_SMALLER_OR_EQUAL, &$$, &$1, &$3 TSRMLS_CC); } + | expr '>' expr { zend_do_binary_op(ZEND_IS_SMALLER, &$$, &$3, &$1 TSRMLS_CC); } + | expr T_IS_GREATER_OR_EQUAL expr { zend_do_binary_op(ZEND_IS_SMALLER_OR_EQUAL, &$$, &$3, &$1 TSRMLS_CC); } + | expr T_INSTANCEOF class_name_reference { zend_do_instanceof(&$$, &$1, &$3, 0 TSRMLS_CC); } + | '(' expr ')' { $$ = $2; } + | expr '?' { zend_do_begin_qm_op(&$1, &$2 TSRMLS_CC); } + expr ':' { zend_do_qm_true(&$4, &$2, &$5 TSRMLS_CC); } + expr { zend_do_qm_false(&$$, &$7, &$2, &$5 TSRMLS_CC); } + | internal_functions_in_yacc { $$ = $1; } + | T_INT_CAST expr { zend_do_cast(&$$, &$2, IS_LONG TSRMLS_CC); } + | T_DOUBLE_CAST expr { zend_do_cast(&$$, &$2, IS_DOUBLE TSRMLS_CC); } + | T_STRING_CAST expr { zend_do_cast(&$$, &$2, IS_STRING TSRMLS_CC); } + | T_ARRAY_CAST expr { zend_do_cast(&$$, &$2, IS_ARRAY TSRMLS_CC); } + | T_OBJECT_CAST expr { zend_do_cast(&$$, &$2, IS_OBJECT TSRMLS_CC); } + | T_BOOL_CAST expr { zend_do_cast(&$$, &$2, IS_BOOL TSRMLS_CC); } + | T_UNSET_CAST expr { zend_do_cast(&$$, &$2, IS_NULL TSRMLS_CC); } + | T_EXIT exit_expr { zend_do_exit(&$$, &$2 TSRMLS_CC); } + | '@' { zend_do_begin_silence(&$1 TSRMLS_CC); } expr { zend_do_end_silence(&$1 TSRMLS_CC); $$ = $3; } + | scalar { $$ = $1; } + | T_ARRAY '(' array_pair_list ')' { $$ = $3; } + | '`' encaps_list '`' { zend_do_shell_exec(&$$, &$2 TSRMLS_CC); } + | T_PRINT expr { zend_do_print(&$$, &$2 TSRMLS_CC); } +; + +function_call: + T_STRING '(' { $2.u.opline_num = zend_do_begin_function_call(&$1 TSRMLS_CC); } + function_call_parameter_list + ')' { zend_do_end_function_call(&$1, &$$, &$4, 0, $2.u.opline_num TSRMLS_CC); zend_do_extended_fcall_end(TSRMLS_C); } + | fully_qualified_class_name T_PAAMAYIM_NEKUDOTAYIM T_STRING '(' { zend_do_begin_class_member_function_call(&$1, &$3 TSRMLS_CC); } + function_call_parameter_list + ')' { zend_do_end_function_call(NULL, &$$, &$6, 1, 1 TSRMLS_CC); zend_do_extended_fcall_end(TSRMLS_C);} + | fully_qualified_class_name T_PAAMAYIM_NEKUDOTAYIM variable_without_objects '(' { zend_do_end_variable_parse(BP_VAR_R, 0 TSRMLS_CC); zend_do_begin_class_member_function_call(&$1, &$3 TSRMLS_CC); } + function_call_parameter_list + ')' { zend_do_end_function_call(NULL, &$$, &$6, 1, 1 TSRMLS_CC); zend_do_extended_fcall_end(TSRMLS_C);} + | variable_without_objects '(' { zend_do_end_variable_parse(BP_VAR_R, 0 TSRMLS_CC); zend_do_begin_dynamic_function_call(&$1 TSRMLS_CC); } + function_call_parameter_list ')' + { zend_do_end_function_call(&$1, &$$, &$4, 0, 1 TSRMLS_CC); zend_do_extended_fcall_end(TSRMLS_C);} +; + +fully_qualified_class_name: + T_STRING { zend_do_fetch_class(&$$, &$1 TSRMLS_CC); } +; + +class_name_reference: + T_STRING { zend_do_fetch_class(&$$, &$1 TSRMLS_CC); } + | dynamic_class_name_reference { zend_do_end_variable_parse(BP_VAR_R, 0 TSRMLS_CC); zend_do_fetch_class(&$$, &$1 TSRMLS_CC); } +; + + +dynamic_class_name_reference: + base_variable T_OBJECT_OPERATOR { zend_do_push_object(&$1 TSRMLS_CC); } + object_property { zend_do_push_object(&$4 TSRMLS_CC); zend_do_declare_implicit_property(TSRMLS_C); } dynamic_class_name_variable_properties + { zend_do_pop_object(&$$ TSRMLS_CC); $$.u.EA.type = ZEND_PARSED_MEMBER; } + | base_variable { $$ = $1; } +; + + +dynamic_class_name_variable_properties: + dynamic_class_name_variable_properties dynamic_class_name_variable_property + | /* empty */ +; + + +dynamic_class_name_variable_property: + T_OBJECT_OPERATOR object_property { zend_do_push_object(&$2 TSRMLS_CC); zend_do_declare_implicit_property(TSRMLS_C); } +; + +exit_expr: + /* empty */ { memset(&$$, 0, sizeof(znode)); $$.op_type = IS_UNUSED; } + | '(' ')' { memset(&$$, 0, sizeof(znode)); $$.op_type = IS_UNUSED; } + | '(' expr ')' { $$ = $2; } +; + + +ctor_arguments: + /* empty */ { Z_LVAL($$.u.constant)=0; } + | '(' function_call_parameter_list ')' { $$ = $2; } +; + + +common_scalar: + T_LNUMBER { $$ = $1; } + | T_DNUMBER { $$ = $1; } + | T_CONSTANT_ENCAPSED_STRING { $$ = $1; } + | T_LINE { $$ = $1; } + | T_FILE { $$ = $1; } + | T_CLASS_C { $$ = $1; } + | T_METHOD_C { $$ = $1; } + | T_FUNC_C { $$ = $1; } +; + + +static_scalar: /* compile-time evaluated scalars */ + common_scalar { $$ = $1; } + | T_STRING { zend_do_fetch_constant(&$$, NULL, &$1, ZEND_CT TSRMLS_CC); } + | '+' static_scalar { $$ = $2; } + | '-' static_scalar { zval minus_one; Z_TYPE(minus_one) = IS_LONG; Z_LVAL(minus_one) = -1; mul_function(&$2.u.constant, &$2.u.constant, &minus_one TSRMLS_CC); $$ = $2; } + | T_ARRAY '(' static_array_pair_list ')' { $$ = $3; Z_TYPE($$.u.constant) = IS_CONSTANT_ARRAY; } + | static_class_constant { $$ = $1; } +; + +static_class_constant: + T_STRING T_PAAMAYIM_NEKUDOTAYIM T_STRING { zend_do_fetch_constant(&$$, &$1, &$3, ZEND_CT TSRMLS_CC); } +; + +scalar: + T_STRING { zend_do_fetch_constant(&$$, NULL, &$1, ZEND_RT TSRMLS_CC); } + | T_STRING_VARNAME { $$ = $1; } + | class_constant { $$ = $1; } + | common_scalar { $$ = $1; } + | '"' encaps_list '"' { $$ = $2; } + | '\'' encaps_list '\'' { $$ = $2; } + | T_START_HEREDOC encaps_list T_END_HEREDOC { $$ = $2; zend_do_end_heredoc(TSRMLS_C); } +; + + +static_array_pair_list: + /* empty */ { $$.op_type = IS_CONST; INIT_PZVAL(&$$.u.constant); array_init(&$$.u.constant); } + | non_empty_static_array_pair_list possible_comma { $$ = $1; } +; + +possible_comma: + /* empty */ + | ',' +; + +non_empty_static_array_pair_list: + non_empty_static_array_pair_list ',' static_scalar T_DOUBLE_ARROW static_scalar { zend_do_add_static_array_element(&$$, &$3, &$5); } + | non_empty_static_array_pair_list ',' static_scalar { zend_do_add_static_array_element(&$$, NULL, &$3); } + | static_scalar T_DOUBLE_ARROW static_scalar { $$.op_type = IS_CONST; INIT_PZVAL(&$$.u.constant); array_init(&$$.u.constant); zend_do_add_static_array_element(&$$, &$1, &$3); } + | static_scalar { $$.op_type = IS_CONST; INIT_PZVAL(&$$.u.constant); array_init(&$$.u.constant); zend_do_add_static_array_element(&$$, NULL, &$1); } +; + +expr: + r_variable { $$ = $1; } + | expr_without_variable { $$ = $1; } +; + + +r_variable: + variable { zend_do_end_variable_parse(BP_VAR_R, 0 TSRMLS_CC); $$ = $1; } +; + + +w_variable: + variable { zend_do_end_variable_parse(BP_VAR_W, 0 TSRMLS_CC); $$ = $1; } + { zend_check_writable_variable(&$1); } +; + +rw_variable: + variable { zend_do_end_variable_parse(BP_VAR_RW, 0 TSRMLS_CC); $$ = $1; } + { zend_check_writable_variable(&$1); } +; + +variable: + base_variable_with_function_calls T_OBJECT_OPERATOR { zend_do_push_object(&$1 TSRMLS_CC); } + object_property { zend_do_push_object(&$4 TSRMLS_CC); } method_or_not variable_properties + { zend_do_pop_object(&$$ TSRMLS_CC); $$.u.EA.type = $1.u.EA.type | ($7.u.EA.type ? $7.u.EA.type : $6.u.EA.type); } + | base_variable_with_function_calls { $$ = $1; } +; + +variable_properties: + variable_properties variable_property { $$.u.EA.type = $2.u.EA.type; } + | /* empty */ { $$.u.EA.type = 0; } +; + + +variable_property: + T_OBJECT_OPERATOR object_property { zend_do_push_object(&$2 TSRMLS_CC); } method_or_not { $$.u.EA.type = $4.u.EA.type; } +; + +method_or_not: + '(' { zend_do_pop_object(&$1 TSRMLS_CC); zend_do_begin_method_call(&$1 TSRMLS_CC); } + function_call_parameter_list ')' + { zend_do_end_function_call(&$1, &$$, &$3, 1, 1 TSRMLS_CC); zend_do_extended_fcall_end(TSRMLS_C); + zend_do_push_object(&$$ TSRMLS_CC); $$.u.EA.type = ZEND_PARSED_METHOD_CALL; } + | /* empty */ { zend_do_declare_implicit_property(TSRMLS_C); $$.u.EA.type = ZEND_PARSED_MEMBER; } +; + +variable_without_objects: + reference_variable { $$ = $1; } + | simple_indirect_reference reference_variable { zend_do_indirect_references(&$$, &$1, &$2 TSRMLS_CC); } +; + +static_member: + fully_qualified_class_name T_PAAMAYIM_NEKUDOTAYIM variable_without_objects { $$ = $3; zend_do_fetch_static_member(&$$, &$1 TSRMLS_CC); } +; + + +base_variable_with_function_calls: + base_variable { $$ = $1; } + | function_call { zend_do_begin_variable_parse(TSRMLS_C); $$ = $1; $$.u.EA.type = ZEND_PARSED_FUNCTION_CALL; } +; + + +base_variable: + reference_variable { $$ = $1; $$.u.EA.type = ZEND_PARSED_VARIABLE; } + | simple_indirect_reference reference_variable { zend_do_indirect_references(&$$, &$1, &$2 TSRMLS_CC); $$.u.EA.type = ZEND_PARSED_VARIABLE; } + | static_member { $$ = $1; $$.u.EA.type = ZEND_PARSED_STATIC_MEMBER; } +; + +reference_variable: + reference_variable '[' dim_offset ']' { fetch_array_dim(&$$, &$1, &$3 TSRMLS_CC); } + | reference_variable '{' expr '}' { fetch_string_offset(&$$, &$1, &$3 TSRMLS_CC); } + | compound_variable { zend_do_begin_variable_parse(TSRMLS_C); fetch_simple_variable(&$$, &$1, 1 TSRMLS_CC); } +; + + +compound_variable: + T_VARIABLE { $$ = $1; } + | '$' '{' expr '}' { $$ = $3; } +; + +dim_offset: + /* empty */ { $$.op_type = IS_UNUSED; } + | expr { $$ = $1; } +; + + +object_property: + object_dim_list { $$ = $1; } + | variable_without_objects { zend_do_end_variable_parse(BP_VAR_R, 0 TSRMLS_CC); } { znode tmp_znode; zend_do_pop_object(&tmp_znode TSRMLS_CC); zend_do_fetch_property(&$$, &tmp_znode, &$1 TSRMLS_CC);} +; + +object_dim_list: + object_dim_list '[' dim_offset ']' { fetch_array_dim(&$$, &$1, &$3 TSRMLS_CC); } + | object_dim_list '{' expr '}' { fetch_string_offset(&$$, &$1, &$3 TSRMLS_CC); } + | variable_name { znode tmp_znode; zend_do_pop_object(&tmp_znode TSRMLS_CC); zend_do_fetch_property(&$$, &tmp_znode, &$1 TSRMLS_CC);} +; + +variable_name: + T_STRING { $$ = $1; } + | '{' expr '}' { $$ = $2; } +; + +simple_indirect_reference: + '$' { Z_LVAL($$.u.constant) = 1; } + | simple_indirect_reference '$' { Z_LVAL($$.u.constant)++; } +; + +assignment_list: + assignment_list ',' assignment_list_element + | assignment_list_element +; + + +assignment_list_element: + variable { zend_do_add_list_element(&$1 TSRMLS_CC); } + | T_LIST '(' { zend_do_new_list_begin(TSRMLS_C); } assignment_list ')' { zend_do_new_list_end(TSRMLS_C); } + | /* empty */ { zend_do_add_list_element(NULL TSRMLS_CC); } +; + + +array_pair_list: + /* empty */ { zend_do_init_array(&$$, NULL, NULL, 0 TSRMLS_CC); } + | non_empty_array_pair_list possible_comma { $$ = $1; } +; + +non_empty_array_pair_list: + non_empty_array_pair_list ',' expr T_DOUBLE_ARROW expr { zend_do_add_array_element(&$$, &$5, &$3, 0 TSRMLS_CC); } + | non_empty_array_pair_list ',' expr { zend_do_add_array_element(&$$, &$3, NULL, 0 TSRMLS_CC); } + | expr T_DOUBLE_ARROW expr { zend_do_init_array(&$$, &$3, &$1, 0 TSRMLS_CC); } + | expr { zend_do_init_array(&$$, &$1, NULL, 0 TSRMLS_CC); } + | non_empty_array_pair_list ',' expr T_DOUBLE_ARROW '&' w_variable { zend_do_add_array_element(&$$, &$6, &$3, 1 TSRMLS_CC); } + | non_empty_array_pair_list ',' '&' w_variable { zend_do_add_array_element(&$$, &$4, NULL, 1 TSRMLS_CC); } + | expr T_DOUBLE_ARROW '&' w_variable { zend_do_init_array(&$$, &$4, &$1, 1 TSRMLS_CC); } + | '&' w_variable { zend_do_init_array(&$$, &$2, NULL, 1 TSRMLS_CC); } +; + +encaps_list: + encaps_list encaps_var { zend_do_end_variable_parse(BP_VAR_R, 0 TSRMLS_CC); zend_do_add_variable(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list T_STRING { zend_do_add_string(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list T_NUM_STRING { zend_do_add_string(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list T_ENCAPSED_AND_WHITESPACE { zend_do_add_string(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list T_CHARACTER { zend_do_add_char(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list T_BAD_CHARACTER { zend_do_add_string(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list '[' { Z_LVAL($2.u.constant) = (long) '['; zend_do_add_char(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list ']' { Z_LVAL($2.u.constant) = (long) ']'; zend_do_add_char(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list '{' { Z_LVAL($2.u.constant) = (long) '{'; zend_do_add_char(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list '}' { Z_LVAL($2.u.constant) = (long) '}'; zend_do_add_char(&$$, &$1, &$2 TSRMLS_CC); } + | encaps_list T_OBJECT_OPERATOR { znode tmp; Z_LVAL($2.u.constant) = (long) '-'; zend_do_add_char(&tmp, &$1, &$2 TSRMLS_CC); Z_LVAL($2.u.constant) = (long) '>'; zend_do_add_char(&$$, &tmp, &$2 TSRMLS_CC); } + | /* empty */ { zend_do_init_string(&$$ TSRMLS_CC); } + +; + + + +encaps_var: + T_VARIABLE { zend_do_begin_variable_parse(TSRMLS_C); fetch_simple_variable(&$$, &$1, 1 TSRMLS_CC); } + | T_VARIABLE '[' { zend_do_begin_variable_parse(TSRMLS_C); } encaps_var_offset ']' { fetch_array_begin(&$$, &$1, &$4 TSRMLS_CC); } + | T_VARIABLE T_OBJECT_OPERATOR T_STRING { zend_do_begin_variable_parse(TSRMLS_C); fetch_simple_variable(&$2, &$1, 1 TSRMLS_CC); zend_do_fetch_property(&$$, &$2, &$3 TSRMLS_CC); } + | T_DOLLAR_OPEN_CURLY_BRACES expr '}' { zend_do_begin_variable_parse(TSRMLS_C); fetch_simple_variable(&$$, &$2, 1 TSRMLS_CC); } + | T_DOLLAR_OPEN_CURLY_BRACES T_STRING_VARNAME '[' expr ']' '}' { zend_do_begin_variable_parse(TSRMLS_C); fetch_array_begin(&$$, &$2, &$4 TSRMLS_CC); } + | T_CURLY_OPEN variable '}' { $$ = $2; } +; + + +encaps_var_offset: + T_STRING { $$ = $1; } + | T_NUM_STRING { $$ = $1; } + | T_VARIABLE { fetch_simple_variable(&$$, &$1, 1 TSRMLS_CC); } +; + + +internal_functions_in_yacc: + T_ISSET '(' isset_variables ')' { $$ = $3; } + | T_EMPTY '(' variable ')' { zend_do_isset_or_isempty(ZEND_ISEMPTY, &$$, &$3 TSRMLS_CC); } + | T_INCLUDE expr { zend_do_include_or_eval(ZEND_INCLUDE, &$$, &$2 TSRMLS_CC); } + | T_INCLUDE_ONCE expr { zend_do_include_or_eval(ZEND_INCLUDE_ONCE, &$$, &$2 TSRMLS_CC); } + | T_EVAL '(' expr ')' { zend_do_include_or_eval(ZEND_EVAL, &$$, &$3 TSRMLS_CC); } + | T_REQUIRE expr { zend_do_include_or_eval(ZEND_REQUIRE, &$$, &$2 TSRMLS_CC); } + | T_REQUIRE_ONCE expr { zend_do_include_or_eval(ZEND_REQUIRE_ONCE, &$$, &$2 TSRMLS_CC); } +; + +isset_variables: + variable { zend_do_isset_or_isempty(ZEND_ISSET, &$$, &$1 TSRMLS_CC); } + | isset_variables ',' { zend_do_boolean_and_begin(&$1, &$2 TSRMLS_CC); } variable { znode tmp; zend_do_isset_or_isempty(ZEND_ISSET, &tmp, &$4 TSRMLS_CC); zend_do_boolean_and_end(&$$, &$1, &tmp, &$2 TSRMLS_CC); } +; + +class_constant: + fully_qualified_class_name T_PAAMAYIM_NEKUDOTAYIM T_STRING { zend_do_fetch_constant(&$$, &$1, &$3, ZEND_RT TSRMLS_CC); } +; + diff --git a/src/ptgen/php5/zend_language_parser.y.foot b/src/ptgen/php5/zend_language_parser.y.foot new file mode 100755 index 0000000..a9ca213 --- /dev/null +++ b/src/ptgen/php5/zend_language_parser.y.foot @@ -0,0 +1,12 @@ +#include + +extern char yytext[]; +extern int column; +extern int line; + +void yyerror( char *s) +{ +fflush(stdout); +fprintf(stderr,"%s: %d.%d\n",s,line,column); +} + diff --git a/src/ptgen/php5/zend_language_parser.y.head b/src/ptgen/php5/zend_language_parser.y.head new file mode 100755 index 0000000..8ba75ff --- /dev/null +++ b/src/ptgen/php5/zend_language_parser.y.head @@ -0,0 +1,97 @@ + +%expect 4 + +%left T_INCLUDE T_INCLUDE_ONCE T_EVAL T_REQUIRE T_REQUIRE_ONCE +%left ',' +%left T_LOGICAL_OR +%left T_LOGICAL_XOR +%left T_LOGICAL_AND +%right T_PRINT +%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL +%left '?' ':' +%left T_BOOLEAN_OR +%left T_BOOLEAN_AND +%left '|' +%left '^' +%left '&' +%nonassoc T_IS_EQUAL T_IS_NOT_EQUAL T_IS_IDENTICAL T_IS_NOT_IDENTICAL +%nonassoc '<' T_IS_SMALLER_OR_EQUAL '>' T_IS_GREATER_OR_EQUAL +%left T_SL T_SR +%left '+' '-' '.' +%left '*' '/' '%' +%right '!' +%nonassoc T_INSTANCEOF +%right '~' T_INC T_DEC T_INT_CAST T_DOUBLE_CAST T_STRING_CAST T_ARRAY_CAST T_OBJECT_CAST T_BOOL_CAST T_UNSET_CAST '@' +%right '[' +%nonassoc T_NEW T_CLONE +%token T_EXIT +%token T_IF +%left T_ELSEIF +%left T_ELSE +%left T_ENDIF +%token T_LNUMBER +%token T_DNUMBER +%token T_STRING +%token T_STRING_VARNAME +%token T_VARIABLE +%token T_NUM_STRING +%token T_INLINE_HTML +%token T_CHARACTER +%token T_BAD_CHARACTER +%token T_ENCAPSED_AND_WHITESPACE +%token T_CONSTANT_ENCAPSED_STRING +%token T_ECHO +%token T_DO +%token T_WHILE +%token T_ENDWHILE +%token T_FOR +%token T_ENDFOR +%token T_FOREACH +%token T_ENDFOREACH +%token T_DECLARE +%token T_ENDDECLARE +%token T_AS +%token T_SWITCH +%token T_ENDSWITCH +%token T_CASE +%token T_DEFAULT +%token T_BREAK +%token T_CONTINUE +%token T_FUNCTION +%token T_CONST +%token T_RETURN +%token T_TRY +%token T_CATCH +%token T_THROW +%token T_USE +%token T_GLOBAL +%right T_STATIC T_ABSTRACT T_FINAL T_PRIVATE T_PROTECTED T_PUBLIC +%token T_VAR +%token T_UNSET +%token T_ISSET +%token T_EMPTY +%token T_HALT_COMPILER +%token T_CLASS +%token T_INTERFACE +%token T_EXTENDS +%token T_IMPLEMENTS +%token T_OBJECT_OPERATOR +%token T_DOUBLE_ARROW +%token T_LIST +%token T_ARRAY +%token T_CLASS_C +%token T_METHOD_C +%token T_FUNC_C +%token T_LINE +%token T_FILE +%token T_COMMENT +%token T_DOC_COMMENT +%token T_OPEN_TAG +%token T_OPEN_TAG_WITH_ECHO +%token T_CLOSE_TAG +%token T_WHITESPACE +%token T_START_HEREDOC +%token T_END_HEREDOC +%token T_DOLLAR_OPEN_CURLY_BRACES +%token T_CURLY_OPEN +%token T_PAAMAYIM_NEKUDOTAYIM diff --git a/src/ptgen/php5/zend_language_scanner.l b/src/ptgen/php5/zend_language_scanner.l new file mode 100755 index 0000000..db44f8a --- /dev/null +++ b/src/ptgen/php5/zend_language_scanner.l @@ -0,0 +1,941 @@ +%{ + +/* + +----------------------------------------------------------------------+ + | Zend Engine | + +----------------------------------------------------------------------+ + | Copyright (c) 1998-2006 Zend Technologies Ltd. (http://www.zend.com) | + +----------------------------------------------------------------------+ + | This source file is subject to version 2.00 of the Zend license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.zend.com/license/2_00.txt. | + | If you did not receive a copy of the Zend license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@zend.com so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Andi Gutmans | + | Zeev Suraski | + +----------------------------------------------------------------------+ +*/ + +/* $Id: zend_language_scanner.l,v 1.131.2.11.2.7 2007/01/18 23:28:08 iliaa Exp $ */ + +#include +#include +#include "pt_zend_language_parser.tab.hh" +#include +#include +using namespace std; + +extern map name2id; +void count(); + +// TODO: potential troublesome global variables; better to have encapsulated states... +/* NOTE: yylineno is expensive, so disable it */ +int column = 0; +int line = 1; +// handle heredoc: +int heredoc_len = 0; +char * heredoc = NULL; + +#define YY_DECL int yylex(YYSTYPE *yylvalp) +#define YY_FATAL_ERROR zend_fatal_scanner_error + +void zend_fatal_scanner_error(char *message) +{ + fprintf(stderr, "%s", message); +} + +%} + +%x ST_IN_SCRIPTING +%x ST_DOUBLE_QUOTES +%x ST_SINGLE_QUOTE +%x ST_BACKQUOTE +%x ST_HEREDOC +%x ST_LOOKING_FOR_PROPERTY +%x ST_LOOKING_FOR_VARNAME +%x ST_COMMENT +%x ST_DOC_COMMENT +%x ST_ONE_LINE_COMMENT +%option stack + +LNUM [0-9]+ +DNUM ([0-9]*[\.][0-9]+)|([0-9]+[\.][0-9]*) +EXPONENT_DNUM (({LNUM}|{DNUM})[eE][+-]?{LNUM}) +HNUM "0x"[0-9a-fA-F]+ +LABEL [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]* +WHITESPACE [ \n\r\t]+ +TABS_AND_SPACES [ \t]* +TOKENS [;:,.\[\]()|^&+-/*=%!~$<>?@] +ENCAPSED_TOKENS [\[\]{}$] +ESCAPED_AND_WHITESPACE [\n\t\r #'.:;,()|^&+-/*=%!~<>?@]+ +ANY_CHAR (.|[\n]) +NEWLINE ("\r"|"\n"|"\r\n") + +%option noyylineno +%option yywrap + +%% + +"exit" { + count(); yylvalp->t=new Terminal(name2id["T_EXIT"],yytext,line); return T_EXIT; +} + +"die" { + count(); yylvalp->t=new Terminal(name2id["T_EXIT"],yytext,line); return T_EXIT; +} + +"function" { + count(); yylvalp->t=new Terminal(name2id["T_FUNCTION"],yytext,line); return T_FUNCTION; +} + +"const" { + count(); yylvalp->t=new Terminal(name2id["T_CONST"],yytext,line); return T_CONST; +} + +"return" { + count(); yylvalp->t=new Terminal(name2id["T_RETURN"],yytext,line); return T_RETURN; +} + +"try" { + count(); yylvalp->t=new Terminal(name2id["T_TRY"],yytext,line); return T_TRY; +} + +"catch" { + count(); yylvalp->t=new Terminal(name2id["T_CATCH"],yytext,line); return T_CATCH; +} + +"throw" { + count(); yylvalp->t=new Terminal(name2id["T_THROW"],yytext,line); return T_THROW; +} + +"if" { + count(); yylvalp->t=new Terminal(name2id["T_IF"],yytext,line); return T_IF; +} + +"elseif" { + count(); yylvalp->t=new Terminal(name2id["T_ELSEIF"],yytext,line); return T_ELSEIF; +} + +"endif" { + count(); yylvalp->t=new Terminal(name2id["T_ENDIF"],yytext,line); return T_ENDIF; +} + +"else" { + count(); yylvalp->t=new Terminal(name2id["T_ELSE"],yytext,line); return T_ELSE; +} + +"while" { + count(); yylvalp->t=new Terminal(name2id["T_WHILE"],yytext,line); return T_WHILE; +} + +"endwhile" { + count(); yylvalp->t=new Terminal(name2id["T_ENDWHILE"],yytext,line); return T_ENDWHILE; +} + +"do" { + count(); yylvalp->t=new Terminal(name2id["T_DO"],yytext,line); return T_DO; +} + +"for" { + count(); yylvalp->t=new Terminal(name2id["T_FOR"],yytext,line); return T_FOR; +} + +"endfor" { + count(); yylvalp->t=new Terminal(name2id["T_ENDFOR"],yytext,line); return T_ENDFOR; +} + +"foreach" { + count(); yylvalp->t=new Terminal(name2id["T_FOREACH"],yytext,line); return T_FOREACH; +} + +"endforeach" { + count(); yylvalp->t=new Terminal(name2id["T_ENDFOREACH"],yytext,line); return T_ENDFOREACH; +} + +"declare" { + count(); yylvalp->t=new Terminal(name2id["T_DECLARE"],yytext,line); return T_DECLARE; +} + +"enddeclare" { + count(); yylvalp->t=new Terminal(name2id["T_ENDDECLARE"],yytext,line); return T_ENDDECLARE; +} + +"instanceof" { + count(); yylvalp->t=new Terminal(name2id["T_INSTANCEOF"],yytext,line); return T_INSTANCEOF; +} + +"as" { + count(); yylvalp->t=new Terminal(name2id["T_AS"],yytext,line); return T_AS; +} + +"switch" { + count(); yylvalp->t=new Terminal(name2id["T_SWITCH"],yytext,line); return T_SWITCH; +} + +"endswitch" { + count(); yylvalp->t=new Terminal(name2id["T_ENDSWITCH"],yytext,line); return T_ENDSWITCH; +} + +"case" { + count(); yylvalp->t=new Terminal(name2id["T_CASE"],yytext,line); return T_CASE; +} + +"default" { + count(); yylvalp->t=new Terminal(name2id["T_DEFAULT"],yytext,line); return T_DEFAULT; +} + +"break" { + count(); yylvalp->t=new Terminal(name2id["T_BREAK"],yytext,line); return T_BREAK; +} + +"continue" { + count(); yylvalp->t=new Terminal(name2id["T_CONTINUE"],yytext,line); return T_CONTINUE; +} + +"echo" { + count(); yylvalp->t=new Terminal(name2id["T_ECHO"],yytext,line); return T_ECHO; +} + +"print" { + count(); yylvalp->t=new Terminal(name2id["T_PRINT"],yytext,line); return T_PRINT; +} + +"class" { + count(); yylvalp->t=new Terminal(name2id["T_CLASS"],yytext,line); return T_CLASS; +} + +"interface" { + count(); yylvalp->t=new Terminal(name2id["T_INTERFACE"],yytext,line); return T_INTERFACE; +} + +"extends" { + count(); yylvalp->t=new Terminal(name2id["T_EXTENDS"],yytext,line); return T_EXTENDS; +} + +"implements" { + count(); yylvalp->t=new Terminal(name2id["T_IMPLEMENTS"],yytext,line); return T_IMPLEMENTS; +} + +"->" { + yy_push_state(ST_LOOKING_FOR_PROPERTY); + count(); yylvalp->t=new Terminal(name2id["T_OBJECT_OPERATOR"],yytext,line); return T_OBJECT_OPERATOR; +} + +{LABEL} { + yy_pop_state(); + count(); yylvalp->t=new Terminal(name2id["T_STRING"],yytext,line); return T_STRING; +} + +{ANY_CHAR} { + yyless(0); + yy_pop_state(); + count(); +} + +"::" { + count(); yylvalp->t=new Terminal(name2id["T_PAAMAYIM_NEKUDOTAYIM"],yytext,line); return T_PAAMAYIM_NEKUDOTAYIM; +} + +"new" { + count(); yylvalp->t=new Terminal(name2id["T_NEW"],yytext,line); return T_NEW; +} + +"clone" { + count(); yylvalp->t=new Terminal(name2id["T_CLONE"],yytext,line); return T_CLONE; +} + +"var" { + count(); yylvalp->t=new Terminal(name2id["T_VAR"],yytext,line); return T_VAR; +} + +"("{TABS_AND_SPACES}("int"|"integer"){TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_INT_CAST"],yytext,line); return T_INT_CAST; +} + +"("{TABS_AND_SPACES}("real"|"double"|"float"){TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_DOUBLE_CAST"],yytext,line); return T_DOUBLE_CAST; +} + +"("{TABS_AND_SPACES}"string"{TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_STRING_CAST"],yytext,line); return T_STRING_CAST; +} + +"("{TABS_AND_SPACES}"binary"{TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_STRING_CAST"],yytext,line); return T_STRING_CAST; +} + +"("{TABS_AND_SPACES}"array"{TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_ARRAY_CAST"],yytext,line); return T_ARRAY_CAST; +} + +"("{TABS_AND_SPACES}"object"{TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_OBJECT_CAST"],yytext,line); return T_OBJECT_CAST; +} + +"("{TABS_AND_SPACES}("bool"|"boolean"){TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_BOOL_CAST"],yytext,line); return T_BOOL_CAST; +} + +"("{TABS_AND_SPACES}("unset"){TABS_AND_SPACES}")" { + count(); yylvalp->t=new Terminal(name2id["T_UNSET_CAST"],yytext,line); return T_UNSET_CAST; +} + +"eval" { + count(); yylvalp->t=new Terminal(name2id["T_EVAL"],yytext,line); return T_EVAL; +} + +"include" { + count(); yylvalp->t=new Terminal(name2id["T_INCLUDE"],yytext,line); return T_INCLUDE; +} + +"include_once" { + count(); yylvalp->t=new Terminal(name2id["T_INCLUDE_ONCE"],yytext,line); return T_INCLUDE_ONCE; +} + +"require" { + count(); yylvalp->t=new Terminal(name2id["T_REQUIRE"],yytext,line); return T_REQUIRE; +} + +"require_once" { + count(); yylvalp->t=new Terminal(name2id["T_REQUIRE_ONCE"],yytext,line); return T_REQUIRE_ONCE; +} + +"use" { + count(); yylvalp->t=new Terminal(name2id["T_USE"],yytext,line); return T_USE; +} + +"global" { + count(); yylvalp->t=new Terminal(name2id["T_GLOBAL"],yytext,line); return T_GLOBAL; +} + +"isset" { + count(); yylvalp->t=new Terminal(name2id["T_ISSET"],yytext,line); return T_ISSET; +} + +"empty" { + count(); yylvalp->t=new Terminal(name2id["T_EMPTY"],yytext,line); return T_EMPTY; +} + +"__halt_compiler" { + count(); yylvalp->t=new Terminal(name2id["T_HALT_COMPILER"],yytext,line); return T_HALT_COMPILER; +} + +"static" { + count(); yylvalp->t=new Terminal(name2id["T_STATIC"],yytext,line); return T_STATIC; +} + +"abstract" { + count(); yylvalp->t=new Terminal(name2id["T_ABSTRACT"],yytext,line); return T_ABSTRACT; +} + +"final" { + count(); yylvalp->t=new Terminal(name2id["T_FINAL"],yytext,line); return T_FINAL; +} + +"private" { + count(); yylvalp->t=new Terminal(name2id["T_PRIVATE"],yytext,line); return T_PRIVATE; +} + +"protected" { + count(); yylvalp->t=new Terminal(name2id["T_PROTECTED"],yytext,line); return T_PROTECTED; +} + +"public" { + count(); yylvalp->t=new Terminal(name2id["T_PUBLIC"],yytext,line); return T_PUBLIC; +} + +"unset" { + count(); yylvalp->t=new Terminal(name2id["T_UNSET"],yytext,line); return T_UNSET; +} + +"=>" { + count(); yylvalp->t=new Terminal(name2id["T_DOUBLE_ARROW"],yytext,line); return T_DOUBLE_ARROW; +} + +"list" { + count(); yylvalp->t=new Terminal(name2id["T_LIST"],yytext,line); return T_LIST; +} + +"array" { + count(); yylvalp->t=new Terminal(name2id["T_ARRAY"],yytext,line); return T_ARRAY; +} + +"++" { + count(); yylvalp->t=new Terminal(name2id["T_INC"],yytext,line); return T_INC; +} + +"--" { + count(); yylvalp->t=new Terminal(name2id["T_DEC"],yytext,line); return T_DEC; +} + +"===" { + count(); yylvalp->t=new Terminal(name2id["T_IS_IDENTICAL"],yytext,line); return T_IS_IDENTICAL; +} + +"!==" { + count(); yylvalp->t=new Terminal(name2id["T_IS_NOT_IDENTICAL"],yytext,line); return T_IS_NOT_IDENTICAL; +} + +"==" { + count(); yylvalp->t=new Terminal(name2id["T_IS_EQUAL"],yytext,line); return T_IS_EQUAL; +} + +"!="|"<>" { + count(); yylvalp->t=new Terminal(name2id["T_IS_NOT_EQUAL"],yytext,line); return T_IS_NOT_EQUAL; +} + +"<=" { + count(); yylvalp->t=new Terminal(name2id["T_IS_SMALLER_OR_EQUAL"],yytext,line); return T_IS_SMALLER_OR_EQUAL; +} + +">=" { + count(); yylvalp->t=new Terminal(name2id["T_IS_GREATER_OR_EQUAL"],yytext,line); return T_IS_GREATER_OR_EQUAL; +} + +"+=" { + count(); yylvalp->t=new Terminal(name2id["T_PLUS_EQUAL"],yytext,line); return T_PLUS_EQUAL; +} + +"-=" { + count(); yylvalp->t=new Terminal(name2id["T_MINUS_EQUAL"],yytext,line); return T_MINUS_EQUAL; +} + +"*=" { + count(); yylvalp->t=new Terminal(name2id["T_MUL_EQUAL"],yytext,line); return T_MUL_EQUAL; +} + +"/=" { + count(); yylvalp->t=new Terminal(name2id["T_DIV_EQUAL"],yytext,line); return T_DIV_EQUAL; +} + +".=" { + count(); yylvalp->t=new Terminal(name2id["T_CONCAT_EQUAL"],yytext,line); return T_CONCAT_EQUAL; +} + +"%=" { + count(); yylvalp->t=new Terminal(name2id["T_MOD_EQUAL"],yytext,line); return T_MOD_EQUAL; +} + +"<<=" { + count(); yylvalp->t=new Terminal(name2id["T_SL_EQUAL"],yytext,line); return T_SL_EQUAL; +} + +">>=" { + count(); yylvalp->t=new Terminal(name2id["T_SR_EQUAL"],yytext,line); return T_SR_EQUAL; +} + +"&=" { + count(); yylvalp->t=new Terminal(name2id["T_AND_EQUAL"],yytext,line); return T_AND_EQUAL; +} + +"|=" { + count(); yylvalp->t=new Terminal(name2id["T_OR_EQUAL"],yytext,line); return T_OR_EQUAL; +} + +"^=" { + count(); yylvalp->t=new Terminal(name2id["T_XOR_EQUAL"],yytext,line); return T_XOR_EQUAL; +} + +"||" { + count(); yylvalp->t=new Terminal(name2id["T_BOOLEAN_OR"],yytext,line); return T_BOOLEAN_OR; +} + +"&&" { + count(); yylvalp->t=new Terminal(name2id["T_BOOLEAN_AND"],yytext,line); return T_BOOLEAN_AND; +} + +"OR" { + count(); yylvalp->t=new Terminal(name2id["T_LOGICAL_OR"],yytext,line); return T_LOGICAL_OR; +} + +"AND" { + count(); yylvalp->t=new Terminal(name2id["T_LOGICAL_AND"],yytext,line); return T_LOGICAL_AND; +} + +"XOR" { + count(); yylvalp->t=new Terminal(name2id["T_LOGICAL_XOR"],yytext,line); return T_LOGICAL_XOR; +} + +"<<" { + count(); yylvalp->t=new Terminal(name2id["T_SL"],yytext,line); return T_SL; +} + +">>" { + count(); yylvalp->t=new Terminal(name2id["T_SR"],yytext,line); return T_SR; +} + +{TOKENS} { + char temp[4] = {'\'', yytext[0], '\'', 0}; + count(); yylvalp->t=new Terminal(name2id[temp],yytext,line); return yytext[0]; +} + + +"{" { + yy_push_state(ST_IN_SCRIPTING); + count(); yylvalp->t=new Terminal(name2id["'{'"],yytext,line); return '{'; +} + + +"${" { + yy_push_state(ST_LOOKING_FOR_VARNAME); + count(); yylvalp->t=new Terminal(name2id["T_DOLLAR_OPEN_CURLY_BRACES"],yytext,line); return T_DOLLAR_OPEN_CURLY_BRACES; +} + + +"}" { + //RESET_DOC_COMMENT(); + /* This is a temporary fix which is dependant on flex and it's implementation */ + if (yy_start_stack_ptr) { // prevent syntax errors in php source files to crash the parser... + yy_pop_state(); + } + count(); yylvalp->t=new Terminal(name2id["'}'"],yytext,line); return '}'; +} + + +{LABEL} { + yy_pop_state(); + yy_push_state(ST_IN_SCRIPTING); // the push is used to reduce certain rules (e.g., "}"), but why need "ST_LOOKING_FOR_VARNAME" at first place? + count(); yylvalp->t=new Terminal(name2id["T_STRING_VARNAME"],yytext,line); return T_STRING_VARNAME; +} + + +{ANY_CHAR} { + yyless(0); + yy_pop_state(); + yy_push_state(ST_IN_SCRIPTING); + count(); +} + + +{LNUM} { + count(); yylvalp->t=new Terminal(name2id["T_LNUMBER"],yytext,line); return T_LNUMBER; +} + +{HNUM} { + count(); yylvalp->t=new Terminal(name2id["T_LNUMBER"],yytext,line); return T_LNUMBER; +} + +{LNUM}|{HNUM} { /* treat numbers (almost) as strings inside encapsulated strings */ + count(); yylvalp->t=new Terminal(name2id["T_NUM_STRING"],yytext,line); return T_NUM_STRING; +} + +{DNUM}|{EXPONENT_DNUM} { + count(); yylvalp->t=new Terminal(name2id["T_DNUMBER"],yytext,line); return T_DNUMBER; +} + +"__CLASS__" { + count(); yylvalp->t=new Terminal(name2id["T_CLASS_C"],yytext,line); return T_CLASS_C; +} + +"__FUNCTION__" { + count(); yylvalp->t=new Terminal(name2id["T_FUNC_C"],yytext,line); return T_FUNC_C; +} + +"__METHOD__" { + count(); yylvalp->t=new Terminal(name2id["T_METHOD_C"],yytext,line); return T_METHOD_C; +} + +"__LINE__" { + count(); yylvalp->t=new Terminal(name2id["T_LINE"],yytext,line); return T_LINE; +} + +"__FILE__" { + count(); yylvalp->t=new Terminal(name2id["T_FILE"],yytext,line); return T_FILE; +} + +(([^<]|"<"[^?%s<]){1,400})|"t=new Terminal(name2id["T_INLINE_HTML"],yytext,line); return T_INLINE_HTML; +} + +"" { + /* "2 means it's not */ + BEGIN(ST_IN_SCRIPTING); + count(); //return T_OPEN_TAG; + } else { + count(); yylvalp->t=new Terminal(name2id["T_INLINE_HTML"],yytext,line); return T_INLINE_HTML; + } +} + + +"<%="|"t=new Terminal(name2id["T_INLINE_HTML"],yytext,line); return T_INLINE_HTML; + } +} + + +"<%" { + if (1) { + BEGIN(ST_IN_SCRIPTING); + count(); //return T_OPEN_TAG; + } else { + count(); yylvalp->t=new Terminal(name2id["T_INLINE_HTML"],yytext,line); return T_INLINE_HTML; + } +} + + +""$"{LABEL} { + count(); yylvalp->t=new Terminal(name2id["T_VARIABLE"],yytext,line); return T_VARIABLE; +} + +{LABEL} { + count(); yylvalp->t=new Terminal(name2id["T_STRING"],yytext,line); return T_STRING; +} + +{LABEL} { + count(); yylvalp->t=new Terminal(name2id["T_STRING"],yytext,line); return T_STRING; +} + + +{WHITESPACE} { + count(); //return T_WHITESPACE; +} + + +"#"|"//" { + BEGIN(ST_ONE_LINE_COMMENT); + //yymore(); + count(); +} + +"?"|"%"|">" { + yymore(); + //count(); caused wrong "line" and "column" numbers +} + +[^\n\r?%>]*{ANY_CHAR} { + switch (yytext[yyleng-1]) { + case '?': case '%': case '>': + yyless(yyleng-1); + yymore(); + //count(); + break; // why need yymore() and why not BEGIN(ST_IN_SCRIPTING) here? it depends on whether we need to use the info about the comments. + default: + BEGIN(ST_IN_SCRIPTING); + count(); //return T_COMMENT; + } +} + +{NEWLINE} { + BEGIN(ST_IN_SCRIPTING); + count(); //return T_COMMENT; +} + +"?>"|"%>" { /* one-line comments can also be ended by "?>" or "%>"? */ + if (yytext[yyleng-2] != '%') { /* asp comment? */ + yyless(yyleng-2); + BEGIN(ST_IN_SCRIPTING); + count(); //return T_COMMENT; + } else { + yymore(); + //count(); + } +} + +"/**"{WHITESPACE} { + //RESET_DOC_COMMENT(); + BEGIN(ST_DOC_COMMENT); + //yymore(); + count(); +} + +"/*" { + BEGIN(ST_COMMENT); + //yymore(); + count(); +} + + +[^*]+ { /* TODO: input buffer of flex overflows when the comments are tooooooooo long */ + //yymore(); + count(); +} + +"*/" { + BEGIN(ST_IN_SCRIPTING); + count(); //return T_DOC_COMMENT; +} + +"*/" { + BEGIN(ST_IN_SCRIPTING); + count(); //return T_COMMENT; +} + +"*" { + //yymore(); + count(); +} + +(";"{WHITESPACE}*"?>"|""){NEWLINE}? { + BEGIN(INITIAL); + count(); yylvalp->t=new Terminal(name2id["T_CLOSE_TAG"],yytext,line); return T_CLOSE_TAG; /* implicit ';' at php-end tag */ +} + +("?>"|""){NEWLINE}? { + BEGIN(INITIAL); + count(); yylvalp->t=new Terminal(name2id["';'"],yytext,line); return ';';//return T_CLOSE_TAG; /* implicit ';' at php-end tag */ +} + + +"%>"{NEWLINE}? { + if (1) { + BEGIN(INITIAL); + count(); //return T_CLOSE_TAG; /* implicit ';' at php-end tag */ + } else { + yyless(1); //?? + count(); //return yytext[0]; + } +} + +";"{WHITESPACE}*"%>"{NEWLINE}? { + if (1) { + BEGIN(INITIAL); + count(); yylvalp->t=new Terminal(name2id["T_CLOSE_TAG"],yytext,line); return T_CLOSE_TAG; /* implicit ';' at php-end tag */ + } else { + yyless(1); //?? + count(); //return yytext[0]; + } +} + + +("b"?["]([^$"\\]|("\\".))*["]) { //?? Does this handle multi-line strings? + count(); yylvalp->t=new Terminal(name2id["T_CONSTANT_ENCAPSED_STRING"],yytext,line); return T_CONSTANT_ENCAPSED_STRING; +} + + +("b"?[']([^'\\]|("\\".))*[']) { //?? Does this handle multi-line strings? + count(); yylvalp->t=new Terminal(name2id["T_CONSTANT_ENCAPSED_STRING"],yytext,line); return T_CONSTANT_ENCAPSED_STRING; +} + + +b?["] { + BEGIN(ST_DOUBLE_QUOTES); + count(); yylvalp->t=new Terminal(name2id["'\"'"],yytext,line); return '\"'; +} + + +"b"?"<<<"{TABS_AND_SPACES}{LABEL}{NEWLINE} { + char *s; + int bprefix = (*yytext == 'b') ? 1 : 0; + + heredoc_len = yyleng-bprefix-3-1-(yytext[yyleng-2]=='\r'?1:0); + s = yytext+bprefix+3; + while ((*s == ' ') || (*s == '\t')) { + s++; + heredoc_len--; + } + heredoc = (char*)malloc(heredoc_len+1); + if ( heredoc ) { + memcpy(heredoc, s, heredoc_len); + heredoc[heredoc_len] = 0; + } else + heredoc_len = 0; + BEGIN(ST_HEREDOC); + count(); yylvalp->t=new Terminal(name2id["T_START_HEREDOC"],yytext,line); return T_START_HEREDOC; +} + + +[`] { + BEGIN(ST_BACKQUOTE); + count(); yylvalp->t=new Terminal(name2id["'`'"],yytext,line); return '`'; +} + + +b?['] { + BEGIN(ST_SINGLE_QUOTE); + count(); yylvalp->t=new Terminal(name2id["'\''"],yytext,line); return '\''; +} + + +^{LABEL}(";")?{NEWLINE} { + int label_len; + + if (yytext[yyleng-2]=='\r') { + label_len = yyleng-2; + } else { + label_len = yyleng-1; + } + + if (yytext[label_len-1]==';') { + label_len--; + } + + if (label_len==heredoc_len && !memcmp(yytext, heredoc, label_len)) { + yyless(yyleng - (yyleng - label_len)); + if ( heredoc ) { + free(heredoc); + heredoc=NULL; + } + heredoc_len=0; + BEGIN(ST_IN_SCRIPTING); + count(); yylvalp->t=new Terminal(name2id["T_END_HEREDOC"],yytext,line); return T_END_HEREDOC; + } else { + count(); yylvalp->t=new Terminal(name2id["T_STRING"],yytext,line); return T_STRING; + } +} + + +{ESCAPED_AND_WHITESPACE} { + count(); yylvalp->t=new Terminal(name2id["T_ENCAPSED_AND_WHITESPACE"],yytext,line); return T_ENCAPSED_AND_WHITESPACE; +} + +([^'\\]|\\[^'\\])+ { + count(); yylvalp->t=new Terminal(name2id["T_ENCAPSED_AND_WHITESPACE"],yytext,line); return T_ENCAPSED_AND_WHITESPACE; +} + + +[`]+ { + count(); yylvalp->t=new Terminal(name2id["T_ENCAPSED_AND_WHITESPACE"],yytext,line); return T_ENCAPSED_AND_WHITESPACE; +} + + +["]+ { + count(); yylvalp->t=new Terminal(name2id["T_ENCAPSED_AND_WHITESPACE"],yytext,line); return T_ENCAPSED_AND_WHITESPACE; +} + + +"$"[^a-zA-Z_\x7f-\xff{] { + if (yyleng == 2) { + yyless(1); + } + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + + +{ENCAPSED_TOKENS} { + char temp[4] = {'\'', yytext[0], '\'', 0}; + count(); yylvalp->t=new Terminal(name2id[temp],yytext,line); return yytext[0]; +} + +"\\{" { + count(); yylvalp->t=new Terminal(name2id["T_STRING"],yytext,line); return T_STRING; +} + +"{$" { + yy_push_state(ST_IN_SCRIPTING); + yyless(1); + count(); yylvalp->t=new Terminal(name2id["T_CURLY_OPEN"],yytext,line); return T_CURLY_OPEN; +} + + +"\\'" { + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + +"\\\\" { + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + +"\\\"" { + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + +"\\`" { + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + +"\\"[0-7]{1,3} { + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + +"\\x"[0-9A-Fa-f]{1,2} { + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + +"\\"{ANY_CHAR} { + switch (yytext[1]) { //?? Is "\\\n" handled? + case 'n': + break; + case 't': + break; + case 'r': + break; + case '\\': + break; + case '$': + break; + default: + count(); yylvalp->t=new Terminal(name2id["T_BAD_CHARACTER"],yytext,line); return T_BAD_CHARACTER; + } + count(); yylvalp->t=new Terminal(name2id["T_CHARACTER"],yytext,line); return T_CHARACTER; +} + + +["'`]+ { + count(); yylvalp->t=new Terminal(name2id["T_ENCAPSED_AND_WHITESPACE"],yytext,line); return T_ENCAPSED_AND_WHITESPACE; +} + + +["] { + BEGIN(ST_IN_SCRIPTING); + count(); yylvalp->t=new Terminal(name2id["'\"'"],yytext,line); return '\"'; +} + + +[`] { + BEGIN(ST_IN_SCRIPTING); + count(); yylvalp->t=new Terminal(name2id["'`'"],yytext,line); return '`'; +} + + +['] { + BEGIN(ST_IN_SCRIPTING); + count(); yylvalp->t=new Terminal(name2id["'\''"],yytext,line); return '\''; +} + + +<> { + BEGIN(INITIAL); + yyterminate(); +} + +<> { + fprintf(stderr,"Unterminated comment: %s\nEOF at line %d\n", line); + BEGIN(INITIAL); + yyterminate(); +} + + + +{ANY_CHAR} { + fprintf(stderr,"Unexpected character in input: '%c' (ASCII=%d) state=%d\n", yytext[0], yytext[0], YYSTATE); + count(); +} + +%% + +int yywrap() +{ + BEGIN(INITIAL); + return(1); +} + +void count() +{ +/* because of yyless, unput, input, and etc, the following counting may not be appropriate: */ + int i; + + for (i = 0; yytext[i] != '\0'; i++) + if (yytext[i] == '\n') { + column = 0; + line++; + } else if (yytext[i] == '\t') + column += 4; + else + column++; + +} + diff --git a/src/ptgen/simple/Makefile b/src/ptgen/simple/Makefile new file mode 100755 index 0000000..1817b0a --- /dev/null +++ b/src/ptgen/simple/Makefile @@ -0,0 +1,54 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# This is a testing directory; may have a lot of errors in the grammar files + +CXXFLAGS= -I../../include -g +OBJS= lex.yy.o pt_c.tab.o main.o head.o +TARGET= c_ptgen + +TARGET: ${OBJS} + g++ -o ${TARGET} ${OBJS} + +lex.yy.cc: c.l pt_c.tab.cc + flex -olex.yy.cc c.l + +pt_c.tab.cc: pt_c.y + bison -d pt_c.y -o pt_c.tab.cc + +pt_c.y head.cc: c.y + ../gcc/mainc.py c.y + +.PHONY: clean +clean: + rm -f *.o lex.yy.cc pt_c.tab* pt_c.y head.cc $(TARGET) + diff --git a/src/ptgen/simple/c.l b/src/ptgen/simple/c.l new file mode 100755 index 0000000..56495b3 --- /dev/null +++ b/src/ptgen/simple/c.l @@ -0,0 +1,450 @@ +D [0-9] +L [a-zA-Z_] +H [a-fA-F0-9] +E [Ee][+-]?{D}+ +FS (f|F|l|L) +IS (u|U|l|L)* + +%{ +#include +#include +#include "pt_c.tab.hh" +#include +#include +using namespace std; + +extern map name2id; +void count(); +int comment(); +int cpp_comment(); +%} + +%% +"/*" { count(); comment(); } +"//" { count(); cpp_comment(); } + +"auto" { + count(); + yylval.t= new Terminal(name2id["AUTO"],yytext); + return(AUTO); } +"break" { + count(); + yylval.t= new Terminal(name2id["BREAK"],yytext); + return(BREAK); } +"case" { + count(); + yylval.t= new Terminal(name2id["CASE"],yytext); + return(CASE); } +"char" { + count(); + yylval.t= new Terminal(name2id["CHAR"],yytext); + return(CHAR); } +"const" { + count(); + yylval.t= new Terminal(name2id["CONST"],yytext); + return(CONST); } +"continue" { + count(); + yylval.t= new Terminal(name2id["CONTINUE"],yytext); + return(CONTINUE); } +"default" { + count(); + yylval.t= new Terminal(name2id["DEFAULT"],yytext); + return(DEFAULT); } +"do" { + count(); + yylval.t= new Terminal(name2id["DO"],yytext); + return(DO); } +"double" { + count(); + yylval.t= new Terminal(name2id["DOUBLE"],yytext); + return(DOUBLE); } +"else" { + count(); + yylval.t= new Terminal(name2id["ELSE"],yytext); + return(ELSE); } +"enum" { + count(); + yylval.t= new Terminal(name2id["ENUM"],yytext); + return(ENUM); } +"extern" { + count(); + yylval.t= new Terminal(name2id["EXTERN"],yytext); + return(EXTERN); } +"float" { + count(); + yylval.t= new Terminal(name2id["FLOAT"],yytext); + return(FLOAT); } +"for" { + count(); + yylval.t= new Terminal(name2id["FOR"],yytext); + return(FOR); } +"goto" { + count(); + yylval.t= new Terminal(name2id["GOTO"],yytext); + return(GOTO); } +"if" { + count(); + yylval.t= new Terminal(name2id["IF"],yytext); + return(IF); } +"inline" { + count(); + yylval.t= new Terminal(name2id["INLINE"],yytext); + return(INLINE); } +"int" { + count(); + yylval.t= new Terminal(name2id["INT"],yytext); + return(INT); } +"long" { + count(); + yylval.t= new Terminal(name2id["LONG"],yytext); + return(LONG); } +"register" { + count(); + yylval.t= new Terminal(name2id["REGISTER"],yytext); + return(REGISTER); } +"return" { + count(); + yylval.t= new Terminal(name2id["RETURN"],yytext); + return(RETURN); } +"short" { + count(); + yylval.t= new Terminal(name2id["SHORT"],yytext); + return(SHORT); } +"signed" { + count(); + yylval.t= new Terminal(name2id["SIGNED"],yytext); + return(SIGNED); } +"sizeof" { + count(); + yylval.t= new Terminal(name2id["SIZEOF"],yytext); + return(SIZEOF); } +"static" { + count(); + yylval.t= new Terminal(name2id["STATIC"],yytext); + return(STATIC); } +"struct" { + count(); + yylval.t= new Terminal(name2id["STRUCT"],yytext); + return(STRUCT); } +"switch" { + count(); + yylval.t= new Terminal(name2id["SWITCH"],yytext); + return(SWITCH); } +"typedef" { + count(); + yylval.t= new Terminal(name2id["TYPEDEF"],yytext); + return(TYPEDEF); } +"union" { + count(); + yylval.t= new Terminal(name2id["UNION"],yytext); + return(UNION); } +"unsigned" { + count(); + yylval.t= new Terminal(name2id["UNSIGNED"],yytext); + return(UNSIGNED); } +"void" { + count(); + yylval.t= new Terminal(name2id["VOID"],yytext); + return(VOID); } +"volatile" { + count(); + yylval.t= new Terminal(name2id["VOLATILE"],yytext); + return(VOLATILE); } +"while" { + count(); + yylval.t= new Terminal(name2id["WHILE"],yytext); + return(WHILE); } + +{L}({L}|{D})* { + count(); + yylval.t= new Terminal(name2id["IDENTIFIER"],yytext); + return(IDENTIFIER); } + +0[xX]{H}+{IS}? { + count(); + yylval.t= new Terminal(name2id["CONSTANT"],yytext); + return(CONSTANT); } +0{D}+{IS}? { + count(); + yylval.t= new Terminal(name2id["CONSTANT"],yytext); + return(CONSTANT); } +{D}+{IS}? { + count(); + yylval.t= new Terminal(name2id["CONSTANT"],yytext); + return(CONSTANT); } +L?'(\\.|[^\\'])+' { + count(); + yylval.t= new Terminal(name2id["CONSTANT"],yytext); + return(CONSTANT); } + +{D}+{E}{FS}? { + count(); + yylval.t= new Terminal(name2id["CONSTANT"],yytext); + return(CONSTANT); } +{D}*"."{D}+({E})?{FS}? { + count(); + yylval.t= new Terminal(name2id["CONSTANT"],yytext); + return(CONSTANT); } +{D}+"."{D}*({E})?{FS}? { + count(); + yylval.t= new Terminal(name2id["CONSTANT"],yytext); + return(CONSTANT); } + +L?\"(\\.|[^\\"])*\" { + count(); + yylval.t= new Terminal(name2id["STRING_LITERAL"],yytext); + return(STRING_LITERAL); } +"..." { + count(); + yylval.t= new Terminal(name2id["ELLIPSIS"],yytext); + return(ELLIPSIS); } +">>=" { + count(); + yylval.t= new Terminal(name2id["RIGHT_ASSIGN"],yytext); + return(RIGHT_ASSIGN); } +"<<=" { + count(); + yylval.t= new Terminal(name2id["LEFT_ASSIGN"],yytext); + return(LEFT_ASSIGN); } +"+=" { + count(); + yylval.t= new Terminal(name2id["ADD_ASSIGN"],yytext); + return(ADD_ASSIGN); } +"-=" { + count(); + yylval.t= new Terminal(name2id["SUB_ASSIGN"],yytext); + return(SUB_ASSIGN); } +"*=" { + count(); + yylval.t= new Terminal(name2id["MUL_ASSIGN"],yytext); + return(MUL_ASSIGN); } +"/=" { + count(); + yylval.t= new Terminal(name2id["DIV_ASSIGN"],yytext); + return(DIV_ASSIGN); } +"%=" { + count(); + yylval.t= new Terminal(name2id["MOD_ASSIGN"],yytext); + return(MOD_ASSIGN); } +"&=" { + count(); + yylval.t= new Terminal(name2id["AND_ASSIGN"],yytext); + return(AND_ASSIGN); } +"^=" { + count(); + yylval.t= new Terminal(name2id["XOR_ASSIGN"],yytext); + return(XOR_ASSIGN); } +"|=" { + count(); + yylval.t= new Terminal(name2id["OR_ASSIGN"],yytext); + return(OR_ASSIGN); } +">>" { + count(); + yylval.t= new Terminal(name2id["RIGHT_OP"],yytext); + return(RIGHT_OP); } +"<<" { + count(); + yylval.t= new Terminal(name2id["LEFT_OP"],yytext); + return(LEFT_OP); } +"++" { + count(); + yylval.t= new Terminal(name2id["INC_OP"],yytext); + return(INC_OP); } +"--" { + count(); + yylval.t= new Terminal(name2id["DEC_OP"],yytext); + return(DEC_OP); } +"->" { + count(); + yylval.t= new Terminal(name2id["PTR_OP"],yytext); + return(PTR_OP); } +"&&" { + count(); + yylval.t= new Terminal(name2id["AND_OP"],yytext); + return(AND_OP); } +"||" { + count(); + yylval.t= new Terminal(name2id["OR_OP"],yytext); + return(OR_OP); } +"<=" { + count(); + yylval.t= new Terminal(name2id["LE_OP"],yytext); + return(LE_OP); } +">=" { + count(); + yylval.t= new Terminal(name2id["GE_OP"],yytext); + return(GE_OP); } +"==" { + count(); + yylval.t= new Terminal(name2id["EQ_OP"],yytext); + return(EQ_OP); } +"!=" { + count(); + yylval.t= new Terminal(name2id["NE_OP"],yytext); + return(NE_OP); } +";" { + count(); + yylval.t= new Terminal(name2id["';'"],yytext); + return(';'); } +("{"|"<%") { + count(); + yylval.t= new Terminal(name2id["'{'"],yytext); + return('{'); } +("}"|"%>") { + count(); + yylval.t= new Terminal(name2id["'}'"],yytext); + return('}'); } +"," { + count(); + yylval.t= new Terminal(name2id["','"],yytext); + return(','); } +":" { + count(); + yylval.t= new Terminal(name2id["':'"],yytext); + return(':'); } +"=" { + count(); + yylval.t= new Terminal(name2id["'='"],yytext); + return('='); } +"(" { + count(); + yylval.t= new Terminal(name2id["'('"],yytext); + return('('); } +")" { + count(); + yylval.t= new Terminal(name2id["')'"],yytext); + return(')'); } +("["|"<:") { + count(); + yylval.t= new Terminal(name2id["'['"],yytext); + return('['); } +("]"|":>") { + count(); + yylval.t= new Terminal(name2id["']'"],yytext); + return(']'); } +"." { + count(); + yylval.t= new Terminal(name2id["'.'"],yytext); + return('.'); } +"&" { + count(); + yylval.t= new Terminal(name2id["'&'"],yytext); + return('&'); } +"!" { + count(); + yylval.t= new Terminal(name2id["'!'"],yytext); + return('!'); } +"~" { + count(); + yylval.t= new Terminal(name2id["'~'"],yytext); + return('~'); } +"-" { + count(); + yylval.t= new Terminal(name2id["'-'"],yytext); + return('-'); } +"+" { + count(); + yylval.t= new Terminal(name2id["'+'"],yytext); + return('+'); } +"*" { + count(); + yylval.t= new Terminal(name2id["'*'"],yytext); + return('*'); } +"/" { + count(); + yylval.t= new Terminal(name2id["'/'"],yytext); + return('/'); } +"%" { + count(); + yylval.t= new Terminal(name2id["'%'"],yytext); + return('%'); } +"<" { + count(); + yylval.t= new Terminal(name2id["'<'"],yytext); + return('<'); } +">" { + count(); + yylval.t= new Terminal(name2id["'>'"],yytext); + return('>'); } +"^" { + count(); + yylval.t= new Terminal(name2id["'^'"],yytext); + return('^'); } +"|" { + count(); + yylval.t= new Terminal(name2id["'|'"],yytext); + return('|'); } +"?" { + count(); + yylval.t= new Terminal(name2id["'?'"],yytext); + return('?'); } + +[ \t\v\n\f] { + count(); + } +. { /* ignore bad characters */ } + +%% + +int yywrap() +{ + return(1); +} + +int column = 0; +int line = 0; + + + +int comment() +{ + int c, c1; + +loop: + while ((c = getchar()) != '*' && c != 0 && c!= EOF) { + column++; + if ( c == '\n' ) { + column= 0; + line++; + } + } + column++; + + if ((c1 = getchar()) != '/' && c1 != 0 && c1!=EOF) + { + unput(c1); + goto loop; + } + column++; + +// if (c != 0) +// putchar(c1); +} + +int cpp_comment() +{ +int c; + while ((c = getchar()) != '\n' && c != 0 && c!=EOF) + column++; + line++; + column= 0; +} + + +void count() +{ + int i; + + for (i = 0; yytext[i] != '\0'; i++) + if (yytext[i] == '\n') { + column = 0; + line++; + } else if (yytext[i] == '\t') + column += 8 - (column % 8); + else + column++; + + //ECHO; +} + diff --git a/src/ptgen/simple/c.y b/src/ptgen/simple/c.y new file mode 100755 index 0000000..e4966db --- /dev/null +++ b/src/ptgen/simple/c.y @@ -0,0 +1,419 @@ + +primary_expression + : IDENTIFIER + | CONSTANT + | STRING_LITERAL + | '(' expression ')' + ; + +postfix_expression + : primary_expression + | postfix_expression '[' expression ']' + | postfix_expression '(' ')' + | postfix_expression '(' argument_expression_list ')' + | postfix_expression '.' IDENTIFIER + | postfix_expression PTR_OP IDENTIFIER + | postfix_expression INC_OP + | postfix_expression DEC_OP + ; + +argument_expression_list + : assignment_expression + | argument_expression_list ',' assignment_expression + ; + +unary_expression + : postfix_expression + | INC_OP unary_expression + | DEC_OP unary_expression + | unary_operator cast_expression + | SIZEOF unary_expression + | SIZEOF '(' type_name ')' + ; + +unary_operator + : '&' + | '*' + | '+' + | '-' + | '~' + | '!' + ; + +cast_expression + : unary_expression + | '(' type_name ')' cast_expression + ; + +multiplicative_expression + : cast_expression + | multiplicative_expression '*' cast_expression + | multiplicative_expression '/' cast_expression + | multiplicative_expression '%' cast_expression + ; + +additive_expression + : multiplicative_expression + | additive_expression '+' multiplicative_expression + | additive_expression '-' multiplicative_expression + ; + +shift_expression + : additive_expression + | shift_expression LEFT_OP additive_expression + | shift_expression RIGHT_OP additive_expression + ; + +relational_expression + : shift_expression + | relational_expression '<' shift_expression + | relational_expression '>' shift_expression + | relational_expression LE_OP shift_expression + | relational_expression GE_OP shift_expression + ; + +equality_expression + : relational_expression + | equality_expression EQ_OP relational_expression + | equality_expression NE_OP relational_expression + ; + +and_expression + : equality_expression + | and_expression '&' equality_expression + ; + +exclusive_or_expression + : and_expression + | exclusive_or_expression '^' and_expression + ; + +inclusive_or_expression + : exclusive_or_expression + | inclusive_or_expression '|' exclusive_or_expression + ; + +logical_and_expression + : inclusive_or_expression + | logical_and_expression AND_OP inclusive_or_expression + ; + +logical_or_expression + : logical_and_expression + | logical_or_expression OR_OP logical_and_expression + ; + +conditional_expression + : logical_or_expression + | logical_or_expression '?' expression ':' conditional_expression + ; + +assignment_expression + : conditional_expression + | unary_expression assignment_operator assignment_expression + ; + +assignment_operator + : '=' + | MUL_ASSIGN + | DIV_ASSIGN + | MOD_ASSIGN + | ADD_ASSIGN + | SUB_ASSIGN + | LEFT_ASSIGN + | RIGHT_ASSIGN + | AND_ASSIGN + | XOR_ASSIGN + | OR_ASSIGN + ; + +expression + : assignment_expression + | expression ',' assignment_expression + ; + +constant_expression + : conditional_expression + ; + +declaration + : declaration_specifiers ';' + | declaration_specifiers init_declarator_list ';' + | IDENTIFIER declaration_specifiers init_declarator_list ';' + | IDENTIFIER init_declarator_list ';' + | error ';' + ; + +declaration_specifiers + : storage_class_specifier + | storage_class_specifier declaration_specifiers + | type_specifier + | type_specifier declaration_specifiers + | type_qualifier + | type_qualifier declaration_specifiers + | IDENTIFIER declaration_specifiers + ; + +init_declarator_list + : init_declarator + | init_declarator_list ',' init_declarator + ; + +init_declarator + : declarator + | declarator '=' initializer + ; + +storage_class_specifier + : TYPEDEF + | EXTERN + | STATIC + | AUTO + | REGISTER + | INLINE + ; + +type_specifier + : VOID + | CHAR + | SHORT + | INT + | LONG + | FLOAT + | DOUBLE + | SIGNED + | UNSIGNED + | struct_or_union_specifier + | enum_specifier + | TYPE_NAME + ; + +struct_or_union_specifier + : struct_or_union IDENTIFIER '{' struct_declaration_list '}' + | struct_or_union '{' struct_declaration_list '}' + | struct_or_union IDENTIFIER + ; + +struct_or_union + : STRUCT + | UNION + ; + +struct_declaration_list + : struct_declaration + | struct_declaration_list struct_declaration + ; + +struct_declaration + : specifier_qualifier_list struct_declarator_list ';' + | IDENTIFIER struct_declarator_list ';' + | error ';' + ; + +specifier_qualifier_list + : type_specifier specifier_qualifier_list + | type_specifier + | type_qualifier specifier_qualifier_list + | type_qualifier + ; + +struct_declarator_list + : struct_declarator + | struct_declarator_list ',' struct_declarator + ; + +struct_declarator + : declarator + | ':' constant_expression + | declarator ':' constant_expression + ; + +enum_specifier + : ENUM '{' enumerator_list '}' + | ENUM IDENTIFIER '{' enumerator_list '}' + | ENUM IDENTIFIER + ; + +enumerator_list + : enumerator + | enumerator_list ',' enumerator + ; + +enumerator + : IDENTIFIER + | IDENTIFIER '=' constant_expression + ; + +type_qualifier + : CONST + | VOLATILE + ; + +declarator + : pointer direct_declarator + | direct_declarator + ; + +direct_declarator + : IDENTIFIER + | '(' declarator ')' + | direct_declarator '[' constant_expression ']' + | direct_declarator '[' error ']' + | direct_declarator '[' ']' + | direct_declarator '(' parameter_type_list ')' + | direct_declarator '(' identifier_list ')' + | direct_declarator '(' ')' + ; + +pointer + : '*' + | '*' type_qualifier_list + | '*' pointer + | '*' type_qualifier_list pointer + ; + +type_qualifier_list + : type_qualifier + | type_qualifier_list type_qualifier + ; + + +parameter_type_list + : parameter_list + | parameter_list ',' ELLIPSIS + ; + +parameter_list + : parameter_declaration + | parameter_list ',' parameter_declaration + ; + +parameter_declaration + : declaration_specifiers declarator + | declaration_specifiers abstract_declarator + | declaration_specifiers + | IDENTIFIER declarator + | IDENTIFIER declaration_specifiers declarator + ; + +identifier_list + : IDENTIFIER + | identifier_list ',' IDENTIFIER + ; + +type_name + : specifier_qualifier_list + | specifier_qualifier_list abstract_declarator + ; + +abstract_declarator + : pointer + | direct_abstract_declarator + | pointer direct_abstract_declarator + ; + +direct_abstract_declarator + : '(' abstract_declarator ')' + | '[' ']' + | '[' constant_expression ']' + | direct_abstract_declarator '[' ']' + | direct_abstract_declarator '[' constant_expression ']' + | '(' ')' + | '(' parameter_type_list ')' + | direct_abstract_declarator '(' ')' + | direct_abstract_declarator '(' parameter_type_list ')' + ; + +initializer + : assignment_expression + | '{' initializer_list '}' + | '{' initializer_list ',' '}' + ; + +initializer_list + : initializer + | initializer_list ',' initializer + ; + +statement + : labeled_statement + | compound_statement + | expression_statement + | selection_statement + | iteration_statement + | jump_statement + | error ';' + ; + +labeled_statement + : IDENTIFIER ':' statement + | CASE constant_expression ':' statement + | DEFAULT ':' statement + ; + +compound_statement + : '{' '}' + | '{' statement_list '}' + | '{' declaration_list '}' + | '{' declaration_list statement_list '}' + | '{' error '}' + ; + +declaration_list + : declaration + | declaration_list declaration + ; + +statement_list + : statement + | statement_list statement + ; + +expression_statement + : ';' + | expression ';' + ; + +selection_statement + : IF '(' expression ')' statement + | IF '(' expression ')' statement ELSE statement + | SWITCH '(' expression ')' statement + ; + +iteration_statement + : WHILE '(' expression ')' statement + | DO statement WHILE '(' expression ')' ';' + | FOR '(' expression_statement expression_statement ')' statement + | FOR '(' expression_statement expression_statement expression ')' statement + ; + +jump_statement + : GOTO IDENTIFIER ';' + | CONTINUE ';' + | BREAK ';' + | RETURN ';' + | RETURN expression ';' + ; + +translation_unit + : external_declaration + | translation_unit external_declaration + ; + +external_declaration + : function_definition + | declaration + ; + +idlist + : IDENTIFIER idlist + | + ; + +function_definition + : declaration_specifiers declarator declaration_list compound_statement + | declaration_specifiers declarator compound_statement + | declarator declaration_list compound_statement + | declarator compound_statement + ; + diff --git a/src/ptgen/simple/c.y.foot b/src/ptgen/simple/c.y.foot new file mode 100644 index 0000000..989f003 --- /dev/null +++ b/src/ptgen/simple/c.y.foot @@ -0,0 +1,13 @@ + +#include + +extern char yytext[]; +extern int column; + +yyerror(s) +char *s; +{ + fflush(stdout); + printf("%s: %d.%d",s,yylineno,column); +} + diff --git a/src/ptgen/simple/c.y.head b/src/ptgen/simple/c.y.head new file mode 100644 index 0000000..e2e47c7 --- /dev/null +++ b/src/ptgen/simple/c.y.head @@ -0,0 +1,14 @@ +%token IDENTIFIER CONSTANT STRING_LITERAL SIZEOF +%token PTR_OP INC_OP DEC_OP LEFT_OP RIGHT_OP LE_OP GE_OP EQ_OP NE_OP +%token AND_OP OR_OP MUL_ASSIGN DIV_ASSIGN MOD_ASSIGN ADD_ASSIGN +%token SUB_ASSIGN LEFT_ASSIGN RIGHT_ASSIGN AND_ASSIGN +%token XOR_ASSIGN OR_ASSIGN TYPE_NAME + +%token TYPEDEF EXTERN STATIC AUTO REGISTER +%token CHAR SHORT INT LONG SIGNED UNSIGNED FLOAT DOUBLE CONST VOLATILE VOID +%token STRUCT UNION ENUM ELLIPSIS + +%token CASE DEFAULT IF ELSE SWITCH WHILE DO FOR GOTO CONTINUE BREAK RETURN + +%start translation_unit + diff --git a/src/ptgen/simple/main.cc b/src/ptgen/simple/main.cc new file mode 100755 index 0000000..ae8f814 --- /dev/null +++ b/src/ptgen/simple/main.cc @@ -0,0 +1,60 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include + +using namespace std; + +map name2id; +map id2name; + +int yyparse(); + +extern Tree *root; + +void id_init(); + +int main() +{ + id_init(); + yyparse(); + if (!root) { + cerr << "failed to parse file" << endl; + return 1; + } + + root->printTok(); + return 0; +} + diff --git a/src/ptgen/simple/tokid.h b/src/ptgen/simple/tokid.h new file mode 100755 index 0000000..68cd5f6 --- /dev/null +++ b/src/ptgen/simple/tokid.h @@ -0,0 +1,179 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#define ID_expression_statement 1 +#define ID_storage_class_specifier 2 +#define ID_'~' 3 +#define ID_struct_or_union_specifier 4 +#define ID_VOID 5 +#define ID_';' 6 +#define ID_init_declarator_list 7 +#define ID_DIV_ASSIGN 8 +#define ID_pointer 9 +#define ID_LE_OP 10 +#define ID_inclusive_or_expression 11 +#define ID_LONG 12 +#define ID_'(' 13 +#define ID_ELLIPSIS 14 +#define ID_struct_declaration 15 +#define ID_GOTO 16 +#define ID_':' 17 +#define ID_ENUM 18 +#define ID_LEFT_OP 19 +#define ID_parameter_list 20 +#define ID_DOUBLE 21 +#define ID_labeled_statement 22 +#define ID_abstract_declarator 23 +#define ID_init_declarator 24 +#define ID_direct_abstract_declarator 25 +#define ID_INC_OP 26 +#define ID_NE_OP 27 +#define ID_SHORT 28 +#define ID_CONSTANT 29 +#define ID_unary_expression 30 +#define ID_DEC_OP 31 +#define ID_struct_or_union 32 +#define ID_initializer 33 +#define ID_struct_declaration_list 34 +#define ID_'%' 35 +#define ID_EQ_OP 36 +#define ID_SIZEOF 37 +#define ID_declaration_list 38 +#define ID_'+' 39 +#define ID_type_specifier 40 +#define ID_'^' 41 +#define ID_compound_statement 42 +#define ID_iteration_statement 43 +#define ID_UNSIGNED 44 +#define ID_OR_OP 45 +#define ID_'*' 46 +#define ID_'=' 47 +#define ID_direct_declarator 48 +#define ID_translation_unit 49 +#define ID_external_declaration 50 +#define ID_FOR 51 +#define ID_UNION 52 +#define ID_multiplicative_expression 53 +#define ID_declarator 54 +#define ID_argument_expression_list 55 +#define ID_'{' 56 +#define ID_ELSE 57 +#define ID_logical_and_expression 58 +#define ID_shift_expression 59 +#define ID_SUB_ASSIGN 60 +#define ID_struct_declarator 61 +#define ID_XOR_ASSIGN 62 +#define ID_INT 63 +#define ID_type_qualifier 64 +#define ID_'-' 65 +#define ID_SIGNED 66 +#define ID_CONTINUE 67 +#define ID_MUL_ASSIGN 68 +#define ID_assignment_operator 69 +#define ID_statement_list 70 +#define ID_expression 71 +#define ID_'<' 72 +#define ID_RIGHT_OP 73 +#define ID_type_name 74 +#define ID_RIGHT_ASSIGN 75 +#define ID_DEFAULT 76 +#define ID_exclusive_or_expression 77 +#define ID_CHAR 78 +#define ID_WHILE 79 +#define ID_EXTERN 80 +#define ID_RETURN 81 +#define ID_additive_expression 82 +#define ID_CASE 83 +#define ID_REGISTER 84 +#define ID_',' 85 +#define ID_SWITCH 86 +#define ID_AND_ASSIGN 87 +#define ID_relational_expression 88 +#define ID_statement 89 +#define ID_cast_expression 90 +#define ID_'?' 91 +#define ID_IDENTIFIER 92 +#define ID_struct_declarator_list 93 +#define ID_ADD_ASSIGN 94 +#define ID_constant_expression 95 +#define ID_'>' 96 +#define ID_parameter_declaration 97 +#define ID_GE_OP 98 +#define ID_primary_expression 99 +#define ID_declaration 100 +#define ID_'&' 101 +#define ID_equality_expression 102 +#define ID_STRUCT 103 +#define ID_jump_statement 104 +#define ID_MOD_ASSIGN 105 +#define ID_idlist 106 +#define ID_'/' 107 +#define ID_TYPE_NAME 108 +#define ID_OR_ASSIGN 109 +#define ID_enumerator_list 110 +#define ID_'[' 111 +#define ID_BREAK 112 +#define ID_VOLATILE 113 +#define ID_INLINE 114 +#define ID_'}' 115 +#define ID_DO 116 +#define ID_STATIC 117 +#define ID_CONST 118 +#define ID_'!' 119 +#define ID_enumerator 120 +#define ID_']' 121 +#define ID_and_expression 122 +#define ID_specifier_qualifier_list 123 +#define ID_selection_statement 124 +#define ID_postfix_expression 125 +#define ID_'.' 126 +#define ID_PTR_OP 127 +#define ID_initializer_list 128 +#define ID_logical_or_expression 129 +#define ID_unary_operator 130 +#define ID_'|' 131 +#define ID_TYPEDEF 132 +#define ID_assignment_expression 133 +#define ID_parameter_type_list 134 +#define ID_AUTO 135 +#define ID_type_qualifier_list 136 +#define ID_AND_OP 137 +#define ID_declaration_specifiers 138 +#define ID_identifier_list 139 +#define ID_IF 140 +#define ID_function_definition 141 +#define ID_STRING_LITERAL 142 +#define ID_enum_specifier 143 +#define ID_FLOAT 144 +#define ID_')' 145 +#define ID_conditional_expression 146 +#define ID_LEFT_ASSIGN 147 diff --git a/src/ptgen/yacc.g b/src/ptgen/yacc.g new file mode 100755 index 0000000..fcf9013 --- /dev/null +++ b/src/ptgen/yacc.g @@ -0,0 +1,156 @@ +// +// +// Copyright (c) 2007-2010, +// Lingxiao Jiang +// Ghassan Misherghi +// Zhendong Su +// Stephane Glondu +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the University of California nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// +header "YaccParser.__init__" { +self.NonTerminals= set([]) +self.Terminals= set([]) +self.Rules=[] +} + +header "YaccLexer.__init__" { +} + +options { +language = "Python"; +} + + +class YaccParser extends Parser; +options { k=1; } + +grammar: (rule)+ EOF; + +rule: +(id:ID) { +self.NonTerminals.add(id.getText()) +} COLON +rhs[id.getText()] SEMICOLON +; + +rhs [lhs] {right=[]}: +( (id:ID) { + right.append(("node",id.getText())) + if id.getText() == id.getText().lower(): + self.NonTerminals.add(id.getText()) + else: + self.Terminals.add(id.getText()) + } + | (c:CHAR) { + right.append(("node",c.getText())) + self.Terminals.add(c.getText()) + } + | (str:STRING) { + right.append(("node",str.getText())) + self.Terminals.add(str.getText()) + } + | ERROR { right.append(("error","error")) } + | PREC ((pi:ID) {right.append(("%prec","%prec "+pi.getText()))}| + (pc:CHAR) {right.append(("%prec","%prec "+pc.getText()))} + ) + | ACTION //ignore actions in grammar +)* +{ +self.Rules.append( (lhs,right) ) +} +(OR rhs[lhs])? ; + +rulespec: HYPHEN ; + +treespec: CARROT | BANG ; + +class YaccLexer extends Lexer; +options { k=7; testLiterals=true; } + +COLON: ':'; +SEMICOLON: ';'; +HYPHEN: '-'; +CARROT: '^'; +BANG: '!'; +OR: '|'; +PREC: "%prec"; +ERROR: "error"; +ID: LETTER (LETTER | DIGIT | '_' | '.')*; +protected +LETTER: ('a'..'z'|'A'..'Z'); +protected +DIGIT: ('0'..'9'); +//CHAR: '\'' (~'\'')+ '\''; // failed to recognize '\'' +CHAR: '\'' (('\\' .)|(~('\''|'\\'))) '\''; +STRING: '"' (~'"')* '"'; +ACTION: '{' // naively handle quoted braces: + { + lcount= 1 + incomment= False + indquote= False + insquote= False + while lcount != 0: + if self.LA(1) == '\\': + self.consume() + elif self.LA(1) == '/' and self.LA(2) == '*': + if not indquote and not insquote: + incomment= True + self.consume() + elif self.LA(1) == '*' and self.LA(2) == '/': + if not indquote and not insquote: + incomment= False + self.consume() + elif self.LA(1) == '\'': + if not indquote and not incomment: + insquote= not insquote + elif self.LA(1) == '"': + if not insquote and not incomment: + indquote= not indquote + elif self.LA(1) == antlr.EOF: + $setType(antlr.EOF) + elif self.LA(1) == '\n': + self.newline() + elif not indquote and not insquote and not incomment: + if self.LA(1)== '}': + lcount -= 1 + elif self.LA(1)== '{': + lcount += 1 + self.consume() + $setType(antlr.SKIP); + } + ; + +COMMENT: "/*" (~('*')|('*')~('/'))* "*/" {$setType(antlr.SKIP)}; +//doesn't recognize '//' though + +WS : ( ' ' + | '\r' '\n' { self.newline() } + | '\n' { self.newline() } + | '\t' + )+ + { $setType(antlr.SKIP) } + ; + diff --git a/src/vgen/samplevec b/src/vgen/samplevec new file mode 100755 index 0000000..e712d01 --- /dev/null +++ b/src/vgen/samplevec @@ -0,0 +1,21 @@ +# FILE:./init/initramfs.c, LINE:14, OFFSET:14, NODE_KIND:51, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:3, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{x:1, message:1, }2, OIDs:{message, x, }2 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/initramfs.c, LINE:13, OFFSET:14, NODE_KIND:54, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:7, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{x:1, message:2, }2, OIDs:{message, message, x, }3 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 2 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/initramfs.c, LINE:13, OFFSET:14, NODE_KIND:56, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:7, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{x:1, message:2, }2, OIDs:{message, message, x, }3 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 2 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/initramfs.c, LINE:12, OFFSET:15, NODE_KIND:34, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:9, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{x:2, message:2, error:1, }3, OIDs:{error, x, message, message, x, }5 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 2 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/initramfs.c, LINE:19, OFFSET:19, NODE_KIND:56, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:8, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{GFP_KERNEL:2, size:2, kmalloc:2, }3, OIDs:{kmalloc, size, GFP_KERNEL, kmalloc, size, GFP_KERNEL, }6 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/initramfs.c, LINE:18, OFFSET:20, NODE_KIND:29, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:9, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{GFP_KERNEL:2, malloc:1, size:2, kmalloc:2, }4, OIDs:{malloc, kmalloc, size, GFP_KERNEL, kmalloc, size, GFP_KERNEL, }7 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 2 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/main.c, LINE:101, OFFSET:101, NODE_KIND:56, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:0, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{}0, OIDs:{}0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/main.c, LINE:101, OFFSET:101, NODE_KIND:29, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:1, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{acpi_early_init:1, }1, OIDs:{acpi_early_init, }1 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/main.c, LINE:143, OFFSET:143, NODE_KIND:51, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:3, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{max_cpus:1, }1, OIDs:{max_cpus, }1 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +# FILE:./init/main.c, LINE:144, OFFSET:144, NODE_KIND:133, CONTEXT_KIND:210, NEIGHBOR_KIND:210, NUM_NODE:4, NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, VARs:{}0, OIDs:{}0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + diff --git a/src/vgen/tra-gen-test.C b/src/vgen/tra-gen-test.C new file mode 100755 index 0000000..0f4e602 --- /dev/null +++ b/src/vgen/tra-gen-test.C @@ -0,0 +1,45 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "../include/ptree.h" +#include "treeTra/tra-gen.h" +#include + +int main() { + ParseTree * t; + + TraGenMain tg(t, NULL, fopen("test", "w")); + + tg.run(); + + return 0; +} diff --git a/src/vgen/treeTra/Makefile b/src/vgen/treeTra/Makefile new file mode 100755 index 0000000..ee8be71 --- /dev/null +++ b/src/vgen/treeTra/Makefile @@ -0,0 +1,76 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +CC=gcc #-g -pg +CFLAGS:=${CFLAGS} -O3 +CXX=g++ #-g -pg +CXXFLAGS:=${CXXFLAGS} -O3 + +treeheaders = ../../include/ptree.h +traversalsrc = tree-traversal.C tra-gen.C vgen-utils.c tree-vector.C vgen-config.C \ + token-counter.C sq-tree.C node-vec-gen.C vector-output.C vector-merger.C tree-accessor.C \ + token-tree-map.C clone-context-php.C +traversalobjs:= $(traversalsrc:.C=.o) +traversalobjs:= $(traversalobjs:.c=.o) +traversalmakefiles:= $(traversalsrc:.C=.d) +traversalmakefiles:= $(traversalmakefiles:.c=.d) +traversallib = libvgen.a + +.PHONY: all clean cleanmakefiles + +all: $(traversalmakefiles) $(traversalobjs) $(traversallib) + +$(traversallib): $(traversalobjs) + ar -qcs libvgen.a $(traversalobjs) + +# currently, this Makefile has problems if *.h (not *.c/*.C) is +# changed to include (other) different header files. At such cases, we +# need "rm -f *.d" first (make cleanmakefiles doesn't work, why?), +# then make. + +%.d:%.C + $(CXX) -MM $(CPPFLAGS) $(CXXFLAGS) $< > $@.$$$$; \ + sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \ + rm -f $@.$$$$ + +%.d:%.c + $(CC) -MM $(CPPFLAGS) $(CFLAGS) $< > $@.$$$$; \ + sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; \ + rm -f $@.$$$$ + +include $(traversalmakefiles) + +clean: + rm -f *~ *.o *.d *.a core* gmon* + +cleanmakefiles: + rm -f *.d diff --git a/src/vgen/treeTra/clone-context-php.C b/src/vgen/treeTra/clone-context-php.C new file mode 100755 index 0000000..130bc43 --- /dev/null +++ b/src/vgen/treeTra/clone-context-php.C @@ -0,0 +1,417 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "clone-context-php.h" + +/************************************************** + * Implementation of ContextInconsistency_PHP + * + *************************************************/ + +CloneContextT ContextInconsistency_PHP:: +getContext2(std::pair tokenrange, std::string & fn) +{ + CloneContextT rsl; + ParseTree* pt = NULL; + map::iterator fn_itr = fn2tree.find(fn); + if ( fn_itr == fn2tree.end() ) { + pt = parseFile(fn.c_str()); + if ( pt!=NULL ) { + fn2tree.insert(pair(fn, pt)); + } + } else + pt = (*fn_itr).second; + + rsl.context_node_begin = rsl.context_node_end = NULL; + rsl.context_node_begin = tokenRange2Tree2(tokenrange, pt); + if ( rsl.context_node_begin!=NULL ) { + // if tokenrange is within (actual subrange) context_node_begin, then start from context_node_begin; + // otherwise, start searching from the parent of context_node_begin; + map::iterator attr_itr = rsl.context_node_begin->attributes.find(NODE_TOKEN_ID); + assert ( attr_itr!=rsl.context_node_begin->attributes.end() ); + pair* startrange = (pair*)(*attr_itr).second; + Tree* startnode = rsl.context_node_begin; + if ( tokenrange.first <= startrange->first && tokenrange.second >= startrange->second ) { + startnode = rsl.context_node_begin->parent; // TODO: this may still not guarantee *proper* enclosure + } + filter_unticked_stmt: + while ( startnode!=NULL ) { + if ( isContextual(startnode) ) { // this condition is language-dependant + rsl.context_node_begin = startnode; + break; + } else + startnode = startnode->parent; + } + if ( startnode!=NULL && startnode->type == getTypeID(name2id, "unticked_statement") + && startnode->children[0]->type != getTypeID(name2id, "T_IF") + && startnode->children[0]->type != getTypeID(name2id, "T_WHILE") + && startnode->children[0]->type != getTypeID(name2id, "T_DO") + && startnode->children[0]->type != getTypeID(name2id, "T_FOR") + && startnode->children[0]->type != getTypeID(name2id, "T_SWITCH") + && startnode->children[0]->type != getTypeID(name2id, "T_FOREACH") + && startnode->children[0]->type != getTypeID(name2id, "T_TRY") ) { + startnode = startnode->parent; + goto filter_unticked_stmt; + } + if ( startnode==NULL ) // at least, rsl.context_node_begin should be the "root": + rsl.context_node_begin = pt->getRoot(); + + } else + rsl.context_node_begin = pt ? pt->getRoot() : NULL; + + // assert ( rsl.context_node_begin!=NULL && rsl.context_node_end==NULL ); + return rsl; +} + +Tree* ContextInconsistency_PHP:: +getContextNode(Tree* node) +{ + return node; +} + +Tree* ContextInconsistency_PHP:: +getContextParent(Tree* node) +{ + if ( node == NULL ) + return NULL; + Tree * rsl = getContextNode(node); + rsl = rsl->parent; + filter_unticked_stmt: + while ( rsl!=NULL ) { + if ( isContextual(rsl) ) { // this condition is language-dependant + break; + } else + rsl = rsl->parent; + } + if ( rsl!=NULL && rsl->type == getTypeID(name2id, "unticked_statement") + && rsl->children[0]->type != getTypeID(name2id, "T_IF") + && rsl->children[0]->type != getTypeID(name2id, "T_WHILE") + && rsl->children[0]->type != getTypeID(name2id, "T_DO") + && rsl->children[0]->type != getTypeID(name2id, "T_FOR") + && rsl->children[0]->type != getTypeID(name2id, "T_SWITCH") + && rsl->children[0]->type != getTypeID(name2id, "T_FOREACH") + && rsl->children[0]->type != getTypeID(name2id, "T_TRY") ) { + rsl = rsl->parent; + goto filter_unticked_stmt; + } + return rsl; +} + +bool ContextInconsistency_PHP:: +comparePHPContext2(CloneContextT & context1, CloneContextT & context2) +{ + // only compare the context_node_begin; + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else { // actual comparison here: + Tree* forcompare1 = context1.context_node_begin; + Tree* forcompare2 = context2.context_node_begin; + // handle "unticked_statement": + if ( forcompare1->type == getTypeID(name2id, "unticked_statement") ) + forcompare1 = forcompare1->children[0]; + if ( forcompare2->type == getTypeID(name2id, "unticked_statement") ) + forcompare2 = forcompare2->children[0]; + // treat "for"=="while", "if"=="switch"; it's grammar-specific: + if ( forcompare1->type == getTypeID(name2id, "elseif_list") || + forcompare1->type == getTypeID(name2id, "new_elseif_list") || + forcompare1->type == getTypeID(name2id, "T_IF") || + forcompare1->type == getTypeID(name2id, "T_SWITCH") ) + if ( forcompare2->type == getTypeID(name2id, "elseif_list") || + forcompare2->type == getTypeID(name2id, "new_elseif_list") || + forcompare2->type == getTypeID(name2id, "T_IF") || + forcompare2->type == getTypeID(name2id, "T_SWITCH") ) + return true; + else + return false; + else if ( forcompare1->type == getTypeID(name2id, "T_WHILE") || + forcompare1->type == getTypeID(name2id, "T_DO") || + forcompare1->type == getTypeID(name2id, "T_FOR") ) + if ( forcompare2->type == getTypeID(name2id, "T_WHILE") || + forcompare2->type == getTypeID(name2id, "T_DO") || + forcompare2->type == getTypeID(name2id, "T_FOR") ) + return true; + else + return false; + else if ( forcompare1->type == forcompare2->type ) + return true; + else + return false; + } +} + +Tree* ContextInconsistency_PHP:: +get_condition_within(Tree* node) +{ + Tree* rsl = NULL; + if ( node==NULL ) + return rsl; + + if ( node->type == getTypeID(name2id, "unticked_statement") ) + if ( node->children[0]->type == getTypeID(name2id, "T_IF") ) + rsl = node->children[2]; + else if ( node->children[0]->type == getTypeID(name2id, "T_WHILE") ) + rsl = node->children[2]; + else if ( node->children[0]->type == getTypeID(name2id, "T_DO") ) + rsl = node->children[4]; + else if ( node->children[0]->type == getTypeID(name2id, "T_FOR") ) { + rsl = node->children[4]; // for_expr + if ( rsl->children.size()>0 ) { + rsl = rsl->children[0]; + if ( rsl->children.size()>2 ) + rsl = rsl->children[2]; + else + rsl = rsl->children[0]; + } else + rsl = NULL; + } else if ( node->children[0]->type == getTypeID(name2id, "T_SWITCH") ) + rsl = node->children[2]; + else + rsl = NULL; + else if ( node->type == getTypeID(name2id, "elseif_list") || + node->type == getTypeID(name2id, "new_elseif_list") ) + if ( node->children.size()>0 ) + rsl = node->children[3]; + else + rsl = NULL; + else + rsl = NULL; + + return rsl; +} + +bool ContextInconsistency_PHP:: +comparePHPConditions(CloneContextT& context1, CloneContextT& context2) +{ + // only compare the context_node_begin; + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else { // actual condition comparison here: + Tree* forcompare1 = context1.context_node_begin; + Tree* forcompare2 = context2.context_node_begin; + + forcompare1 = get_condition_within(forcompare1); + forcompare2 = get_condition_within(forcompare2); + return compareTree(forcompare1, forcompare2); // exact match is not good: an extra pair of () may change the result. + } +} + +Tree* ContextInconsistency_PHP:: +get_conditional_operator(Tree* node) +{ + Tree * rsl = NULL; + if ( node == NULL ) + return rsl; + if ( node->type == getTypeID(name2id, "expr") + || node->type == getTypeID(name2id, "r_variable") ) + return get_conditional_operator(node->children[0]); + else if ( node->type == getTypeID(name2id, "expr_without_variable") ) + if ( node->children[0]->type == getTypeID(name2id, "T_LIST") ) + return node->children[4]; + else if ( node->children[0]->type == getTypeID(name2id, "variable") + || node->children[0]->type == getTypeID(name2id, "rw_variable") + || node->children[0]->type == getTypeID(name2id, "expr") ) + return node->children[1]; + else if ( node->children[0]->type == getTypeID(name2id, "'('") ) + return get_conditional_operator(node->children[1]); + else if ( node->children[0]->type == getTypeID(name2id, "internal_functions_in_yacc") ) + return node->children[0]->children[0]; + else if ( node->children[0]->type == getTypeID(name2id, "scalar") ) + return node->children[0]; // return the "scalar"; + else + return node->children[0]; + else if ( node->type == getTypeID(name2id, "variable") ) + return node; + else + return node; + return rsl; +} + +bool ContextInconsistency_PHP:: +isMainOperator(Tree * op1) +{ + if ( op1==NULL ) + return false; + else + return false; +} + +bool ContextInconsistency_PHP:: +filter1() +{ + vector groups; + GetContextFuncT getContext = &ContextInconsistency_PHP::getContext2; + CompareContextFuncT compareContext = &ContextInconsistency_PHP::comparePHPContext2; + + for (int i = 0; i < clusterbuffer.size(); i++) { + int j = 0; + int tmps = groups.size(); + for (; j < tmps; j++) { + CloneContextT context1 = ((*this).*getContext)(pair(groups[j]->tbid, groups[j]->teid), groups[j]->filename); + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + if ( (this->*compareContext)(context1, context2) ) { + // same context, so no need to store it: + break; + } + } + if ( j >= tmps ) { + // a different context: + groups.push_back(&clusterbuffer[i]); + } + } + + if ( groups.size() > 1 ) + return false; // ENUM_RANK_CXT_NODE; + else + return true; // ENUM_RANK_NOTHING; +} + +bool ContextInconsistency_PHP:: +filter2() +{ + vector groups; + GetContextFuncT getContext = &ContextInconsistency_PHP::getContext2; + CompareContextFuncT compareCondition = &ContextInconsistency_PHP::comparePHPConditions; + + for (int i = 0; i < clusterbuffer.size(); i++) { + int j = 0; + int tmps = groups.size(); + for (; j < tmps; j++) { + CloneContextT context1 = ((*this).*getContext)(pair(groups[j][0]->tbid, groups[j][0]->teid), groups[j][0]->filename); + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + if ( (this->*compareCondition)(context1, context2) ) { + groups[j].push_back(&clusterbuffer[i]); + break; + } + } + if ( j >= tmps ) { + groups.push_back(ClonePointGroupT()); + groups.back().push_back(&clusterbuffer[i]); + } + } + + if ( groups.size() <= 1 ) { + return true; // ENUM_RANK_NOTHING + } else + return false; // ENUM_RANK_CXT_COND +} + +int ContextInconsistency_PHP:: +buggy1() +{ + int hasCond = 0, hasLoop = 0, hasNone = 0, hasSwitch = 0, hasTry = 0; + int condInLoop = 0, ifInLevels = 0; + GetContextFuncT getContext = &ContextInconsistency_PHP::getContext2; + + for (int i = 0; i < clusterbuffer.size(); i++) { + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + Tree* forcompare2 = context2.context_node_begin; + if ( forcompare2 == NULL ) + continue; + Tree* contextparent = getContextParent(forcompare2); + + if ( ( forcompare2->type == getTypeID(name2id, "unticked_statement") && + ( forcompare2->children[0]->type == getTypeID(name2id, "T_IF") || + forcompare2->children[0]->type == getTypeID(name2id, "T_SWITCH") ) ) || + forcompare2->type == getTypeID(name2id, "elseif_list") || + forcompare2->type == getTypeID(name2id, "new_elseif_list") ) { + hasCond++; + if ( forcompare2->type == getTypeID(name2id, "unticked_statement") && + forcompare2->children[0]->type == getTypeID(name2id, "T_SWITCH") ) + hasSwitch++; // hasSwitch <= hasCond + else // if_* + if ( contextparent!=NULL && ( ( contextparent->type == getTypeID(name2id, "unticked_statement") && + ( contextparent->children[0]->type == getTypeID(name2id, "T_IF") || + contextparent->children[0]->type == getTypeID(name2id, "T_WHILE") || + contextparent->children[0]->type == getTypeID(name2id, "T_DO") || + contextparent->children[0]->type == getTypeID(name2id, "T_FOR") || + contextparent->children[0]->type == getTypeID(name2id, "T_SWITCH") || + contextparent->children[0]->type == getTypeID(name2id, "T_FOREACH") || + contextparent->children[0]->type == getTypeID(name2id, "T_TRY") ) ) || + contextparent->type == getTypeID(name2id, "elseif_list") || + contextparent->type == getTypeID(name2id, "new_elseif_list") ) ) + ifInLevels++; // ifInLevels <= hasCond-hasSwitch + if ( contextparent!=NULL && ( contextparent->type == getTypeID(name2id, "unticked_statement") && + ( contextparent->children[0]->type == getTypeID(name2id, "T_WHILE") || + contextparent->children[0]->type == getTypeID(name2id, "T_DO") || + contextparent->children[0]->type == getTypeID(name2id, "T_FOR") || + contextparent->children[0]->type == getTypeID(name2id, "T_FOREACH") ) ) ) + condInLoop++; + } else if ( forcompare2->type == getTypeID(name2id, "unticked_statement") && + ( forcompare2->children[0]->type == getTypeID(name2id, "T_WHILE") || + forcompare2->children[0]->type == getTypeID(name2id, "T_DO") || + forcompare2->children[0]->type == getTypeID(name2id, "T_FOR") || + forcompare2->children[0]->type == getTypeID(name2id, "T_FOREACH") ) ) + hasLoop++; + else if ( forcompare2->type == getTypeID(name2id, "additional_catch") || + ( forcompare2->type == getTypeID(name2id, "unticked_statement") && + forcompare2->children[0]->type == getTypeID(name2id, "T_TRY") ) ) + hasTry++; + else + hasNone++; + } + + setscores: + if ( hasLoop>0 && hasCond>0 ) + buggy_score[3] = condInLoop; + + if ( hasTry!=0 && hasTry!=clusterbuffer.size() ) + return 2; + else if ( hasCond>0 && hasLoop==0 && hasNone==0 ) + return 4; + else if ( hasCond>0 && hasLoop==0 && hasNone>0 ) + if ( hasSwitch==hasCond || // switch vs. none + ifInLevels==hasCond-hasSwitch // all "if"s are deep inside + ) + return -5; + else + return 6; + else if ( hasCond>0 && hasLoop>0 && hasNone==0 ) + return 1; + else if ( hasCond>0 && hasLoop>0 && hasNone>0 ) + return 0; + else if ( hasCond==0 && hasLoop>0 && hasNone>0 ) + return -4; + else if ( hasCond==0 && hasLoop>0 && hasNone==0 ) + return -2; + else + return -8; +} diff --git a/src/vgen/treeTra/clone-context-php.h b/src/vgen/treeTra/clone-context-php.h new file mode 100755 index 0000000..edfcc8b --- /dev/null +++ b/src/vgen/treeTra/clone-context-php.h @@ -0,0 +1,59 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _CLONE_CONTEXT_PHP_H_ +#define _CLONE_CONTEXT_PHP_H_ + +#include "token-tree-map.h" + +class ContextInconsistency_PHP : public TokenTreeMap { + public: + typedef CloneContextT (ContextInconsistency_PHP::*GetContextFuncT)(std::pair, std::string &); + typedef bool (ContextInconsistency_PHP::*CompareContextFuncT)(CloneContextT&, CloneContextT&); + + virtual CloneContextT getContext2(std::pair tokenrange, std::string & fn); + virtual Tree* getContextNode(Tree* node); /* get the highest node which represents the context type of "node"; language dependent */ + virtual Tree* getContextParent(Tree* node); /* get the contextual parent; return NULL if not found. */ + + bool comparePHPContext2(CloneContextT& context1, CloneContextT& context2); /* for Php only */ + bool comparePHPConditions(CloneContextT& context1, CloneContextT& context2); /* for Php only, compare conditions of "if" etc. */ + virtual Tree* get_conditional_operator(Tree* node); /* grammar-dependent */ + virtual bool isMainOperator(Tree* node); /* grammar-dependent */ + virtual Tree* get_condition_within(Tree* node); /* grammar-dependent */ + + virtual bool filter1(); + virtual bool filter2(); + // virtual bool filter3(); + virtual int buggy1(); +}; + +#endif diff --git a/src/vgen/treeTra/node-vec-gen.C b/src/vgen/treeTra/node-vec-gen.C new file mode 100755 index 0000000..f9cbff4 --- /dev/null +++ b/src/vgen/treeTra/node-vec-gen.C @@ -0,0 +1,119 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "node-vec-gen.h" + +using namespace std; + +/****************************************** + * Implementation for VecGenerator: + * A basic vector counter. + * + *****************************************/ +VecGenerator:: +VecGenerator(TraGenConfiguration & cfg) + : vecGen_config(cfg) +{ +} + +bool VecGenerator:: +skipNode(Tree* astNode, Tree* inh) +{ + // skippable nodes are not counted in vectors: + return vecGen_config.isSkippable(astNode); +} + +bool VecGenerator:: +skipSubTree(Tree* astNode, Tree* inh) +{ + return false; +} + +Tree* VecGenerator:: +evaluateInheritedAttribute(Tree* astNode, Tree* inh) +{ + return astNode; +} + +TreeVector* VecGenerator:: +evaluateSynthesizedAttribute(Tree* astNode, Tree* inh, + SynthesizedAttributesList& synl) +{ + assert( astNode->type>=0 && astNode->typeincreaseCounters(astNode)==true ); + assert ( vecGen_config.increaseVecCounters(astNode, tv)==true ); + + // merge vectors from its children. + for (SynthesizedAttributesList::iterator sa_itr=synl.begin(); + sa_itr!=synl.end(); ++sa_itr) + if ( *sa_itr!=NULL ) // valid TreeVector from the child + tv->operator+=(*(*sa_itr)); + + tv->node = astNode; + astNode->attributes.insert(pair(NODE_VECTOR, tv)); + + return tv; +} + +// must pass vectors from children to parents to accumulate vectors +TreeVector* VecGenerator:: +defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl) +{ + TreeVector * tv = new TreeVector(vecGen_config.parse_tree); + + // only merge vectors from its children: + int nChildren = 0; + for (SynthesizedAttributesList::iterator sa_itr=synl.begin(); + sa_itr!=synl.end(); ++sa_itr) + if ( *sa_itr!=NULL ) { + if ( nChildren==0 ) + tv->operator= (*(*sa_itr)); + else + tv->operator+=(*(*sa_itr)); + + nChildren++; + } + + if ( nChildren<=0 ) { + delete tv; + return NULL; + } else { + node->attributes.insert(pair(NODE_VECTOR, tv)); + return tv; + } +} + diff --git a/src/vgen/treeTra/node-vec-gen.h b/src/vgen/treeTra/node-vec-gen.h new file mode 100755 index 0000000..97d5362 --- /dev/null +++ b/src/vgen/treeTra/node-vec-gen.h @@ -0,0 +1,66 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _NODE_VEC_GENERATOR_H_ +#define _NODE_VEC_GENERATOR_H_ + +#include "../../include/ptree.h" +#include "tree-traversal.C" +#include "tree-vector.h" +#include "vgen-config.h" + +/* This class generates vectors for each tree node first, preparing + for vector merging later on. Thus, algorithm is more clear than + integrating merging with basic vector generation. However, this may + consume more memory than necessary. <--TODO (how to save memory? + integerate generating and merging in certain way, but + algorithmically not very clear because of the sliding windows) */ +class VecGenerator : public ParseTreeTraversal /* parent node and node vector */ +{ + protected: + TraGenConfiguration vecGen_config; + + public: + VecGenerator(TraGenConfiguration & cfg); + + virtual bool skipNode(Tree* astNode, Tree* inh); + virtual bool skipSubTree(Tree* astNode, Tree* inh); + + virtual Tree* evaluateInheritedAttribute(Tree* astNode, Tree* inh); + virtual TreeVector* evaluateSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& l); + virtual TreeVector* defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl); +}; + + +#endif /* _NODE_VEC_GENERATOR_H_ */ diff --git a/src/vgen/treeTra/sq-tree.C b/src/vgen/treeTra/sq-tree.C new file mode 100755 index 0000000..1cc6d66 --- /dev/null +++ b/src/vgen/treeTra/sq-tree.C @@ -0,0 +1,213 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include "sq-tree.h" +#include "tree-accessor.h" + +using namespace std; + +#define VGDEBUG + +/****************************************** + * Implementation for TreeSerializer: + * + *****************************************/ +TreeSerializer:: +TreeSerializer(TraGenConfiguration & cfg) + : vecGen_config(cfg), serialized_tree(NULL, NULL), + previous_node(NULL), id(0), id2node() +{ +} + +bool TreeSerializer:: +skipNode(Tree* astNode, Tree* inh) +{ + return false; +} + +bool TreeSerializer:: +skipSubTree(Tree* astNode, Tree* inh) +{ + return false; +} + +Tree* TreeSerializer:: +evaluateInheritedAttribute(Tree* astNode, Tree* inheritedValue) +{ + return astNode; +} + +pair TreeSerializer:: +evaluateSynthesizedAttribute(Tree* node, Tree* in, + SynthesizedAttributesList& synl) +{ + // establish serialized tree chains: + if ( previous_node!=NULL ) { + map::iterator attr_itr = previous_node->attributes.find(NODE_SERIALIZED_NEIGHBOR); + assert( attr_itr!=previous_node->attributes.end() ); + ((pair*)(*attr_itr).second)->second = node; + node->attributes.insert(pair*>(NODE_SERIALIZED_NEIGHBOR, new pair(previous_node, NULL))); + } else { + // its the first visited node: + serialized_tree.chain_header = node; + node->attributes.insert(pair*>(NODE_SERIALIZED_NEIGHBOR, new pair(NULL, NULL))); + } + + previous_node = node; + serialized_tree.chain_tail = node; + + // compute node id and its low_id + long low_id = id; // It's the id of the current node. + for (SynthesizedAttributesList::iterator sa_itr = synl.begin(); + sa_itr!=synl.end(); ++sa_itr) { + if ( (*sa_itr).first>=0 ) // valid id and low_id + low_id = min(low_id, (*sa_itr).second); + } + +#ifdef outputnodeids + fprintf(stdout, "Tree node type = %d(%s), #tokens = %d, id = %d, low_id = %d, value=%s\n", node->type, node->terminal_number, id, low_id, node->toTerminal()?node->toTerminal()->value->c_str():""); +#endif + node->attributes.insert(pair*>(NODE_ID, new pair(id, low_id))); + id2node.insert(pair(id, node)); + id++; + + return pair(id-1, low_id); +} + +// mush pass low_ids from children to parents to accumulate. ids do +// not need to be passed because they are an accumulative attribute. +pair TreeSerializer:: +defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl) +{ + // compute node's low_id + long low_id = -1; + int nChildren = 0; + + for (SynthesizedAttributesList::iterator sa_itr = synl.begin(); + sa_itr!=synl.end(); ++sa_itr) { + if ( (*sa_itr).second>=0 ) { // valid low_id + if ( nChildren==0 ) + low_id = (*sa_itr).second; + else + low_id = min(low_id, (*sa_itr).second); + + nChildren++; + } + } + + // These skipped nodes do not store ids or low_ids to save time and space. + if ( nChildren<=0 ) + return pair(-1, -1); // return invalid ids and invalid low_ids; + else + return pair(-1, low_id); // return invalid ids and valid low_ids accumulated from children. +} + +long TreeSerializer:: +sqtree_length() +{ +// long len = 0; +// Tree* itr = serialized_tree.chain_header; + +// while ( itr!=NULL ) { +// len++; +// itr = TreeAccessor::get_serialized_next_neighbor(itr); +// } + +// return len; + + // alternative: use id2node.size(); should be faster. + return id2node.size(); +} + + +/*********************************************************** + * Implementation for RelevantNoAtomicParent_TreeSerializer: + * + ***********************************************************/ +RelevantNoAtomicParent_TreeSerializer:: +RelevantNoAtomicParent_TreeSerializer(TraGenConfiguration & cfg) + : TreeSerializer(cfg) +{ +} + +bool RelevantNoAtomicParent_TreeSerializer:: +skipNode(Tree* astNode, Tree* inh) +{ + // a node is skipped iff it is skippable or has an atomic ancestor (not itself). + if ( vecGen_config.isSkippable(astNode)==true ) + return true; + else { + // TODO: it's possible to improve the tiem complexity from O(logn) + // to O(1) by assuming that all children of an atomic node are + // skipped (because of the following skipSubTree). +// Tree* atomic_anc = TreeAccessor::get_greatest_atomic_relevant_ancestor_in_parsetree(astNode, vecGen_config); // O(logn) +// if ( atomic_anc!=NULL && atomic_anc!=astNode ) +// return true; +// else + // SO, from now on, assume an atomic node in the Sq-Tree is the oldest atomic ancestor of its own. + return false; + } +} + +bool RelevantNoAtomicParent_TreeSerializer:: +skipSubTree(Tree* astNode, Tree* inh) +{ + // skip all children if the node is atomic: + return vecGen_config.isAtomic(astNode); +} + +// Skipped nodes are not linked into the Sq-Tree. We can use the fact to improve performance. +pair RelevantNoAtomicParent_TreeSerializer:: +defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl) +{ + // want to assign each node a unique id, even if the node is + // skippable, to make is_tree_in_subtree logically clearer. + + long low_id = id; // ids always valid for this case. + + for (SynthesizedAttributesList::iterator sa_itr = synl.begin(); + sa_itr!=synl.end(); ++sa_itr) { + low_id = min(low_id, (*sa_itr).second); + } + +#ifdef outputnodeids + fprintf(stdout, "Skipped tree node type = %d, #tokens = %d, id = %d, low_id = %d\n", node->type, node->terminal_number, id, low_id); +#endif + node->attributes.insert(pair*>(NODE_ID, new pair(id, low_id))); + id++; + + return pair(id-1, low_id); +} + diff --git a/src/vgen/treeTra/sq-tree.h b/src/vgen/treeTra/sq-tree.h new file mode 100755 index 0000000..bbdfb75 --- /dev/null +++ b/src/vgen/treeTra/sq-tree.h @@ -0,0 +1,93 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _SERIALIZED_TREE_H_ +#define _SERIALIZED_TREE_H_ + +#include +#include +#include "../../include/ptree.h" +#include "tree-traversal.C" +#include "vgen-config.h" + +struct SerializedTree { + Tree *chain_header; + Tree *chain_tail; + SerializedTree(Tree* h, Tree* t) { + chain_header = h; + chain_tail = t; + } +}; + +/* serialize the tree in post order. */ +class TreeSerializer : public ParseTreeTraversal > + /* > */ +{ + /* all nodes are serialized in their post order. */ + protected: + TraGenConfiguration vecGen_config; + + public: + SerializedTree serialized_tree; /* head/tail of the tree chain */ + Tree *previous_node; + long id; /* The number of (valid, depending on implementation) nodes; initialized to 0. NOTE: it is better to put "id" here than in class TokenCounter: it's easier to make sure that nodes with valid ids are those nodes in the serialized tree (but code is maybe more interweaved). */ + std::map id2node; /* keep a map between ids and nodes to make traversal over the sq-tree faster. TODO. */ + + public: + TreeSerializer(TraGenConfiguration & cfg); + + virtual bool skipNode(Tree* astNode, Tree* inh); + virtual bool skipSubTree(Tree* astNode, Tree* inh); + + virtual Tree* evaluateInheritedAttribute(Tree* astNode, Tree* inheritedValue); + virtual std::pair evaluateSynthesizedAttribute(Tree* node, Tree* in, + SynthesizedAttributesList& synl); + virtual std::pair defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl); + virtual long sqtree_length(); +}; + +class RelevantNoAtomicParent_TreeSerializer : public TreeSerializer +{ + /* relevant nodes without atomic ancestors in the parse tree are serialized in their post order. */ + public: + RelevantNoAtomicParent_TreeSerializer(TraGenConfiguration & cfg); + + virtual bool skipNode(Tree* astNode, Tree* inh); + virtual bool skipSubTree(Tree* astNode, Tree* inh); + /* use unique id for different nodes, even if they are skippable. */ + virtual std::pair defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl); +}; + +#endif /* _SERIALIZED_TREE_H_ */ + diff --git a/src/vgen/treeTra/token-counter.C b/src/vgen/treeTra/token-counter.C new file mode 100755 index 0000000..1eb6814 --- /dev/null +++ b/src/vgen/treeTra/token-counter.C @@ -0,0 +1,205 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "token-counter.h" + +using namespace std; + +#define VGDEBUG +extern ParseTree* global_tree_for_debugging; + +/****************************************** + * Implementation for TokenCounter + * + *****************************************/ +TokenCounter:: +TokenCounter(TraGenConfiguration & cfg) + : vecGen_config(cfg) +{ +} + +bool TokenCounter:: +skipNode(Tree* astNode, Tree* inh) +{ + return false; +} + +bool TokenCounter:: +skipSubTree(Tree* astNode, Tree* inh) +{ + return false; +} + +Tree* TokenCounter:: +evaluateInheritedAttribute(Tree* astNode, Tree* inheritedValue) +{ + // done in Ghassan's code: +// astNode->parent = inheritedValue; + + return astNode; +} + +long TokenCounter:: +evaluateSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl) +{ + if ( node->isTerminal()==true ) { + node->terminal_number = 1; +#ifdef outputtokens + fprintf(stdout, "token = %s\n", node->toTerminal()->value->c_str()); +#endif + } else { + node->terminal_number = 0; + for (SynthesizedAttributesList::iterator sa_itr=synl.begin(); + sa_itr!=synl.end(); ++sa_itr) { + node->terminal_number += (*sa_itr); + } + } + +#ifdef outputallnodes + fprintf(stdout, "Tree node type = %d (%s), #tokens = %d, value=`%s'\n", node->type, global_tree_for_debugging ? global_tree_for_debugging->getTypeName(node->type).c_str() : "", node->terminal_number, node->isTerminal() ? node->toTerminal()->value->c_str():""); +#endif + + return node->terminal_number; +} + +// must pass token numbers from children to parents to accumulate. +long TokenCounter:: +defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl) +{ + // do not increase terminal numbers. + if ( node==NULL ) + return 0; + + node->terminal_number = 0; + if ( node->isNonTerminal()==true ) { // the condition must be redundent. + for (SynthesizedAttributesList::iterator sa_itr=synl.begin(); + sa_itr!=synl.end(); ++sa_itr) { + node->terminal_number += (*sa_itr); + } + } + +#ifdef outputallnodes + fprintf(stdout, "Skipped tree node type = %d (%s), #tokens = %d, value=`%s'\n", node->type, global_tree_for_debugging ? global_tree_for_debugging->getTypeName(node->type).c_str() : "", node->terminal_number, node->isTerminal() ? node->toTerminal()->value->c_str():""); +#endif + + return node->terminal_number; +} + + +/****************************************** + * Implementation for RelevantTokenCounter + * + *****************************************/ +RelevantTokenCounter:: +RelevantTokenCounter(TraGenConfiguration & cfg) + : TokenCounter(cfg) +{ +} + +bool RelevantTokenCounter:: +skipNode(Tree* astNode, Tree* inh) +{ + return vecGen_config.isSkippable(astNode); +} + +/****************************************** + * Implementation for TokenRangeCounter + * + *****************************************/ +TokenRangeCounter:: +TokenRangeCounter(TraGenConfiguration & cfg) + : TokenCounter(cfg), tokennumber(0) +{ +} + +void TokenRangeCounter:: +reinit() +{ + tokennumber = 0; +} + +long TokenRangeCounter:: +evaluateSynthesizedAttribute(Tree* node, Tree* in, + SynthesizedAttributesList& synl) +{ + long low_token_id = -1; + if ( node->isTerminal()==true ) { + low_token_id = tokennumber++; + } else { + for (SynthesizedAttributesList::iterator sa_itr=synl.begin(); + sa_itr!=synl.end(); ++sa_itr) { + if ( low_token_id<0 ) + low_token_id = (*sa_itr); + else if ( (*sa_itr)<0 ) + ; + else + low_token_id = min(low_token_id, (*sa_itr)); + } + } + + if ( low_token_id<0 ) + node->attributes.insert(pair*>(NODE_TOKEN_ID, new pair(low_token_id, -1))); + else + node->attributes.insert(pair*>(NODE_TOKEN_ID, new pair(low_token_id, tokennumber-1))); + + return low_token_id; +} + +long TokenRangeCounter:: +defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl) +{ + if ( node==NULL ) + return -1; + + long low_token_id = -1; + if ( node->isNonTerminal()==true ) { // the condition must be redundent. + for (SynthesizedAttributesList::iterator sa_itr=synl.begin(); + sa_itr!=synl.end(); ++sa_itr) { + if ( low_token_id<0 ) + low_token_id = (*sa_itr); + else if ( (*sa_itr)<0 ) + ; + else + low_token_id = min(low_token_id, (*sa_itr)); + } + } + + if ( low_token_id<0 ) + node->attributes.insert(pair*>(NODE_TOKEN_ID, new pair(low_token_id, -1))); + else + node->attributes.insert(pair*>(NODE_TOKEN_ID, new pair(low_token_id, tokennumber-1))); + + return low_token_id; +} diff --git a/src/vgen/treeTra/token-counter.h b/src/vgen/treeTra/token-counter.h new file mode 100755 index 0000000..bac1a83 --- /dev/null +++ b/src/vgen/treeTra/token-counter.h @@ -0,0 +1,90 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _TOKEN_COUNTER_H_ +#define _TOKEN_COUNTER_H_ + +#include "../../include/ptree.h" +#include "tree-traversal.C" +#include "vgen-config.h" + +/* Count the numbers of tokens and fill in parent pointers for every + tree node if not filled in yet. */ +class TokenCounter : public ParseTreeTraversal /* parent pointer, token number */ +{ + /* count all tokens (terminal nodes). */ + protected: + TraGenConfiguration vecGen_config; + + public: + TokenCounter(TraGenConfiguration & cfg); + + virtual bool skipNode(Tree* astNode, Tree* inh); + virtual bool skipSubTree(Tree* astNode, Tree* inh); + + virtual Tree* evaluateInheritedAttribute(Tree* astNode, Tree* inheritedValue); + virtual long evaluateSynthesizedAttribute(Tree* node, Tree* in, + SynthesizedAttributesList& synl); + virtual long defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl); +}; + +class RelevantTokenCounter : public TokenCounter +{ + /* count relevant tokens only. */ + public: + RelevantTokenCounter(TraGenConfiguration & cfg); + + virtual bool skipNode(Tree* astNode, Tree* inh); +}; + +class TokenRangeCounter : public TokenCounter /* : */ +{ + /* set the token range for each node */ + private: + long tokennumber; /* number of tokens; initialized to 0. Valid: [0, + tokennumber-1]. If "traverse" is called again + and again on an object of this class, it's up + to callers to decide whether to reset + "tokennumber" to 0 or not */ + + public: + TokenRangeCounter(TraGenConfiguration & cfg); + void reinit(); + virtual long evaluateSynthesizedAttribute(Tree* node, Tree* in, + SynthesizedAttributesList& synl); + virtual long defaultSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& synl); +}; + +#endif /* _TOKEN_COUNTER_H_ */ + diff --git a/src/vgen/treeTra/token-tree-map.C b/src/vgen/treeTra/token-tree-map.C new file mode 100755 index 0000000..d0af605 --- /dev/null +++ b/src/vgen/treeTra/token-tree-map.C @@ -0,0 +1,1414 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "token-tree-map.h" + +/************************************************** + * Implementation of ClonePointT + * + *************************************************/ +bool ClonePointT:: +parse(char * line, regex_t patterns[], int dim) +{ + // up to me to make sure patterns is of length dim. + int a, b; + regmatch_t pmatch[2]; + bool rslflag = true; + + if ( line==NULL || strcmp(line, "")==0 || strncmp(line, "\n", 1)==0 ) + return false; + + linebuf = line; + + if ( regexec(&(patterns[ENUM_CLONE_FILE]), line, 2, pmatch, 0)==0 && + (a = pmatch[1].rm_so) != -1 ) { + b = pmatch[1].rm_eo; + char t = line[b]; + line[b] = '\0'; + filename = line+a; + line[b] = t; + } else + rslflag = false; + +#define parseAux(buf, pat, fld, func) { \ + if ( regexec(&(pat), buf, 2, pmatch, 0)==0 && \ + (a = pmatch[1].rm_so) != -1 ) { \ + b = pmatch[1].rm_eo; \ + char t = buf[b]; \ + buf[b] = '\0'; \ + fld = func(buf+a); \ + buf[b] = t; \ + } else { \ + fld = func("0"); \ + rslflag = false; \ + } \ + } + + // relax return flag: as long as we have "FILE", then return true: + bool fileflag = rslflag; + parseAux(line, patterns[ENUM_CLONE_LINE], begin_line_id, atol); + parseAux(line, patterns[ENUM_CLONE_INDEX], index, atol); + parseAux(line, patterns[ENUM_CLONE_DIST], dist, atof); + parseAux(line, patterns[ENUM_CLONE_OFFSET], line_offset, atol); + parseAux(line, patterns[ENUM_CLONE_TBID], tbid, atol); + parseAux(line, patterns[ENUM_CLONE_TEID], teid, atol); + parseAux(line, patterns[ENUM_CLONE_NODE_KIND], node_kind, atoi); + parseAux(line, patterns[ENUM_CLONE_NODE_NUM], node_number, atoi); + parseAux(line, patterns[ENUM_CLONE_nVARs], uni_var_number, atoi); + return fileflag; +} + +ostream & ClonePointT:: +out2html(ostream & os) +{ + // os.width(9); os.fill('0'); + string showfilename = filename; +#define MAXLENFORSHOW 56 + if ( showfilename.length() > MAXLENFORSHOW ) + showfilename = "..." + showfilename.substr(showfilename.length() - MAXLENFORSHOW); + os << index << "\t\tdist:" << /*setprecision(2) <<*/ dist << "\t\t" + << "FILE " << showfilename << " " + << "LINE:" << begin_line_id << ":" << line_offset << " " + << "NODE_KIND:" << node_kind << " nVARs:" << uni_var_number << " NUM_NODE:" << node_number << " TBID:" << tbid << " TEID:" << teid << "
" << endl; + return os; +} + +ostream & ClonePointT:: +out2html0(ostream & os) +{ + string showfilename = filename; +#define MAXLENFORSHOW 56 + if ( showfilename.length() > MAXLENFORSHOW ) + showfilename = "..." + showfilename.substr(showfilename.length() - MAXLENFORSHOW); + string::size_type loc = linebuf.find("FILE "); + if ( loc!=string::npos ) + os << linebuf.substr(0, loc); + os << "FILE " << showfilename << " " + << "LINE:" + << begin_line_id << ":" << line_offset << " "; + loc = linebuf.find("NODE_KIND:"); + if ( loc!=string::npos ) + os << linebuf.substr(loc); + os << "
" << endl; + return os; +} + +ostream & ClonePointT:: +out2xml(ostream & os) +{ + os << "index=\"" << index << "\" "; + os << "dist=\"" << /*setprecision(2) << */ dist << "\" "; + os << "file=\"" << filename << "\" "; + os << "lineno=\"" << begin_line_id << "\" "; + os << "lineoffset=\"" << line_offset << "\" "; + os << "nodekind=\"" << node_kind << "\" "; + os << "nvars=\"" << uni_var_number << "\" "; + os << "tbid=\"" << tbid << "\" "; + os << "teid=\"" << teid << "\" "; + + return os; +} + +ostream & operator<< (ostream& os, const struct _ClonePointT & cp) +{ + os << cp.linebuf; + return os; +} + +/****************************************** + * Implementation of TokenTreeMap + * + *****************************************/ +regex_t TokenTreeMap::clone_patterns[ENUM_CLONE_THE_END]; + +bool TokenTreeMap:: +init_shared_data() +{ + int errcode; char errmsg[100]; + if ( 0 != (errcode = regcomp(&clone_patterns[ENUM_CLONE_INDEX], "^([0-9]+)", REG_EXTENDED | REG_NEWLINE)) ) { + regerror(errcode, &clone_patterns[ENUM_CLONE_INDEX], errmsg, 100); + cerr << "error code: " << errcode << endl << errmsg << endl; + } + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_DIST], "dist:([0-9\\.]+)", REG_EXTENDED) ); + //NOTE: filenames can contain spaces... + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_FILE], "FILE (.*) LINE", REG_EXTENDED) ); + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_LINE], "LINE:([0-9]+):", REG_EXTENDED) ); + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_OFFSET], "LINE:[0-9]+:([0-9]+)", REG_EXTENDED) ); + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_TBID], "TBID:([-]?[0-9]+)", REG_EXTENDED) ); + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_TEID], "TEID:([-]?[0-9]+)", REG_EXTENDED) ); + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_NODE_KIND], "NODE_KIND:([0-9]+)", REG_EXTENDED) ); + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_NODE_NUM], "NUM_NODE:([0-9]+)", REG_EXTENDED) ); + assert ( 0 == regcomp(&clone_patterns[ENUM_CLONE_nVARs], "nVARs:([0-9]+)", REG_EXTENDED) ); + return true; +} + +TokenTreeMap:: +TokenTreeMap() : fn2tree(), clusterbuffer(), + contexualNodes(id2name.size(), false) +{ + vecGen_config = new TraGenConfiguration((const char*)NULL); // no actual use; just a dummy + token_range_counter = new TokenRangeCounter(*vecGen_config); +} + +TokenTreeMap:: +~TokenTreeMap() +{ + if ( token_range_counter!=NULL ) { + delete token_range_counter; + token_range_counter = NULL; + } + if ( vecGen_config!=NULL ) { + delete vecGen_config; + vecGen_config = NULL; + } +} + +ostream & TokenTreeMap:: +outputCluster(ostream & out) +{ + out << "Rank score: " << rank << " * " << clusterbuffer.size() << " =" << rank*clusterbuffer.size() + << " buggy score: "; + for (int i=0; i 0 ); + contexualNodes = vector(id2name.size(), false); + for (const char **s= nodeconfig; *s != NULL; s++) { + map::iterator i= name2id.find(*s); + if (i == name2id.end()) { + cerr << "unknown node type: " << *s << endl; + errflag = true; + continue; + } + contexualNodes[i->second] = true; + } + return errflag; +} + +ParseTree* TokenTreeMap:: +parseFile(const char * fn) +{ + // NOTE: yyparse is NOT re-entrant, which caused weird bugs when I called yyparse for more than one files! + yyin = fopen(fn, "r"); + if (!yyin) { + cerr << "Can't open file: " << fn << endl; + } + yyrestart(yyin); // This is maybe unnecessary because FLEX's manual + // says this is equivalent to changing yyin directly + // when EOF is met. But it doesn't reset start conditions; this is + // why we need to use "yywrap" instead of exporting some internal + // values in FLEX. + yyparse(); + fclose(yyin); + yyin = NULL; + if (!root) { + cerr << "failed to parse file: " << fn << endl; + return NULL; + } else { + Tree* initial_inh = NULL; + root->lineRange(); + ParseTree* pt = new ParseTree(root, id2name.size(), &id2name, &name2id); + token_range_counter->reinit(); // this is necessary to correctly match terminal IDs. + token_range_counter->traverse(pt->getRoot(), initial_inh); + return pt; + } +} + +#define parseClonePointAux(line, pcp, reg, field, func) { \ + if ( regexec(&(reg), line, 2, pmatch, 0)==0 && \ + (a = pmatch[1].rm_so) != -1 ) { \ + b = pmatch[1].rm_eo; \ + char t = line[b]; \ + line[b] = '\0'; \ + pcp->field = func(line+a); \ + line[b] = t; \ + } else \ + rslflag = false; \ + } // no use any more, coz I re-organized the code. + +bool TokenTreeMap:: +parseClonePoint(char * line, PClonePointT pcp) +{ + return pcp->parse(line, clone_patterns); +} + +bool TokenTreeMap:: +createFN2Tree() // read lines from stdin +{ + string line; + int linecount = 0; + + clearBuffer(); + + while ( !cin.eof() ) { + getline(cin, line); + ClonePointT temp; + linecount++; + char * charline = new char[line.length()+1]; + strcpy(charline, line.c_str()); + if ( parseClonePoint(charline, &temp) ) { + clusterbuffer.push_back(temp); + } else + cerr << "Warning: error line " << linecount << ": " << line << endl; + delete charline; + } + // set ranks: + rank = 0; + for ( int i=0; itype >= 0 && node->type < id2name.size() ); + if ( contexualNodes[node->type] == true ) + return true; + else + return false; +} + +void TokenTreeMap:: +clearBuffer() +{ + clusterbuffer.clear(); +} + +Tree* TokenTreeMap:: +tokenRange2TreeAux1(pair tokenrange, Tree* node) +// This function can't find the smallest subtree containing the tokenrange because of certain inconsistent logical error. +{ + Tree* rsl = NULL; + map::iterator attr_itr = node->attributes.find(NODE_TOKEN_ID); + assert( attr_itr!=node->attributes.end() ); + const pair* tokens = (pair*)(*attr_itr).second; + // assert ( tokens->first <= tokens->second ); + // assert ( tokenrange.first <= tokenrange.second ); + if ( tokens->first < tokenrange.first ) // logical bug here. "<=" instead of "<" + if ( tokens->second < tokenrange.first ) + return NULL; + else if ( tokens->second <= tokenrange.second ) // logical bug here. "<" instead of "<=" + return node; + else { // recursion: + int num_nullchild = 0; + for (int i = 0; i < node->children.size(); i++) { + Tree* rslchild = tokenRange2TreeAux1(tokenrange, node->children[i]); + if ( rslchild!=NULL ) { + num_nullchild++; + if ( num_nullchild>1 ) + return node; + else + rsl = rslchild; + } + } + return rsl; + } + else if ( tokens->first <= tokenrange.second ) // logical bug here. TODO: may cut this branch... + return node; + else + return NULL; +} + +Tree* TokenTreeMap:: +tokenRange2Tree1(std::pair tokenrange, ParseTree* pt) +{ + if ( pt==NULL ) + return NULL; + + Tree * root = pt->getRoot(); + if ( root == NULL ) + return NULL; + else + return tokenRange2TreeAux1(tokenrange, root); // shouldn't return NULL here. +} + +Tree* TokenTreeMap:: +tokenRange2Tree2(std::pair tokenrange, ParseTree* pt) +{ + if ( pt==NULL ) + return NULL; + + Tree * root = pt->getRoot(); + if ( root == NULL ) + return NULL; + + list* path1 = root2Token(tokenrange.first, pt); + list* path2 = root2Token(tokenrange.second, pt); + Tree * rsl = NULL; + + if ( path1==NULL || path2==NULL ) + rsl = root; + else { // compare the two paths + list::iterator itr1 = path1->begin(); + list::iterator itr2 = path2->begin(); + for (; itr1 != path1->end() && itr2 != path2->end(); ++itr1, ++itr2) { + if ( (*itr1) != (*itr2) ) // find the smallest common ancestor + break; + rsl = (*itr1); + } + // print the paths for debugging: +#define print_path_list(path_name) \ + for (list::iterator path_itr = (path_name)->begin(); \ + path_itr != (path_name)->end(); ++path_itr) \ + cerr << "NODE: %X" << (*path_itr) << " type:" << getTypeName(id2name, (*path_itr)->type) << endl; + /* print_path_list(path1); + cerr << endl; + print_path_list(path2); */ + } + if ( rsl == NULL ) + rsl = root; + + if ( path1!=NULL ) + delete path1; + if ( path2!=NULL ) + delete path2; + + return rsl; // shouldn't return NULL here. +} + +bool TokenTreeMap:: +root2TokenAux(long tid, Tree* node, list& path) +{ + if ( node == NULL ) + return false; + + map::iterator attr_itr = node->attributes.find(NODE_TOKEN_ID); + assert( attr_itr!=node->attributes.end() ); + const pair* tokens = (pair*)(*attr_itr).second; + if ( tid >= tokens->first && tid <= tokens->second ) { + // TODO: try to make it tail recursive: + + // add the current node to the path: + path.push_back(node); + // select a child with the correct range: + for (int i=0; i < node->children.size(); i++) { + if ( root2TokenAux(tid, node->children[i], path) ) + return true; // within range. + } // the path must exist and be unique (it's true for well-formed + // ASTs), otherwise the code may return a wrong list. + return false; // out of range. should NOT be here for well-formed ASTs. + } else + return false; // out of range. +} + +list* TokenTreeMap:: +root2Token(long tid, ParseTree* pt) +{ + list* path = new list(); + root2TokenAux(tid, pt->getRoot(), *path); + if ( path->empty() ) { + delete path; + path = NULL; + } + return path; +} + +CloneContextT TokenTreeMap:: +getContext1(pair tokenrange, string & fn) +{ + CloneContextT rsl; + ParseTree* pt = NULL; + map::iterator fn_itr = fn2tree.find(fn); + if ( fn_itr == fn2tree.end() ) { + pt = parseFile(fn.c_str()); + if ( pt!=NULL ) { + fn2tree.insert(pair(fn, pt)); + } + } else + pt = (*fn_itr).second; + + rsl.context_node_begin = rsl.context_node_end = NULL; + rsl.context_node_begin = tokenRange2Tree2(tokenrange, pt); + if ( rsl.context_node_begin!=NULL ) + rsl.context_node_end = rsl.context_node_begin->parent; + else + rsl.context_node_begin = pt ? pt->getRoot() : NULL; + + // assert ( rsl.context_node_begin!=NULL ); + return rsl; +} + +CloneContextT TokenTreeMap:: +getContext2(pair tokenrange, string & fn) +{ + CloneContextT rsl; + ParseTree* pt = NULL; + map::iterator fn_itr = fn2tree.find(fn); + if ( fn_itr == fn2tree.end() ) { + pt = parseFile(fn.c_str()); + if ( pt!=NULL ) { + fn2tree.insert(pair(fn, pt)); + } + } else + pt = (*fn_itr).second; + + rsl.context_node_begin = rsl.context_node_end = NULL; + rsl.context_node_begin = tokenRange2Tree2(tokenrange, pt); + // cerr << "getContext2 type: " << (rsl.context_node_begin ? getTypeName(id2name, rsl.context_node_begin->type) : "-1") << endl; + if ( rsl.context_node_begin!=NULL ) { + // if tokenrange is within (actual subrange) context_node_begin, then start from context_node_begin; + // otherwise, start searching from the parent of context_node_begin; + map::iterator attr_itr = rsl.context_node_begin->attributes.find(NODE_TOKEN_ID); + assert ( attr_itr!=rsl.context_node_begin->attributes.end() ); + pair* startrange = (pair*)(*attr_itr).second; + Tree* startnode = rsl.context_node_begin; + if ( tokenrange.first <= startrange->first && tokenrange.second >= startrange->second ) { + startnode = rsl.context_node_begin->parent; + } + while ( startnode!=NULL ) { + if ( isContextual(startnode) ) { // this condition is language-dependant + rsl.context_node_begin = startnode; + break; + } else + startnode = startnode->parent; + } + if ( startnode==NULL ) // at least, rsl.context_node_begin should be the "root": + rsl.context_node_begin = pt->getRoot(); + } else + rsl.context_node_begin = pt ? pt->getRoot() : NULL; + + // assert ( rsl.context_node_begin!=NULL && rsl.context_node_end==NULL ); + return rsl; +} + +Tree* TokenTreeMap:: +getContextNode(Tree* node) +{ + if ( node == NULL ) + return NULL; + Tree * rsl = node; + if ( isContextual(rsl) && + ( rsl->type == getTypeID(name2id, "simple_if") || + rsl->type == getTypeID(name2id, "do_stmt_start") || + rsl->type == getTypeID(name2id, "WHILE") || + rsl->type == getTypeID(name2id, "FOR") || + rsl->type == getTypeID(name2id, "SWITCH") ) ) + rsl = rsl->parent; + return rsl; // assert rsl!=NULL +} + +Tree* TokenTreeMap:: +getContextParent(Tree* node) +{ + if ( node == NULL ) + return NULL; + Tree * rsl = getContextNode(node); + rsl = rsl->parent; + while ( rsl!=NULL ) { + if ( isContextual(rsl) ) { // this condition is language-dependant + break; + } else + rsl = rsl->parent; + } + return rsl; +} + +bool TokenTreeMap:: +compareContext1(CloneContextT& context1, CloneContextT& context2) +{ + // pair-wise comparison of node kinds: better to use a loop instead (TODO) + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else if ( context1.context_node_end==NULL ) + if ( context2.context_node_end!=NULL ) + return false; + else + return true; + else if ( context2.context_node_end==NULL ) + return false; + else if ( context1.context_node_end->type != context2.context_node_end->type ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else if ( context1.context_node_begin->type != context2.context_node_begin->type ) + return false; + else if ( context1.context_node_end==NULL ) + if ( context2.context_node_end!=NULL ) + return false; + else + return true; + else if ( context2.context_node_end==NULL ) + return false; + else if ( context1.context_node_end->type != context2.context_node_end->type ) + return false; + else + return true; +} + +bool TokenTreeMap:: +compareCContext2(CloneContextT& context1, CloneContextT& context2) +{ + // only compare the context_node_begin; + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else { // actual comparison here: + Tree* forcompare1 = context1.context_node_begin; + Tree* forcompare2 = context2.context_node_begin; + // because of non-clear structure in C's grammar file, we need the "if" hack: + if ( forcompare1->type == getTypeID(name2id, "select_or_iter_stmt") ) + forcompare1 = forcompare1->children[0]; + if ( forcompare2->type == getTypeID(name2id, "select_or_iter_stmt") ) + forcompare2 = forcompare2->children[0]; + assert ( forcompare1!=NULL && forcompare2!=NULL ); + // may be better to let "for"=="while", "if"=="switch" + if ( forcompare1->type == getTypeID(name2id, "simple_if") || + forcompare1->type == getTypeID(name2id, "SWITCH") ) + if ( forcompare2->type == getTypeID(name2id, "simple_if") || + forcompare2->type == getTypeID(name2id, "SWITCH") ) + return true; + else + return false; + else if ( forcompare1->type == getTypeID(name2id, "WHILE") || + forcompare1->type == getTypeID(name2id, "FOR") ) + if ( forcompare2->type == getTypeID(name2id, "WHILE") || + forcompare2->type == getTypeID(name2id, "FOR") ) + return true; + else + return false; + else if ( forcompare1->type == forcompare2->type ) + return true; + else + return false; + } +} + +Tree* TokenTreeMap:: +get_condition_within(Tree* node) +{ + Tree* rsl = NULL; + if ( node==NULL ) + return rsl; + // the following relies on C's grammar file and ccontextualNodes.h: + if ( node->type == getTypeID(name2id, "select_or_iter_stmt") ) + if ( node->children[0]->type == getTypeID(name2id, "simple_if") ) /* simple_if : if_prefix : IF '(' expr */ + rsl = node->children[0]->children[0]->children[2]; + else if ( node->children[0]->type == getTypeID(name2id, "WHILE") ) /* WHILE '(' expr */ + rsl = node->children[2]; + else if ( node->children[0]->type == getTypeID(name2id, "do_stmt_start") ) + if ( node->children.size() >= 3 ) + rsl = node->children[2]; + else /* parsing error */ + rsl = NULL; + else if ( node->children[0]->type == getTypeID(name2id, "FOR") ) + if ( node->children[3]->children.size() <= 0 ) + rsl = NULL; + else + rsl = node->children[3]->children[0]; + else if ( node->children[0]->type == getTypeID(name2id, "SWITCH") ) + rsl = node->children[2]; + else /* shouldn't happen for the C grammar */ + rsl = NULL; + else if ( node->type == getTypeID(name2id, "simple_if") ) + rsl = node->children[0]->children[2]; + else if ( node->type == getTypeID(name2id, "do_stmt_start") ) + if ( node->parent->children.size() >= 3 ) + rsl = node->parent->children[2]; + else /* parsing error */ + rsl = NULL; + else if ( node->type == getTypeID(name2id, "WHILE") ) + rsl = node->parent->children[2]; + else if ( node->type == getTypeID(name2id, "FOR") ) + if ( node->parent->children[3]->children.size() <= 0 ) + rsl = NULL; + else + rsl = node->parent->children[3]->children[0]; + else if ( node->type == getTypeID(name2id, "SWITCH") ) + rsl = node->parent->children[2]; + else + rsl = NULL; + + return rsl; +} + +bool TokenTreeMap:: +compareCConditions(CloneContextT& context1, CloneContextT& context2) +{ + // only compare the context_node_begin; + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else { // actual condition comparison here: + Tree* forcompare1 = context1.context_node_begin; + Tree* forcompare2 = context2.context_node_begin; + + forcompare1 = get_condition_within(forcompare1); + forcompare2 = get_condition_within(forcompare2); + return compareTree(forcompare1, forcompare2); // exact match is not good, e.g., an extra pair of () may change the result. + } +} + +Tree* TokenTreeMap:: +get_conditional_operator(Tree* node) +{ + // look for the main operator (for C only) in an expr: + Tree * rsl = NULL; + if ( node==NULL ) + return rsl; + if ( node->type == getTypeID(name2id, "expr") ) + return get_conditional_operator(node->children[0]); + else if ( node->type == getTypeID(name2id, "nonnull_exprlist") ) + if ( node->children.size()==3 ) // nonnull_exprlist ',' expr_no_commas + return node->children[1]; // ',' + else + return get_conditional_operator(node->children[0]); // expr_no_commas + else if ( node->type == getTypeID(name2id, "expr_no_commas") ) + if ( node->children.size()==1 ) // cast_expr + return get_conditional_operator(node->children[0]); + else + return node->children[1]; + else if ( node->type == getTypeID(name2id, "cast_expr") ) + if ( node->children.size()==1 ) // unary_expr + return get_conditional_operator(node->children[0]); + else + return get_conditional_operator(node->children[3]); + else if ( node->type == getTypeID(name2id, "unary_expr") ) + if ( node->children.size()==1 ) // primary + return get_conditional_operator(node->children[0]); + else if ( node->children[0]->type == getTypeID(name2id, "unop") ) + return node->children[0]->children[0]; + else if ( node->children[0]->type == getTypeID(name2id, "extension") ) + return get_conditional_operator(node->children[1]); // cast_expr + else + return node->children[0]; + else if ( node->type == getTypeID(name2id, "primary") ) + if ( node->children.size()>1 ) + if ( node->children[1]->type == getTypeID(name2id, "expr") ) + return get_conditional_operator(node->children[1]); + else + return node; // return the primary itself + else + return node; // return the primary itself + else + return node; // return other exprs themselves + return rsl; +} + +bool TokenTreeMap:: +isMainOperator(Tree * op1) +{ + if ( op1==NULL ) + return false; + /* the definition is not so good ... + else if ( op1->type == getTypeID(name2id, "!") || + op1->type == getTypeID(name2id, "~") || + op1->type == getTypeID(name2id, "-") || + op1->type == getTypeID(name2id, "&") || + op1->type == getTypeID(name2id, "PLUSPLUS") || + op1->type == getTypeID(name2id, "MINUSMINUS") || + op1->type == getTypeID(name2id, "+") || + op1->type == getTypeID(name2id, "*") || + op1->type == getTypeID(name2id, "/") || + op1->type == getTypeID(name2id, "%") || + op1->type == getTypeID(name2id, "LSHIFT") || + op1->type == getTypeID(name2id, "RSHIFT") || + op1->type == getTypeID(name2id, "ARITHCOMPARE") || + op1->type == getTypeID(name2id, "EQCOMPARE") || + op1->type == getTypeID(name2id, "|") || + op1->type == getTypeID(name2id, "^") || + op1->type == getTypeID(name2id, "ANDAND") || + op1->type == getTypeID(name2id, "OROR") || + op1->type == getTypeID(name2id, "?") || + op1->type == getTypeID(name2id, "=") || + op1->type == getTypeID(name2id, "ASSIGN") ) + return true; + */ + else + return false; +} + +bool TokenTreeMap:: +compareConditionalOperators(CloneContextT& context1, CloneContextT& context2) +{ + // only compare the context_node_begin; + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else { + Tree* forcompare1 = context1.context_node_begin; + Tree* forcompare2 = context2.context_node_begin; + + forcompare1 = get_condition_within(forcompare1); + forcompare2 = get_condition_within(forcompare2); + Tree* op1 = get_conditional_operator(forcompare1); + Tree* op2 = get_conditional_operator(forcompare2); + /* + cerr << "cond1: " << forcompare1->type << "=" << ( forcompare1==NULL ? "NULL" : getTypeName(id2name, forcompare1->type) ) << " " + << "cond2: " << forcompare2->type << "=" << ( forcompare2==NULL ? "NULL" : getTypeName(id2name, forcompare2->type) ) << endl + << "op1: " << op1->type << "=" << ( op1==NULL ? "NULL" : getTypeName(id2name, op1->type) ) << " " + << "op2: " << op2->type << "=" << ( op2==NULL ? "NULL" : getTypeName(id2name, op2->type) ) << endl; + */ + if ( op1==NULL && op2==NULL ) + return true; + else if ( op1==NULL || op2==NULL ) + return false; + else if ( op1->type != op2->type ) + return false; + else if ( isMainOperator(op1) ) /* language dependent...its definition is not so good */ + return compareTree(forcompare1, forcompare2); + else + return true; + } +} + +bool TokenTreeMap:: +isAnyFiltered() +{ + bool rslflag = false; + if ( clusterbuffer.size() < 1 ) { + cerr << "No clone cluster is in the buffer..." << endl; + rslflag = true; + } else { + rslflag = filter1(); + if ( rslflag == false ) + rslflag = filter2(); + } + + return rslflag; +} + +bool TokenTreeMap:: +isAllFiltered() +{ + bool rslflag = false; + if ( clusterbuffer.size() < 1 ) { + cerr << "No clone cluster is in the buffer..." << endl; + rslflag = true; + } else { + rslflag = filter1(); + if ( rslflag == true ) + rslflag = filter2(); + } + + return rslflag; +} + +bool TokenTreeMap:: +filter1() +{ + // group the clone points; if #groups<=1, filter it; + vector groups; + GetContextFuncT getContext = &TokenTreeMap::getContext2; + CompareContextFuncT compareContext = &TokenTreeMap::compareCContext2; + // when calling a member function through a + // pointer-to-member-function, we can't use (*ptrMemFun) directly + // because the address the pointer points to may only be an offset + // in an ACTUAL object. So, we must use (obj.(*ptrMemFun)) instead. + // Better to use "functionoid". + + for (int i = 0; i < clusterbuffer.size(); i++) { + int j = 0; + int tmps = groups.size(); + for (; j < tmps; j++) { + CloneContextT context1 = ((*this).*getContext)(pair(groups[j][0]->tbid, groups[j][0]->teid), groups[j][0]->filename); + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + if ( (this->*compareContext)(context1, context2) ) { + groups[j].push_back(&clusterbuffer[i]); + break; + } + } + if ( j >= tmps ) { + groups.push_back(ClonePointGroupT()); + groups.back().push_back(&clusterbuffer[i]); + } + } + + if ( groups.size() <= 1 ) + return true; // ENUM_RANK_NOTHING + else + return false; // ENUM_RANK_CXT_NODE +} + +bool TokenTreeMap:: +filter2() +{ + // group the clone points based on contexts AND conditions; + // if #groups<=1 AND #nVARs are different, filter it; + vector groups; + GetContextFuncT getContext = &TokenTreeMap::getContext2; + CompareContextFuncT compareCondition = &TokenTreeMap::compareConditionalOperators; + + for (int i = 0; i < clusterbuffer.size(); i++) { + int j = 0; + int tmps = groups.size(); + for (; j < tmps; j++) { + CloneContextT context1 = ((*this).*getContext)(pair(groups[j][0]->tbid, groups[j][0]->teid), groups[j][0]->filename); + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + if ( (this->*compareCondition)(context1, context2) ) { // compare contextual conditions. + groups[j].push_back(&clusterbuffer[i]); + break; + } + } + if ( j >= tmps ) { + groups.push_back(ClonePointGroupT()); + groups.back().push_back(&clusterbuffer[i]); + } + } + + if ( groups.size() <= 1 ) { + return true; // ENUM_RANK_NOTHING + } else + return false; // ENUM_RANK_CXT_COND +} + +bool TokenTreeMap:: +filter3() +{ + // filter based on nVARs and set some buggy_scores: + int min_diff = 10000, max_distance = -10000; + bool rslflag = true; // ENUM_RANK_NOTHING + for ( int i = 1; i < clusterbuffer.size(); i++ ) { + if ( clusterbuffer[i].uni_var_number != clusterbuffer[0].uni_var_number ) { + min_diff = min(min_diff, abs(clusterbuffer[i].uni_var_number - clusterbuffer[0].uni_var_number)); + rslflag = false; // ENUM_RANK_nVARS + } + if ( clusterbuffer[i].filename == clusterbuffer[0].filename ) { + int line_distance; + if ( clusterbuffer[i].begin_line_id >= clusterbuffer[0].begin_line_id ) + line_distance = clusterbuffer[i].begin_line_id - (clusterbuffer[0].begin_line_id + clusterbuffer[0].line_offset -1); + else + line_distance = clusterbuffer[0].begin_line_id - (clusterbuffer[i].begin_line_id + clusterbuffer[i].line_offset -1); + max_distance = max(max_distance, line_distance); + } else + max_distance = 10000; // TODO: compare absolute filenames + } + if ( rslflag==true ) + buggy_score[1] = 0; + else + buggy_score[1] = min_diff; + buggy_score[2] = max_distance; + return rslflag; +} + +int TokenTreeMap:: +buggy1() +{ + // filtering heuristic (see: notes.txt) + int hasCond = 0, hasLoop = 0, hasNone = 0, hasSwitch = 0; + int condInLoop = 0, ifInLevels = 0; + GetContextFuncT getContext = &TokenTreeMap::getContext2; + + for (int i = 0; i < clusterbuffer.size(); i++) { + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + Tree* forcompare2 = context2.context_node_begin; + if ( forcompare2 == NULL ) + continue; + Tree* contextparent = getContextParent(forcompare2); + /* + cerr << "Node: " << forcompare2->type << "=" << getTypeName(id2name, forcompare2->type) << " " + << "Parent: " << (contextparent==NULL ? -1 : contextparent->type) + << "=" << (contextparent==NULL ? "NULL" : getTypeName(id2name, contextparent->type)) << endl; + */ + if ( forcompare2->type == getTypeID(name2id, "select_or_iter_stmt") ) + forcompare2 = forcompare2->children[0]; + if ( contextparent!=NULL && contextparent->type == getTypeID(name2id, "select_or_iter_stmt") ) + contextparent = contextparent->children[0]; + + if ( forcompare2->type == getTypeID(name2id, "simple_if") || + forcompare2->type == getTypeID(name2id, "SWITCH") ) { + hasCond++; + if ( forcompare2->type == getTypeID(name2id, "SWITCH") ) + hasSwitch++; // hasSwitch <= hasCond + else // simple_if + if ( contextparent!=NULL && + ( contextparent->type == getTypeID(name2id, "simple_if") || + contextparent->type == getTypeID(name2id, "SWITCH") || + contextparent->type == getTypeID(name2id, "WHILE") || + contextparent->type == getTypeID(name2id, "FOR") || + contextparent->type == getTypeID(name2id, "do_stmt_start") ) ) + ifInLevels++; // ifInLevels <= hasCond-hasSwitch + if ( contextparent!=NULL && + ( contextparent->type == getTypeID(name2id, "WHILE") || + contextparent->type == getTypeID(name2id, "FOR") || + contextparent->type == getTypeID(name2id, "do_stmt_start") ) ) + condInLoop++; + } else if ( forcompare2->type == getTypeID(name2id, "WHILE") || + forcompare2->type == getTypeID(name2id, "FOR") || + forcompare2->type == getTypeID(name2id, "do_stmt_start") ) + hasLoop++; + else + hasNone++; + } + + setscores: + if ( hasLoop>0 && hasCond>0 ) + buggy_score[3] = condInLoop; + + if ( hasCond>0 && hasLoop==0 && hasNone==0 ) + return 4; + else if ( hasCond>0 && hasLoop==0 && hasNone>0 ) + if ( hasSwitch==hasCond || // switch vs. none + ifInLevels==hasCond-hasSwitch // all "if"s are deep inside + ) + return -5; + else + return 6; + else if ( hasCond>0 && hasLoop>0 && hasNone==0 ) + return 1; + else if ( hasCond>0 && hasLoop>0 && hasNone>0 ) + return 0; + else if ( hasCond==0 && hasLoop>0 && hasNone>0 ) + return -4; + else if ( hasCond==0 && hasLoop>0 && hasNone==0 ) + return -2; + else + return -8; +} + + +/***************************** + * TokenTreeMap For Java + * + ****************************/ +Tree* TokenTreeMap_Java:: +getContextNode(Tree* node) +{ + if ( node == NULL ) + return NULL; + Tree * rsl = node; + if ( isContextual(rsl) ) + if ( rsl->type == getTypeID(name2id, "class_declaration") + || rsl->type == getTypeID(name2id, "constructor_declaration") + || rsl->type == getTypeID(name2id, "finally") ) + rsl = rsl->parent; + else if ( rsl->type == getTypeID(name2id, "method_declaration") + || rsl->type == getTypeID(name2id, "interface_declaration") /* this case needs more care, but should be fine here */ + || rsl->type == getTypeID(name2id, "abstract_method_declaration") ) + rsl = rsl->parent->parent; + else if ( rsl->type == getTypeID(name2id, "interface_body") ) + rsl = rsl->parent->parent->parent; + else if ( rsl->type == getTypeID(name2id, "catch_clause") ) + do { + rsl = rsl->parent; + } while ( rsl->type != getTypeID(name2id, "try_statement") ); + return rsl; +} + +bool TokenTreeMap_Java:: +compareJContext2(CloneContextT & context1, CloneContextT & context2) +{ + // only compare the context_node_begin; + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else { // actual comparison here: + Tree* forcompare1 = context1.context_node_begin; + Tree* forcompare2 = context2.context_node_begin; + // treat "for"=="while", "if"=="switch" + // because of the particular grammar for Java, we need the ad-hoc hack: + if ( forcompare1->type == getTypeID(name2id, "if_then_statement") || + forcompare1->type == getTypeID(name2id, "if_then_else_statement") || + forcompare1->type == getTypeID(name2id, "if_then_else_statement_nsi") || + forcompare1->type == getTypeID(name2id, "switch_statement") ) + if ( forcompare2->type == getTypeID(name2id, "if_then_statement") || + forcompare2->type == getTypeID(name2id, "if_then_else_statement") || + forcompare2->type == getTypeID(name2id, "if_then_else_statement_nsi") || + forcompare2->type == getTypeID(name2id, "switch_statement") ) + return true; + else + return false; + else if ( forcompare1->type == getTypeID(name2id, "for_statement") || + forcompare1->type == getTypeID(name2id, "for_statement_nsi") || + forcompare1->type == getTypeID(name2id, "while_statement") || + forcompare1->type == getTypeID(name2id, "while_statement_nsi") ) + if ( forcompare2->type == getTypeID(name2id, "for_statement") || + forcompare2->type == getTypeID(name2id, "for_statement_nsi") || + forcompare2->type == getTypeID(name2id, "while_statement") || + forcompare2->type == getTypeID(name2id, "while_statement_nsi") ) + return true; + else + return false; + else if ( forcompare1->type == getTypeID(name2id, "catch_clause") || + forcompare1->type == getTypeID(name2id, "finally") ) + if ( forcompare2->type == getTypeID(name2id, "catch_clause") || + forcompare2->type == getTypeID(name2id, "finally") ) + return true; + else + return false; + else if ( forcompare1->type == forcompare2->type ) + return true; + else + return false; + } +} + +Tree* TokenTreeMap_Java:: +get_condition_within(Tree* node) +{ + Tree* rsl = NULL; + if ( node==NULL ) + return rsl; + + if ( node->type == getTypeID(name2id, "if_then_statement") || + node->type == getTypeID(name2id, "if_then_else_statement") || + node->type == getTypeID(name2id, "if_then_else_statement_nsi") ) + if ( node->children.size() >=4 ) + rsl = node->children[2]; + else + rsl = NULL; + else if ( node->type == getTypeID(name2id, "while_statement") || + node->type == getTypeID(name2id, "while_statement_nsi") ) + if ( node->children[0]->type == getTypeID(name2id, "while_expression") ) + rsl = node->children[0]->children[2]; + else if ( node->children.size() >=4 ) + rsl = node->children[2]; + else + rsl = NULL; + else if ( node->type == getTypeID(name2id, "for_statement") || + node->type == getTypeID(name2id, "for_statement_nsi") ) + if ( node->children.size() == 7 || node->children.size() == 5 ) + rsl = node->children[2]; + else + rsl = NULL; + else if ( node->type == getTypeID(name2id, "switch_statement") ) + if ( node->children[0]->children.size() >=4 ) + rsl = node->children[0]->children[2]; + else + rsl = NULL; + else if ( node->type == getTypeID(name2id, "do_statement") ) + rsl = node->children[4]; + else if ( node->type == getTypeID(name2id, "synchronized_statement") ) + if ( node->children.size() >=5 ) + rsl = node->children[2]; + else + rsl = NULL; + else if ( node->type == getTypeID(name2id, "throw_statement") ) + if ( node->children.size() >=3 ) + rsl = node->children[1]; + else + rsl = NULL; + else if ( node->type == getTypeID(name2id, "catch_clause") ) + if ( node->children[0]->children.size() ==4 && + node->children[0]->children[2]->type == getTypeID(name2id, "formal_parameter") ) + rsl = node->children[0]->children[2]; + else + rsl = NULL; + else + rsl = NULL; + + return rsl; +} + +bool TokenTreeMap_Java:: +compareJConditions(CloneContextT& context1, CloneContextT& context2) +{ + // only compare the context_node_begin; + if ( context1.context_node_begin==NULL ) + if ( context2.context_node_begin!=NULL ) + return false; + else + return true; + else if ( context2.context_node_begin==NULL ) + return false; + else { // actual condition comparison here: + Tree* forcompare1 = context1.context_node_begin; + Tree* forcompare2 = context2.context_node_begin; + // the following relies on the particular grammar for Java and jcontextualNodes.h: + + forcompare1 = get_condition_within(forcompare1); + forcompare2 = get_condition_within(forcompare2); + return compareTree(forcompare1, forcompare2); // exact match is not good: an extra pair of () may change the result. + } +} + +Tree* TokenTreeMap_Java:: +get_conditional_operator(Tree* node) +{ + // for Java only: + Tree * rsl = NULL; + if ( node == NULL ) + return rsl; + if ( node->type == getTypeID(name2id, "expression") + || node->type == getTypeID(name2id, "assignment_expression") + || node->type == getTypeID(name2id, "postfix_expression") + || node->type == getTypeID(name2id, "name") + || node->type == getTypeID(name2id, "primary") ) + return get_conditional_operator(node->children[0]); + else if ( node->type == getTypeID(name2id, "assignment") ) + return get_conditional_operator(node->children[1]); + else if ( node->type == getTypeID(name2id, "assignment_operator") ) + if ( node->children[0]->type == getTypeID(name2id, "assign_any") ) + return node->children[0]->children[0]; + else + return node->children[0]; + else if ( node->type == getTypeID(name2id, "conditional_expression") + || node->type == getTypeID(name2id, "conditional_or_expression") + || node->type == getTypeID(name2id, "conditional_and_expression") + || node->type == getTypeID(name2id, "inclusive_or_expression") + || node->type == getTypeID(name2id, "exclusive_or_expression") + || node->type == getTypeID(name2id, "and_expression") + || node->type == getTypeID(name2id, "equality_expression") + || node->type == getTypeID(name2id, "relational_expression") + || node->type == getTypeID(name2id, "shift_expression") + || node->type == getTypeID(name2id, "additive_expression") + || node->type == getTypeID(name2id, "multiplicative_expression") ) + if ( node->children.size() > 1 ) + return node->children[1]; + else + return get_conditional_operator(node->children[0]); + else if ( node->type == getTypeID(name2id, "unary_expression") + || node->type == getTypeID(name2id, "trap_overflow_corner_case") + || node->type == getTypeID(name2id, "unary_expression_not_plus_minus") ) + if ( node->children.size() > 1 ) + return node->children[0]; + else + return get_conditional_operator(node->children[0]); + else if ( node->type == getTypeID(name2id, "pre_increment_expression") + || node->type == getTypeID(name2id, "pre_decrement_expression") ) + return node->children[0]; + else if ( node->type == getTypeID(name2id, "post_increment_expression") + || node->type == getTypeID(name2id, "post_decrement_expression") ) + return node->children[1]; + else if ( node->type == getTypeID(name2id, "qualified_name") ) + return node->children[1]; + else if ( node->type == getTypeID(name2id, "formal_parameter") + || node->type == getTypeID(name2id, "error") + || node->type == getTypeID(name2id, "cast_expression") + || node->type == getTypeID(name2id, "simple_name") ) + return node; + else if ( node->type == getTypeID(name2id, "primary_no_new_array") ) + if ( node->children[0]->type == getTypeID(name2id, "OP_TK") ) + return get_conditional_operator(node->children[1]); + else if ( node->children.size() > 1 ) + return node->children[1]; + else if ( node->children[0]->type == getTypeID(name2id, "literal") ) + return node->children[0]->children[0]; + else if ( node->children[0]->type == getTypeID(name2id, "THIS_TK") + || node->children[0]->type == getTypeID(name2id, "class_instance_creation_expression") + || node->children[0]->type == getTypeID(name2id, "method_invocation") + || node->children[0]->type == getTypeID(name2id, "array_access") ) + return node->children[0]; + else if ( node->children[0]->type == getTypeID(name2id, "field_access") + || node->children[0]->type == getTypeID(name2id, "type_literals") ) + return node->children[0]->children[1]; + else + return node; + else if ( node->type == getTypeID(name2id, "array_creation_expression") ) + return node->children[0]->children[0]; + else + return node; + return rsl; +} + +bool TokenTreeMap_Java:: +isMainOperator(Tree * op1) +{ + if ( op1==NULL ) + return false; + else + return false; +} + +bool TokenTreeMap_Java:: +filter1() +{ + vector groups; + GetContextFuncT getContext = &TokenTreeMap::getContext2; + CompareContextFuncT compareContext = &TokenTreeMap_Java::compareJContext2; + + for (int i = 0; i < clusterbuffer.size(); i++) { + int j = 0; + int tmps = groups.size(); + for (; j < tmps; j++) { + CloneContextT context1 = ((*this).*getContext)(pair(groups[j]->tbid, groups[j]->teid), groups[j]->filename); + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + if ( (this->*compareContext)(context1, context2) ) { + // same context, so no need to store it: + break; + } + } + if ( j >= tmps ) { + // a different context: + groups.push_back(&clusterbuffer[i]); + } + } + + if ( groups.size() > 1 ) + return false; // ENUM_RANK_CXT_NODE; + else + return true; // ENUM_RANK_NOTHING; +} + +bool TokenTreeMap_Java:: +filter2() +{ + vector groups; + GetContextFuncT getContext = &TokenTreeMap::getContext2; + CompareContextFuncT compareCondition = &TokenTreeMap_Java::compareConditionalOperators; + + for (int i = 0; i < clusterbuffer.size(); i++) { + int j = 0; + int tmps = groups.size(); + for (; j < tmps; j++) { + CloneContextT context1 = ((*this).*getContext)(pair(groups[j][0]->tbid, groups[j][0]->teid), groups[j][0]->filename); + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + if ( (this->*compareCondition)(context1, context2) ) { + groups[j].push_back(&clusterbuffer[i]); + break; + } + } + if ( j >= tmps ) { + groups.push_back(ClonePointGroupT()); + groups.back().push_back(&clusterbuffer[i]); + } + } + + if ( groups.size() <= 1 ) { + return true; // ENUM_RANK_NOTHING + } else + return false; // ENUM_RANK_CXT_COND +} + +int TokenTreeMap_Java:: +buggy1() +{ + int hasCond = 0, hasLoop = 0, hasNone = 0, hasSwitch = 0, hasSync = 0, hasTry = 0; + int condInLoop = 0, ifInLevels = 0; + GetContextFuncT getContext = &TokenTreeMap::getContext2; + + for (int i = 0; i < clusterbuffer.size(); i++) { + CloneContextT context2 = (this->*getContext)(pair(clusterbuffer[i].tbid, clusterbuffer[i].teid), clusterbuffer[i].filename); + Tree* forcompare2 = context2.context_node_begin; + if ( forcompare2 == NULL ) + continue; + Tree* contextparent = getContextParent(forcompare2); + + if ( forcompare2->type == getTypeID(name2id, "if_then_statement") || + forcompare2->type == getTypeID(name2id, "if_then_else_statement") || + forcompare2->type == getTypeID(name2id, "if_then_else_statement_nsi") || + forcompare2->type == getTypeID(name2id, "switch_statement") ) { + hasCond++; + if ( forcompare2->type == getTypeID(name2id, "switch_statement") ) + hasSwitch++; // hasSwitch <= hasCond + else // if_* + if ( contextparent!=NULL && + ( contextparent->type == getTypeID(name2id, "if_then_statement") || + contextparent->type == getTypeID(name2id, "if_then_else_statement") || + contextparent->type == getTypeID(name2id, "if_then_else_statement_nsi") || + contextparent->type == getTypeID(name2id, "switch_statement") || + contextparent->type == getTypeID(name2id, "for_statement") || + contextparent->type == getTypeID(name2id, "for_statement_nsi") || + contextparent->type == getTypeID(name2id, "while_statement") || + contextparent->type == getTypeID(name2id, "while_statement_nsi") || + contextparent->type == getTypeID(name2id, "do_statement") ) ) + ifInLevels++; // ifInLevels <= hasCond-hasSwitch + if ( contextparent!=NULL && + ( contextparent->type == getTypeID(name2id, "for_statement") || + contextparent->type == getTypeID(name2id, "for_statement_nsi") || + contextparent->type == getTypeID(name2id, "while_statement") || + contextparent->type == getTypeID(name2id, "while_statement_nsi") || + contextparent->type == getTypeID(name2id, "do_statement") ) ) + condInLoop++; + } else if ( forcompare2->type == getTypeID(name2id, "for_statement") || + forcompare2->type == getTypeID(name2id, "for_statement_nsi") || + forcompare2->type == getTypeID(name2id, "while_statement") || + forcompare2->type == getTypeID(name2id, "while_statement_nsi") || + forcompare2->type == getTypeID(name2id, "do_statement") ) + hasLoop++; + else if ( forcompare2->type == getTypeID(name2id, "synchronized_statement") ) + hasSync++; + else if ( forcompare2->type == getTypeID(name2id, "throw_statement") || + forcompare2->type == getTypeID(name2id, "try_statement") || + forcompare2->type == getTypeID(name2id, "catch_clause") || + forcompare2->type == getTypeID(name2id, "finally") ) + hasTry++; + else + hasNone++; + } + + setscores: + if ( hasLoop>0 && hasCond>0 ) + buggy_score[3] = condInLoop; + + if ( hasSync!=0 && hasSync!=clusterbuffer.size() ) + return 3; + else if ( hasTry!=0 && hasTry!=clusterbuffer.size() ) + return 2; + else if ( hasCond>0 && hasLoop==0 && hasNone==0 ) + return 4; + else if ( hasCond>0 && hasLoop==0 && hasNone>0 ) + if ( hasSwitch==hasCond || // switch vs. none + ifInLevels==hasCond-hasSwitch // all "if"s are deep inside + ) + return -5; + else + return 6; + else if ( hasCond>0 && hasLoop>0 && hasNone==0 ) + return 1; + else if ( hasCond>0 && hasLoop>0 && hasNone>0 ) + return 0; + else if ( hasCond==0 && hasLoop>0 && hasNone>0 ) + return -4; + else if ( hasCond==0 && hasLoop>0 && hasNone==0 ) + return -2; + else + return -8; +} + diff --git a/src/vgen/treeTra/token-tree-map.h b/src/vgen/treeTra/token-tree-map.h new file mode 100755 index 0000000..acf4674 --- /dev/null +++ b/src/vgen/treeTra/token-tree-map.h @@ -0,0 +1,201 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _TOKEN_TREE_MAP_H_ +#define _TOKEN_TREE_MAP_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../include/ptree.h" +#include "vgen-config.h" +#include "token-counter.h" + +extern std::map name2id; +extern std::map id2name; /* */ +extern Tree *root; +extern FILE *yyin; + +void yyrestart( FILE *new_file ); /* switch input file for flex. */ +int yyparse(); +void id_init(); + +typedef enum { + ENUM_CLONE_INDEX, ENUM_CLONE_DIST, ENUM_CLONE_FILE, ENUM_CLONE_LINE, ENUM_CLONE_OFFSET, + ENUM_CLONE_TBID, ENUM_CLONE_TEID, ENUM_CLONE_NODE_KIND, ENUM_CLONE_NODE_NUM, ENUM_CLONE_nVARs, + ENUM_CLONE_THE_END +} pattern_enum_t; + +typedef struct _ClonePointT { + std::string linebuf; + long index; /* the ID of the vector. */ + float dist; + std::string filename; + long begin_line_id, line_offset; + long tbid, teid; + int node_kind, node_number; + int uni_var_number; + + bool parse(char * line, regex_t patterns[], int dim=ENUM_CLONE_THE_END); + ostream & out2html(ostream & os); + ostream & out2html0(ostream & os); + ostream & out2xml(ostream & os); + friend ostream & operator<< (ostream& os, const struct _ClonePointT & cp); +} ClonePointT, *PClonePointT; +typedef std::vector ClonePointGroupT, *PClonePointGroupT; /* Q: why can't use "ClonePointT&"? */ + +typedef enum { + ENUM_RANK_NOTHING = 0, + ENUM_RANK_CXT_NODE = 1, + ENUM_RANK_CXT_COND = 2, + ENUM_RANK_nVARS = 4, + ENUM_RANK_THE_END = 8 +} bug_rank_enum_t; + +typedef struct _CloneContextT { + /* Definitions of "context": + 1. the smallest common ancestor + one node (or a path of node) above; + 2. the smallest common ancestor which is a "if"/"while"/"switch", etc + */ + Tree * context_node_begin; + Tree * context_node_end; +} CloneContextT, *PCloneContextT; + +class TokenTreeMap { + public: + static regex_t clone_patterns[ENUM_CLONE_THE_END]; + static bool init_shared_data(); + + std::map fn2tree; /* map finenames to parse tree roots. */ + std::vector clusterbuffer; /* outputs for a clone cluster from LSH. */ + unsigned int rank; /* the higher, more buggy */ +#define NUM_BUGGY_SCORES 5 + int buggy_score[NUM_BUGGY_SCORES]; /* the higher, more buggy */ + /* buggy_score[0]: context types + buggy_score[1]: nVARs difference + buggy_score[2]: location distance + buggy_score[3]: cond. in loop. + buggy_score[4]: unused + */ + + TraGenConfiguration * vecGen_config; /* only used as a dummy for TokenRangeCounter. */ + std::vector contexualNodes; /* contexualNodes[i]==true iff the node kind is considered as contexts */ + TokenRangeCounter * token_range_counter; + public: + TokenTreeMap(); /* all nodes are contexual by default. */ + ~TokenTreeMap(); + ostream & outputCluster(ostream & out); + + bool initNodes(const char ** nodeconfig); + ParseTree* parseFile(const char * fn); + // use #define instead before we have a good general way of handling this: bool parseClonePointAux(char * line, PCloneContextT pcp, const regex_t *preg, size_t nmatch, regmatch_t pmatch[], int eflags); + bool parseClonePoint(char * line, PClonePointT pcp); /* parse a line from the clone report into ClonePointT */ + virtual bool createFN2Tree(); /* get each filename, parse the file, assign ID to each token, save . */ + void clearMap(); /* in order to save memory; called manually some times by users. */ + virtual bool isContextual(Tree* node); + private: + void clearBuffer(); /* automatically called in createFN2Tree to save memory and seperate a clone cluster from the previous one. */ + Tree* tokenRange2TreeAux1(std::pair tokenrange, Tree* node); + bool root2TokenAux(long tid, Tree* node, std::list&); + public: + Tree* tokenRange2Tree1(std::pair tokenrange, ParseTree* pt); /* return the smallest common ancestor, but a bug in it - TODO. */ + Tree* tokenRange2Tree2(std::pair tokenrange, ParseTree* pt); /* return the smallest common ancestor, different alg, OK. */ + std::list* root2Token(long tid, ParseTree* pt); /* return the path from the root to the terminal. */ + + /* NOTE: "pointer-to-member" function has a different type from normal C function pointer. cf. C++ FAQ LITE, Marshall Cline. + better to use "functionoid"! + */ + typedef CloneContextT (TokenTreeMap::*GetContextFuncT)(std::pair, std::string &); + typedef bool (TokenTreeMap::*CompareContextFuncT)(CloneContextT&, CloneContextT&); + typedef bool (TokenTreeMap::*ClusterFilterT)(); + + virtual CloneContextT getContext1(std::pair tokenrange, std::string & fn); /* see the definitions of "context" above. */ + virtual CloneContextT getContext2(std::pair tokenrange, std::string & fn); + virtual Tree* getContextNode(Tree* node); /* get the highest node which represents the context type of "node"; language dependent */ + virtual Tree* getContextParent(Tree* node); /* get the contextual parent; return NULL if not found. */ + + bool compareContext1(CloneContextT& context1, CloneContextT& context2); /* return true iff the contexts are the same. */ + bool compareCContext2(CloneContextT& context1, CloneContextT& context2); /* for C only */ + bool compareCConditions(CloneContextT& context1, CloneContextT& context2); /* for C only, compare conditions of "if" etc. */ + virtual Tree* get_conditional_operator(Tree* node); /* grammar-dependent */ + virtual bool isMainOperator(Tree * op1); /* define what are "first" "main" operators. grammar-dependent. */ + virtual Tree* get_condition_within(Tree* node); /* grammar-dependent */ + virtual bool compareConditionalOperators(CloneContextT& context1, CloneContextT& context2); /* compare the "first" "main" operators in conditions. */ + + virtual bool isAnyFiltered(); /* check "clusterbuffer" to see whether it may be filtered by ANY filter. TODO: make it generic to be able to accept any filter. + May need a list of function pointers. */ + virtual bool isAllFiltered(); /* check "clusterbuffer" to see whether it may be filtered by ALL filter. */ +#define isFilteredI(ttm, i, rslflag) /* check "clusterbuffer" to see whether it may be filtered by filter##i(). */ \ + { \ + rslflag = false; \ + if ( ttm.clusterbuffer.size() < 1 ) { \ + cerr << "No clone cluster is in the buffer..." << endl; \ + rslflag = true; \ + } else { \ + rslflag = ttm.filter##i (); \ + } \ + } + + virtual bool filter1(); + virtual bool filter2(); + virtual bool filter3(); + virtual int buggy1(); + // virtual int buggy2(); +}; + +class TokenTreeMap_Java : public TokenTreeMap { + public: + typedef bool (TokenTreeMap_Java::*CompareContextFuncT)(CloneContextT&, CloneContextT&); + + virtual Tree* getContextNode(Tree* node); /* get the highest node which represents the context type of "node"; language dependent */ + + bool compareJContext2(CloneContextT& context1, CloneContextT& context2); /* for Java only */ + bool compareJConditions(CloneContextT& context1, CloneContextT& context2); /* for Java only, compare conditions of "if" etc. */ + virtual Tree* get_conditional_operator(Tree* node); /* grammar-dependent */ + virtual bool isMainOperator(Tree* node); /* grammar-dependent */ + virtual Tree* get_condition_within(Tree* node); /* grammar-dependent */ + + virtual bool filter1(); + virtual bool filter2(); + // virtual bool filter3(); + virtual int buggy1(); +}; + +#endif diff --git a/src/vgen/treeTra/tra-gen.C b/src/vgen/treeTra/tra-gen.C new file mode 100755 index 0000000..c703e75 --- /dev/null +++ b/src/vgen/treeTra/tra-gen.C @@ -0,0 +1,215 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include "tra-gen.h" +#include "tree-accessor.h" + +#define VGDEBUG + +/****************************************** + * Implementation for TraGenMain + * + *****************************************/ +TraGenMain:: +TraGenMain(ParseTree* rt, const char * fn, FILE * out) + : parse_tree(rt), vecGen_config(NULL), token_merger(NULL), list_merger(NULL), + vecGen_outfile(out), token_counter(NULL), token_range_counter(NULL), + tree_serializer(NULL), vec_generator(NULL), vec_outputor(NULL) +{ + int mergeTokens = 30, mergeStride = 1, mergeLists = 3; + getParameters(fn, mergeTokens, mergeStride, mergeLists); + + vecGen_config = new TraGenConfiguration(rt, mergeTokens, mergeStride, mergeLists); + token_counter = new TokenCounter(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + token_range_counter = new TokenRangeCounter(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + tree_serializer = new RelevantNoAtomicParent_TreeSerializer(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + vec_generator = new VecGenerator(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + vec_outputor = new TraVecOutput(*vecGen_config, vecGen_outfile); // DO depend on vecGen_config->mergeTokens, NOT on moveStride. +} + +TraGenMain:: +TraGenMain(ParseTree* rt, int mergeTokens, int mergeStride, int mergeLists, FILE * out) + : parse_tree(rt), vecGen_config(NULL), token_merger(NULL), list_merger(NULL), + vecGen_outfile(out), token_counter(NULL), token_range_counter(NULL), + tree_serializer(NULL), vec_generator(NULL), vec_outputor(NULL) +{ + vecGen_config = new TraGenConfiguration(rt, mergeTokens, mergeStride, mergeLists); + token_counter = new TokenCounter(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + token_range_counter = new TokenRangeCounter(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + tree_serializer = new RelevantNoAtomicParent_TreeSerializer(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + vec_generator = new VecGenerator(*vecGen_config); // NOT depend on vecGen_config->mergeTokens/moveStride. + vec_outputor = new TraVecOutput(*vecGen_config, vecGen_outfile); // DO depend on vecGen_config->mergeTokens, NOT on moveStride. +} + +bool TraGenMain:: +getParameters(const char * fn, int & mergeTokens, int & mergeStride, int & mergeLists) +{ + bool flag = false; + if ( fn!=NULL ) { + ifstream configfile(fn); + try { + if ( configfile.is_open() ) { + if ( !configfile.eof() ) + configfile >> mergeTokens; + if ( !configfile.eof() ) + configfile >> mergeStride; + if ( !configfile.eof() ) + configfile >> mergeLists; + flag = true; + } else { + cerr << "Warning: can't open configuration file `" << fn << "', use previous parameters." << endl; + } + } catch (...) { + // do nothing. + } + // 'finally' not in c++/gcc? + configfile.close(); + } + return flag; +} + +TraGenMain:: +~TraGenMain() +{ + if ( token_merger!=NULL ) { + delete token_merger; + token_merger = NULL; + } + if ( list_merger!=NULL ) { + delete list_merger; + list_merger = NULL; + } + if ( vec_outputor!=NULL ) { + delete vec_outputor; + vec_outputor = NULL; + } + if ( vec_generator!=NULL ) { + delete vec_generator; + vec_generator = NULL; + } + if ( tree_serializer!=NULL ) { + delete tree_serializer; + tree_serializer = NULL; + } + if ( token_counter!=NULL ) { + delete token_counter; + token_counter = NULL; + } + if ( token_range_counter!=NULL ) { + delete token_range_counter; + token_range_counter = NULL; + } + if ( vecGen_config!=NULL ) { + delete vecGen_config; + vecGen_config = NULL; + } +} + +void TraGenMain:: +run(int startln, int endln) +{ + Tree* initial_inh = NULL; + // count token nnumbers and assign token IDs (and, optionally, set parent pointers for e) + token_counter->traverse(parse_tree->getRoot(), initial_inh); +#ifdef VGDEBUG + fprintf(stderr, "Total counted terminals:%ld\n", parse_tree->getRoot()->terminal_number); +#endif + // count token ranges for each node: mainly for bug finding purpose so far + token_range_counter->traverse(parse_tree->getRoot(), initial_inh); + + // generate basic vectors for nodes: + vec_generator->traverse(parse_tree->getRoot(), initial_inh); + + // output basic vectors for the lines are specified, + if(startln>endln || startln<0 || endln<0) { + cerr << "Error: no vec. Check startln<=endln && startln>=0 && endln>=0: " << startln << ", " << endln << endl; + cerr << "Also, startln==0 means default vec gen for all lines." << endl; + return; + } + cerr << "Line range used: " << (startln==0 ? "All " : " ") << "[" << startln << ", " << endln << "]" << endl; + if(startln>0) { + Tree* line_node = parse_tree->line2Tree(startln, endln); + if(line_node!=NULL) { + TreeVector* tv = TreeAccessor::get_node_vector(line_node); + if (! tv->output(vecGen_outfile) ) { + cerr << "Warning: no vec." << endl; + } + } + return; + } + + // otherwise, output basic vectors for all nodes + if ( vecGen_config->mergeTokens>0 ) + vec_outputor->traverse(parse_tree->getRoot(), initial_inh); + else if ( vecGen_config->mergeTokens==0 ) + vec_outputor->multipleTraverse(parse_tree->getRoot(), initial_inh); + +#ifdef VGDEBUG + fprintf(stderr, "# basic vectors:%ld\n", vec_outputor->nNodeVectors); +#endif + + if ( vecGen_config->moveStride<=0 ) + return; + + // serialize the whole tree in post order and assign ids to nodes: + tree_serializer->traverse(parse_tree->getRoot(), initial_inh); +#ifdef VGDEBUG + fprintf(stderr, "Total nodes > %ld (some children of atomic nodes are skipped), # nodes in the Sq tree:%ld\n", tree_serializer->id, tree_serializer->sqtree_length()); +#endif +#ifdef outputnodeids + Tree* tree_itr = tree_serializer->serialized_tree.chain_header; + while ( tree_itr!=NULL ) { + fprintf(stdout, "Tree %p id=%ld (%s), tokens = %ld, low_id = %ld, value=`%s'\n", tree_itr, TreeAccessor::get_serialized_id(tree_itr), parse_tree->getTypeName(tree_itr->type).c_str(), tree_itr->terminal_number, TreeAccessor::get_serialized_low_id(tree_itr), tree_itr->isTerminal()?tree_itr->toTerminal()->value->c_str():""); + tree_itr = TreeAccessor::get_serialized_next_neighbor(tree_itr); + } +#endif + + // create different vector mergers: + token_merger = new VectorMergerOnTokens(tree_serializer->serialized_tree, *vecGen_config); // DO depend on vecGen_config->mergeTokens/moveStride. + vec_outputor->addMerger(token_merger); +#ifdef useVGDEBUGifwantthis + list_merger = new VectorMergerOnLists(tree_serializer->serialized_tree, *vecGen_config); // DO depend on vecGen_config->mergeTokens/moveStride. + vec_outputor->addMerger(list_merger); +#endif + + // output merged vectors + if ( vecGen_config->mergeTokens==0 ) + vec_outputor->multipleOutputMergedVectors(); + else + vec_outputor->outputMergedVectors(); +#ifdef VGDEBUG + fprintf(stderr, "# vectors:%ld, including must-gen nodes:%ld\n", vec_outputor->nAllOutputedVectors(), vec_outputor->nNodeVectors); +#endif +} + diff --git a/src/vgen/treeTra/tra-gen.h b/src/vgen/treeTra/tra-gen.h new file mode 100755 index 0000000..e1a25de --- /dev/null +++ b/src/vgen/treeTra/tra-gen.h @@ -0,0 +1,70 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _TRA_GEN_H_ +#define _TRA_GEN_H_ + +#include +#include "../../include/ptree.h" +#include "vgen-config.h" +#include "token-counter.h" +#include "sq-tree.h" +#include "node-vec-gen.h" +#include "vector-output.h" +#include "vector-merger.h" + +class TraGenMain { + ParseTree * parse_tree; + TraGenConfiguration * vecGen_config; + TokenCounter * token_counter; + TokenRangeCounter * token_range_counter; + TreeSerializer * tree_serializer; + VecGenerator * vec_generator; + TraVecOutput * vec_outputor; + VectorMerger * token_merger; + VectorMerger * list_merger; /* TODO: being debugged; too inefficient. */ + FILE * vecGen_outfile; + + public: + TraGenMain(ParseTree* rt, const char * fn, FILE * out); + public: + TraGenMain(ParseTree* rt, int mergeTokens, int mergeStride, int mergeLists, FILE * out); + + static bool getParameters(const char * fn, int & mergeTokens, int & mergeStride, int & mergeLists); + + ~TraGenMain(); + + void run(int startln=0, int endln=0); +}; + +#endif /* _TRA_GEN_H_ */ + diff --git a/src/vgen/treeTra/tree-accessor.C b/src/vgen/treeTra/tree-accessor.C new file mode 100755 index 0000000..239d38d --- /dev/null +++ b/src/vgen/treeTra/tree-accessor.C @@ -0,0 +1,342 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "tree-accessor.h" + +using namespace std; + +/****************************************** + * Implementation for TreeAccessor + * + *****************************************/ +Tree* TreeAccessor:: +get_serialized_next_neighbor(Tree* t) +{ + map::iterator attr_itr = t->attributes.find(NODE_SERIALIZED_NEIGHBOR); + assert ( attr_itr!=t->attributes.end() ); + + return ((pair*)(*attr_itr).second)->second; +} + +Tree* TreeAccessor:: +get_serialized_previous_neighbor(Tree* t) +{ + map::iterator attr_itr = t->attributes.find(NODE_SERIALIZED_NEIGHBOR); + assert ( attr_itr!=t->attributes.end() ); + + return ((pair*)(*attr_itr).second)->first; +} + +long TreeAccessor:: +get_serialized_id(Tree* t) +{ + map::iterator attr_itr = t->attributes.find(NODE_ID); + assert ( attr_itr!=t->attributes.end() ); + + return ((pair*)(*attr_itr).second)->first; +} + +long TreeAccessor:: +get_serialized_low_id(Tree* t) +{ + map::iterator attr_itr = t->attributes.find(NODE_ID); + assert ( attr_itr!=t->attributes.end() ); + + return ((pair*)(*attr_itr).second)->second; +} + +TreeVector* TreeAccessor:: +get_node_vector(Tree* t) +{ + map::iterator attr_itr = t->attributes.find(NODE_VECTOR); + assert ( attr_itr!=t->attributes.end() ); + + return (TreeVector*)(*attr_itr).second; +} + +bool TreeAccessor:: +is_tree_in_subtree(Tree* t, Tree* p) +{ + // assert ( t!=NULL && p!=NULL ); + map::iterator attr_id = t->attributes.find(NODE_ID), + attr_id2 = p->attributes.find(NODE_ID); + assert ( attr_id!=t->attributes.end() && attr_id2!=p->attributes.end() ); + + pair * t_id = (pair*)(*attr_id).second, * p_id = (pair*)(*attr_id2).second; + if ( p_id->second<=t_id->first && p_id->first>=t_id->first ) + return true; + else + return false; +} + +bool TreeAccessor:: +is_tree_node_in_range(Tree* t, VectorMerger & vm) +{ + if ( t==NULL ) + return false; + + // assert ( front!=NULL ); + map::iterator attr_itr = t->attributes.find(NODE_ID), + front_attr_itr = vm.front->attributes.find(NODE_ID); + assert( attr_itr!=t->attributes.end() && front_attr_itr!=vm.front->attributes.end() ); + pair * t_id = (pair*)(*attr_itr).second, * front_id = (pair*)(*front_attr_itr).second; + + if ( vm.tail==NULL ) { + if ( t_id->first>=front_id->first ) + return true; +// goto ad_hoc_check_to_avoid_bug_caused_by_duplicated_ids; + else + return false; + } else { + map::iterator tail_attr_itr = vm.tail->attributes.find(NODE_ID); + assert ( tail_attr_itr!=vm.tail->attributes.end() ); + pair * tail_id = (pair*)(*tail_attr_itr).second; + + if ( t_id->first>=front_id->first && t_id->first<=tail_id->first ) { + return true; +// goto ad_hoc_check_to_avoid_bug_caused_by_duplicated_ids; + } + else + return false; + } +#if 0 + ad_hoc_check_to_avoid_bug_caused_by_duplicated_ids: + // better to separate computing node ids from couting tokens. + Tree* chain_itr = vm.front; + while ( chain_itr!=vm.tail && chain_itr!=NULL ) { + if ( chain_itr==t ) + return true; + chain_itr = get_serialized_next_neighbor(chain_itr); + } + return false; +#endif +} + +bool TreeAccessor:: +is_tree_inrange_complete(Tree* t, VectorMerger & vm) +{ + // check t in range or not. + if ( is_tree_node_in_range(t, vm)==false ) + return false; + + // check whether t is complete in the sliding window. + map::iterator attr_itr = t->attributes.find(NODE_ID), + front_attr_itr = vm.front->attributes.find(NODE_ID); + assert( attr_itr!=t->attributes.end() && front_attr_itr!=vm.front->attributes.end() ); + pair * t_id = (pair*)(*attr_itr).second, * front_id = (pair*)(*front_attr_itr).second; + + if ( vm.tail==NULL ) { + if ( t_id->second>=front_id->first ) + return true; + else + return false; + } else { + map::iterator tail_attr_itr = vm.tail->attributes.find(NODE_ID); + assert ( tail_attr_itr!=vm.tail->attributes.end() ); + pair * tail_id = (pair*)(*tail_attr_itr).second; + + if ( t_id->second>=front_id->first && t_id->first<=tail_id->first ) + return true; + else if ( t_id->first>=front_id->first && t_id->first<=tail_id->first + && vm.vecGen_config.isAtomic(t) ) + return true; + else + return false; + } +} + +Tree* TreeAccessor:: +get_greatest_relevant_ancestor_inrange(Tree* t, VectorMerger & vm) +// maybe return t itself; return NULL iff t is outside the sliding window. +{ + list ancestors = get_all_relevant_ancestors_inrange(t, vm); + + if ( ancestors.size()<=0 ) + return NULL; + else + return ancestors.front(); +} + +std::list TreeAccessor:: +get_all_relevant_ancestors_inrange(Tree* t, VectorMerger & vm) +// return all unskippable ancestors in range, including t itself. +{ + list ancestors; + Tree* p = t; + + // get all ancestors in range; may include t itself. + // p is not within range ==> p->parent is not within range. + while ( p!=NULL && is_tree_node_in_range(p, vm)==true ) { + if ( vm.vecGen_config.isSkippable(p)==false ) + ancestors.push_front(p); + p = p->parent; + } + + return ancestors; +} + +Tree* TreeAccessor:: +get_greatest_atomic_relevant_ancestor_inrange(Tree* t, VectorMerger & vm) +{ + list ancestors = get_all_relevant_ancestors_inrange(t, vm); + + // find the oldest atomic ancestor (may be itself). + for (list::iterator anc_itr = ancestors.begin(); anc_itr!=ancestors.end(); ++anc_itr) + if ( vm.vecGen_config.isAtomic(*anc_itr)==true ) + return *anc_itr; + + return NULL; // no atomic ancestor in range. +} + +list TreeAccessor:: +get_all_relevant_ancestors_in_parsetree(Tree* t, TraGenConfiguration & cfg) +{ + list ancestors; + Tree* p = t; + + // get all ancestors in the whole tree; may include t itself. + while ( p!=NULL ) { + if ( cfg.isSkippable(p)==false ) + ancestors.push_front(p); + p = p->parent; + } + + return ancestors; +} + +Tree* TreeAccessor:: +get_greatest_atomic_relevant_ancestor_in_parsetree(Tree* t, TraGenConfiguration & cfg) +{ + list ancestors = get_all_relevant_ancestors_in_parsetree(t, cfg); + + // find the oldest atomic ancestor (may be itself). + for (list::iterator anc_itr = ancestors.begin(); anc_itr!=ancestors.end(); ++anc_itr) + if ( cfg.isAtomic(*anc_itr)==true ) + return *anc_itr; + + return NULL; // no atomic ancestor in the whole tree. +} + +Tree* TreeAccessor:: +get_greatest_complete_ancestor_inrange(Tree* t, VectorMerger & vm) +{ + // linearly search backward to find the oldest complete ancestor of + // t. Could be t itself. + // assert ( t is within [front, tail) ); + + Tree* chain_itr = NULL; + + if ( t==NULL || vm.front==NULL ) + return NULL; + if ( vm.tail==NULL ) + chain_itr = vm.serialized_tree.chain_tail; + else + chain_itr = get_serialized_previous_neighbor(vm.tail); + + long front_id = get_serialized_id(vm.front); + long t_low_id = get_serialized_low_id(t); + long t_id = get_serialized_id(t); + + while ( chain_itr!=vm.front ) { + long tmp_id = get_serialized_id(chain_itr); + long tmp_low_id = get_serialized_low_id(chain_itr); + + if ( tmp_low_id<=t_low_id && tmp_low_id>=front_id ) + return chain_itr; + else + chain_itr = get_serialized_previous_neighbor(chain_itr); + } + if ( t==vm.front && t_low_id>=front_id ) // t is a terminal + return t; + else + return NULL; +} + +Tree* TreeAccessor:: +get_greatest_mergeable_ancestor_inrange(Tree* t, VectorMerger & vm) +{ + // definition for "mergeable" is one a piece of paper... + + list ancestors = get_all_relevant_ancestors_inrange(t, vm); + for (list::iterator anc_itr = ancestors.begin(); anc_itr!=ancestors.end(); ++anc_itr) { + Tree * chain_itr = (*anc_itr); + assert( is_tree_node_in_range(chain_itr, vm)==true ); + + // An example of "logical" programming which determines whether a + // node is "mergeable" in the sliding window ;-) + if ( vm.vecGen_config.isAtomic(chain_itr)==true ) { + if ( vm.vecGen_config.isSkippable(chain_itr)==false ) + return chain_itr; + else continue; + } else { + if ( vm.vecGen_config.isSkippable(chain_itr)==false && is_tree_inrange_complete(chain_itr, vm)==true ) + return chain_itr; + else continue; + } + } // may return t itself. + + return NULL; // nodes above t (including t itself) are all + // unmergeable; then could move vm.front to the node + // after t's greatest_ancestor_inrange. +} + +Tree* TreeAccessor:: +get_youngest_unskippable_child(Tree* t, VectorMerger & vm) /* could be t itself */ +{ + Tree* chain_itr = t; + Tree* atomic_anc = NULL; + Tree* result = NULL; + + if ( t==NULL ) + return NULL; + + // too slow now: + while ( chain_itr!=NULL && is_tree_in_subtree(chain_itr, t)==true ) { + if ( vm.vecGen_config.isSkippable(chain_itr)==true ) + ; + else if ( vm.vecGen_config.isMergeable(chain_itr)==false ) + ; + else if ( atomic_anc!=NULL && is_tree_in_subtree(chain_itr, atomic_anc) ) + ; + else + result = chain_itr; + + if ( vm.vecGen_config.isAtomic(chain_itr) ) { + if ( atomic_anc==NULL || is_tree_in_subtree(chain_itr, atomic_anc)==false ) + atomic_anc = chain_itr; + } + chain_itr = get_serialized_previous_neighbor(chain_itr); + } + + return t; +} + diff --git a/src/vgen/treeTra/tree-accessor.h b/src/vgen/treeTra/tree-accessor.h new file mode 100755 index 0000000..5a9a132 --- /dev/null +++ b/src/vgen/treeTra/tree-accessor.h @@ -0,0 +1,66 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _TREE_ACCESSOR_H_ +#define _TREE_ACCESSOR_H_ + +#include +#include "../../include/ptree.h" +#include "vgen-config.h" +#include "vector-merger.h" + +/* wrappers for accessors to tree nodes */ +class TreeAccessor { + public: + static Tree* get_serialized_next_neighbor(Tree* t); + static Tree* get_serialized_previous_neighbor(Tree* t); + static long get_serialized_id(Tree* t); + static long get_serialized_low_id(Tree* t); + static TreeVector* get_node_vector(Tree* t); + + static bool is_tree_in_subtree(Tree* t, Tree* p); + static bool is_tree_node_in_range(Tree* t, VectorMerger & vm); + /* decide whether the whole subtree t is within the sliding window. */ + static bool is_tree_inrange_complete(Tree* t, VectorMerger & vm); + + static Tree* get_greatest_relevant_ancestor_inrange(Tree* t, VectorMerger & vm); + static std::list get_all_relevant_ancestors_inrange(Tree* t, VectorMerger & vm); + static Tree* get_greatest_atomic_relevant_ancestor_inrange(Tree* t, VectorMerger & vm); + static std::list get_all_relevant_ancestors_in_parsetree(Tree* t, TraGenConfiguration & cfg); + static Tree* get_greatest_atomic_relevant_ancestor_in_parsetree(Tree* t, TraGenConfiguration & cfg); /* t is also an ancestor of itself. */ + static Tree* get_least_complete_ancestor_inrange(Tree* t, VectorMerger & vm); + static Tree* get_greatest_complete_ancestor_inrange(Tree* t, VectorMerger & vm); /* could be t itself */ + static Tree* get_greatest_mergeable_ancestor_inrange(Tree* t, VectorMerger & vm); + static Tree* get_youngest_unskippable_child(Tree* t, VectorMerger & vm); /* could be t itself */ +}; + +#endif /* _TREE_ACCESSOR_H_ */ diff --git a/src/vgen/treeTra/tree-traversal.C b/src/vgen/treeTra/tree-traversal.C new file mode 100755 index 0000000..b223a5c --- /dev/null +++ b/src/vgen/treeTra/tree-traversal.C @@ -0,0 +1,157 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +/* The file defines the generic tree traversal interface, shameless + adapted from ROSE: a compiler infrastructure from Dan Quinlan at + LLNL. The difference is mainly in the tree node structure, so maybe + we can have a tree traversal mechanism generator for different tree + node structures ;-) */ +#ifndef _TREE_TRAVERSAL_C_ +#define _TREE_TRAVERSAL_C_ + +#include +#include "../../include/ptree.h" + +typedef enum { + TT_PREORDER, + TT_INORDER, + TT_POSTORDER +} t_traverseOrder; + +template +class ParseTreeTraversal { + public: + typedef std::vector SynthesizedAttributesList; + SynthesizedAttributeType traverse(Tree* basenode, + InheritedAttributeType inheritedValue, + t_traverseOrder travOrder=TT_PREORDER); + + ParseTreeTraversal/**/(); + virtual ~ParseTreeTraversal(); + + /* TODO: how to recognize whether a synthesized attribute is from defaultSynthesizedAttribute? */ + + virtual bool skipNode(Tree* astNode, InheritedAttributeType inh)=0; /* skip astNode (not affect the subtree under astNode) */ + virtual bool skipSubTree(Tree* astNode, InheritedAttributeType inh)=0; /* skip the subtree under astNode (not include astNode) */ + virtual InheritedAttributeType evaluateInheritedAttribute(Tree* astNode, + InheritedAttributeType inheritedValue)=0; + virtual SynthesizedAttributeType evaluateSynthesizedAttribute(Tree* node, + InheritedAttributeType in, + SynthesizedAttributesList& l)=0; + virtual InheritedAttributeType defaultInheritedAttribute(Tree* node, + InheritedAttributeType inh); + virtual SynthesizedAttributeType defaultSynthesizedAttribute(Tree* node, + InheritedAttributeType inh, + SynthesizedAttributesList& synl); + +}; + + +//bad using namespace std; + +template +ParseTreeTraversal:: +ParseTreeTraversal( ) +{ +} + +template +ParseTreeTraversal:: +~ParseTreeTraversal( ) +{ +} + +template +SynthesizedAttributeType ParseTreeTraversal:: +traverse(Tree* node, InheritedAttributeType inheritedValue, t_traverseOrder treeTraversalOrder) +{ + SynthesizedAttributeType returnValue; //=defaultSynthesizedAttribute(inheritedValue); + std::vector subtreeSynthesizedAttributes; + + if ( treeTraversalOrder==TT_PREORDER ) { + if ( node && skipNode(node, inheritedValue)==false ) + inheritedValue = evaluateInheritedAttribute(node, inheritedValue); + else + inheritedValue = defaultInheritedAttribute(node, inheritedValue); + } + + if ( node && skipSubTree(node, inheritedValue)==false ) { + // Visit the children + int i=0; + for (std::vector::iterator iter=node->children.begin(); iter!=node->children.end(); (++iter, ++i)) { + // NOTE: because this for loop doesn't check the validity of + // children's synthesized attributes, users of this functions + // should check them. The same for inherited attributes from + // parents. + if (*iter) { + SynthesizedAttributeType subtreeReturnValue = traverse(*iter, inheritedValue, treeTraversalOrder); + subtreeSynthesizedAttributes.push_back(subtreeReturnValue); + } else { + SynthesizedAttributeType defaultValue=defaultSynthesizedAttribute(NULL, inheritedValue, subtreeSynthesizedAttributes); + subtreeSynthesizedAttributes.push_back(defaultValue); + } + } + } else + ; // let subtreeSynthesizedAttributes be empty; + + if( treeTraversalOrder!=TT_PREORDER ) { + if ( node && skipNode(node, inheritedValue)==false ) + inheritedValue = evaluateInheritedAttribute(node, inheritedValue); + else + inheritedValue = defaultInheritedAttribute(node, inheritedValue); + } + + if ( node && skipNode(node, inheritedValue)==false ) + returnValue = evaluateSynthesizedAttribute(node, inheritedValue, subtreeSynthesizedAttributes); + else + returnValue = defaultSynthesizedAttribute(node, inheritedValue, subtreeSynthesizedAttributes); + + return returnValue; +} + +template +InheritedAttributeType ParseTreeTraversal:: +defaultInheritedAttribute(Tree* node, InheritedAttributeType inh) +{ + return inh; +} + +template +SynthesizedAttributeType ParseTreeTraversal:: +defaultSynthesizedAttribute(Tree* node, InheritedAttributeType inh, SynthesizedAttributesList& synl) +{ + SynthesizedAttributeType s; + return s; +} + +#endif // _TREE_TRAVERSAL_C_ + diff --git a/src/vgen/treeTra/tree-vector.C b/src/vgen/treeTra/tree-vector.C new file mode 100755 index 0000000..a87a812 --- /dev/null +++ b/src/vgen/treeTra/tree-vector.C @@ -0,0 +1,397 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include "tree-vector.h" + +using namespace std; + +#define VGDEBUG + +/****************************************** + * Implementation for TreeVector + * + *****************************************/ +TreeVector:: +TreeVector(int len, int nl, const char * fn) + : counters(len, 0), nLines(nl), filename(fn), minLine(0), maxLine(0), + token_begin_id(-1), token_end_id(-1), + node(NULL), name_counters(), ordered_names() +{ +} + +TreeVector:: +TreeVector(ParseTree* rt) + : counters(rt->typeCount(), 0), filename(rt->filename.c_str()), minLine(0), maxLine(0), + token_begin_id(-1), token_end_id(-1), + node(NULL), name_counters(), ordered_names() +{ + nLines = rt->getRoot()->max; // casued seg faults: - rt->getRoot()->min + 1; +} + +TreeVector:: +TreeVector(const TreeVector & cp) + : counters(cp.counters), nLines(cp.nLines), filename(cp.filename), minLine(cp.minLine), maxLine(cp.maxLine), + token_begin_id(cp.token_begin_id), token_end_id(cp.token_end_id), + node(cp.node), name_counters(cp.name_counters), ordered_names(cp.ordered_names) +{ +} + +void TreeVector:: +clearVector() +{ + // clear node counts: + for (int i=0; ifilename==cv.filename ) + return true; + else if ( this->filename==NULL || cv.filename==NULL ) + return false; + else if ( strcmp(this->filename, cv.filename)==0 ) + return true; + else + return false; +} + +bool TreeVector:: +increaseCounters(Tree* n) +{ + if ( n->type<0 || n->type>=counters.size() ) + return false; + + // increase node counts: + counters[n->type] += 1; + + // update token range: + map::iterator attr_itr = n->attributes.find(NODE_TOKEN_ID); + assert( attr_itr!=n->attributes.end() ); + if ( token_begin_id<0 || ( ((pair*)(*attr_itr).second)->first>=0 && ((pair*)(*attr_itr).second)->first*)(*attr_itr).second)->first; + if ( token_end_id<0 || ((pair*)(*attr_itr).second)->second>token_end_id ) + token_end_id = ((pair*)(*attr_itr).second)->second; + + if ( n->isTerminal()==true ) { + // update line counts: + Terminal* tn = n->toTerminal(); +// lines.insert(tn->line); + if ( minLine<=0 || (tn->line>0 && tn->lineline; + if ( maxLine<=0 || tn->line>maxLine ) + maxLine = tn->line; + + // increase name counts: TODO, not all names should be recorded...cf. increaseVecCounters in TraGenConfiguration. + map::iterator id = name_counters.find(*(tn->value)); + if ( id==name_counters.end() ) + name_counters[*(tn->value)] = 1; + else + (*id).second += 1; + ordered_names.push_back(tn->value); + } + + return true; +} + +TreeVector TreeVector:: +operator+(const TreeVector & cv) +{ + TreeVector rsl(counters.size(), nLines, filename); + + // update token range: + if ( token_begin_id<0 ) + rsl.token_begin_id = cv.token_begin_id; + else if ( cv.token_begin_id<0 ) + rsl.token_begin_id = token_begin_id; + else + rsl.token_begin_id = min(token_begin_id, cv.token_begin_id); + rsl.token_end_id = max(token_end_id, cv.token_end_id); + + // merge line ranges: + if ( isFromSameFile(cv)==false ) { + // Meaningless maybe. TODO: ignore such cases... + assert ( false ); // because set lines doesn't support lines from different files... + string n1(this->filename); + string n2(cv.filename); + string n3 = n1+n2; + rsl.filename = stringclone(n3.c_str()); + + rsl.nLines = nLines + cv.nLines; + } else { + // assert (rsl.nLines == cv.nLines); +// set_union(lines.begin(), lines.end(), cv.lines.begin(), cv.lines.end(), inserter(rsl.lines, rsl.lines.begin())); + if ( minLine<=0 ) + rsl.minLine = cv.minLine; + else if ( cv.minLine<=0 ) + rsl.minLine = minLine; + else + rsl.minLine = min(minLine, cv.minLine); + rsl.maxLine = max(maxLine, cv.maxLine); + } + + // merge node vectors: + for(int i=0; i::const_iterator id2 = cv.name_counters.begin(); + id2!=cv.name_counters.end(); ++id2) { + map::iterator id1 = name_counters.find((*id2).first); + // assert ( (*id1).first==(*id2).first ); + if ( id1==name_counters.end() ) + rsl.name_counters[(*id2).first] = (*id2).second; + else + rsl.name_counters[(*id1).first] = (*id1).second + (*id2).second; + } + // TODO: the order may not be right at all cases. + rsl.ordered_names = ordered_names; + rsl.ordered_names.insert(rsl.ordered_names.end(), cv.ordered_names.begin(), cv.ordered_names.end()); + + return rsl; +} + +TreeVector & TreeVector:: +operator+=(const TreeVector & cv) +{ + // keep the "node" from *this*. + + // update token range: + if ( token_begin_id<0 ) + token_begin_id = cv.token_begin_id; + else if ( cv.token_begin_id<0 ) + ; + else + token_begin_id = min(token_begin_id, cv.token_begin_id); + token_end_id = max(token_end_id, cv.token_end_id); + + // merge line ranges: + if ( isFromSameFile(cv)==false ) { + // Meaningless maybe. TODO: ignore such cases... + assert (false); + string n1(this->filename); + string n2(cv.filename); + string n3 = n1+n2; + filename = stringclone(n3.c_str()); + + nLines += cv.nLines; + } else { + // assert (rsl.nLines == cv.nLines); +// set newlines; +// set_union(lines.begin(), lines.end(), cv.lines.begin(), cv.lines.end(), inserter(newlines, newlines.begin())); +// lines = newlines; + if ( minLine<=0 ) + minLine = cv.minLine; + else if ( cv.minLine<=0 ) + ; + else + minLine = min(minLine, cv.minLine); + maxLine = max(maxLine, cv.maxLine); + } + + // merge node vectors: + for (int i=0; i::const_iterator id2 = cv.name_counters.begin(); + id2!=cv.name_counters.end(); ++id2) { + map::iterator id1 = name_counters.find((*id2).first); + if ( id1==name_counters.end() ) + name_counters[(*id2).first] = (*id2).second; + else + (*id1).second = (*id1).second + (*id2).second; + } + // TODO: the order may not be right at all cases. + ordered_names.insert(ordered_names.end(), cv.ordered_names.begin(), cv.ordered_names.end()); + + return *this; +} + +TreeVector & TreeVector:: +operator=(const TreeVector & cv) +{ + // copy line counts: + filename = cv.filename; + nLines = cv.nLines; + // copy line ranges: +// lines = cv.lines; + minLine = cv.minLine; + maxLine = cv.maxLine; + + // copy token range: + token_begin_id = cv.token_begin_id; + token_end_id = cv.token_end_id; + + node = cv.node; + // copy node counts: + for (int i=0; i0 ) { +#endif + // output additional info: + fprintf (buf, "# FILE:%s, ", filename); + fprintf (buf, "LINE:%d, ", minlineno); + fprintf (buf, "OFFSET:%d, ", maxLineContained()); + // TODO: + fprintf (buf, "NODE_KIND:%d, ", node==NULL ? 0 : node->type); + fprintf (buf, "CONTEXT_KIND:%d, ", 0); + fprintf (buf, "NEIGHBOR_KIND:%d, ", 0); + + fprintf (buf, "NUM_NODE:%d, ", nNodesContained()); + // TODO + fprintf (buf, "NUM_DECL:0, NUM_STMT:0, NUM_EXPR:0, "); + // output token range: + fprintf(buf, "TBID:%d, TEID:%d, ", token_begin_id, token_end_id); +#define outputidentifiersforbugdetection +#ifdef outputidentifiersforbugdetection + // output identifiers for bug detection + fprintf (buf, "VARs:{"); +#ifdef outputeachidentifier + for (map::iterator vars = name_counters.begin(); + vars!=name_counters.end(); ++vars) { + fprintf (buf, "%s:%d, ", (*vars).first.c_str(), (*vars).second); + } +#endif + fprintf (buf, "}%d, ", nDiffNamesContained()); +#ifdef outputorderedidentifiers + // output ordered identifiers for bug detection + fprintf (buf, "OIDs:{"); + for (list::iterator oids = ordered_names.begin(); + oids!=ordered_names.end(); ++oids) { + fprintf (buf, "%s, ", (*oids)->c_str()); + } + fprintf (buf, "}%d", nNamesContained()); +#endif +#endif + fprintf (buf, "\n"); + + // output the vector itself: + for (int i=0; i TreeVector:: +tokenRange() +{ + return pair(token_begin_id, token_end_id); +} + +inline int TreeVector:: +nDiffNamesContained() +{ + return name_counters.size(); +} + +inline int TreeVector:: +nNamesContained() +{ + return ordered_names.size(); +} + diff --git a/src/vgen/treeTra/tree-vector.h b/src/vgen/treeTra/tree-vector.h new file mode 100755 index 0000000..1f59413 --- /dev/null +++ b/src/vgen/treeTra/tree-vector.h @@ -0,0 +1,89 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _TREE_VECTOR_H_ +#define _TREE_VECTOR_H_ + +#include +#include +#include +#include +#include +#include "../../include/ptree.h" +#include "vgen-utils.h" + +class TreeVector { + public: + std::vector counters; /* of length `len' */ + + /* TODO: these data fields are only good for intra-file vectors for now... */ + const char * filename; /* the name of the src file; (partially) use singleton to save mem. */ + int nLines; /* the total number of lines in "filename". */ +/* std::set lines; /\* use this instead of std::vector to improve performance. *\/ */ + int minLine, maxLine; /* use these instead of set to save memory; increase imprecision though. */ + long token_begin_id, token_end_id; /* *estimate* the (inclusive) range of tokens (from the source code) which the vector contains. Initialized to -1 (invalid). */ + Tree* node; /* pointer to the node from which the vector is generated; not always meaningful when merging vectors. */ + + /* record identifiers. */ + // std::map name_counters; /* Use string*, instead of strng, to save memory. Also consistent with ptree.h where string* is used for identifiers. But extra careful should be employed when looking for certain strings (even if they are literally equals, they may be treated as different strings because of pointers. So, it's easier to use "string" directly: */ + std::map name_counters; + std::list ordered_names; /* ordered names. It's fine to use string pointers here, because we want to treat literally equivalent strings as different things. */ + + TreeVector(int len, int nl=0, const char *fn=NULL); + TreeVector(ParseTree* rt); + TreeVector(const TreeVector & cp); + + void clearVector(); + bool isFromSameFile(const TreeVector & cv); + + bool increaseCounters(Tree* n); + + TreeVector operator+(const TreeVector & cv); + TreeVector & operator+=(const TreeVector & cv); + TreeVector & operator=(const TreeVector & cv); + + int nNodesContained(); /* the total number of nodes counted in the vector. */ + + bool output(FILE * out); + + int nLinesContained(); /* the total number of lines contained in the vector. */ + int minLineContained(); /* the minimal line number. */ + int maxLineContained(); /* the maximal line number. */ + std::pair tokenRange(); /* the token range */ + int nodeType(); /* the node type for this vector. TODO */ + + int nDiffNamesContained(); /* the number of different names counted in the vector. */ + int nNamesContained(); /* the total number of names counted. */ +}; + + +#endif /* _TREE_VECTOR_H_ */ diff --git a/src/vgen/treeTra/vector-merger.C b/src/vgen/treeTra/vector-merger.C new file mode 100755 index 0000000..4a255ff --- /dev/null +++ b/src/vgen/treeTra/vector-merger.C @@ -0,0 +1,670 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "vector-merger.h" +#include "tree-accessor.h" + +using namespace std; + +#define VGDEBUG + +/****************************************** + * Implementation for VectorMerger + * + *****************************************/ +VectorMerger:: +VectorMerger(SerializedTree & head, TraGenConfiguration & cfg) + : vecGen_config(cfg), serialized_tree(head), front(head.chain_header), tail(NULL), + ok_for_merge(false) +{ + mergeUnit = 1; // useful only in child classes. + if ( front!=NULL ) // this is possible when the parse tree has no nodes. + tail = TreeAccessor::get_serialized_next_neighbor(head.chain_header); // point to the next node; could be NULL when pointing to the end. +} + +bool VectorMerger:: +reset() +{ + front = serialized_tree.chain_header; + if ( front!=NULL ) + tail = TreeAccessor::get_serialized_next_neighbor(front); + else + tail = NULL; + ok_for_merge = false; + + return true; +} + +long VectorMerger:: +outputAllMergedVectors(FILE * out) +{ + TreeVector tv(vecGen_config.parse_tree); + long moved_steps = 0; // total numbers of steps the sliding window moved. + long nOutputedVectors = 0; // total number of vectors outputed. + + // handle the case when the serialized tree is too short, but is + // this really needed? TODO + if ( moveForward()==false ) { + mergeTreeVectors(&tv); + tv.output(out); + moved_steps++; + nOutputedVectors++; + goto return_check; + } + + do { + if ( moved_steps % vecGen_config.moveStride == 0 ) { // TODO: maybe not a good condition + tv.clearVector(); + mergeTreeVectors(&tv); + tv.output(out); + nOutputedVectors++; + } + moved_steps++; + } while ( moveForward()==true ); + + return_check: +#ifdef VGDEBUG + fprintf(stderr, "Moved steps = %ld, merged vectors = %ld\n", moved_steps, nOutputedVectors); +#endif + + return nOutputedVectors; +} + + +/****************************************** + * Implementation for VectorMergerOnTokens + * + *****************************************/ +VectorMergerOnTokens:: +VectorMergerOnTokens(SerializedTree & head, TraGenConfiguration & cfg) + : VectorMerger(head, cfg) +{ + mergeUnit = cfg.mergeTokens; +} + +bool VectorMergerOnTokens:: +mergeTreeVectors(TreeVector * mv) /* merge tree vectors into mv */ +{ + assert ( mv!=NULL ); + + if ( ok_for_merge==true ) { + int mergeable_counts = 0; + Tree * chain_itr = front; + +#ifdef detectthebugcausedbyduplicatenodeids + static int mvstep = 0; + mvstep++; + while ( chain_itr!=tail ) { + assert ( chain_itr!=NULL ); + fprintf(stdout, "Node %p of type:%d, id:%d, low_id:%d, tokens:%d\n", chain_itr, chain_itr->type, TreeAccessor::get_serialized_id(chain_itr), TreeAccessor::get_serialized_low_id(chain_itr), chain_itr->terminal_number); + Tree * mergeable_ancestor = TreeAccessor::get_greatest_mergeable_ancestor_inrange(chain_itr, *this); + if ( mergeable_ancestor!=NULL ) { + assert ( TreeAccessor::is_tree_node_in_range(mergeable_ancestor, *this)==true ); + chain_itr = mergeable_ancestor; + } + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } + fprintf(stdout, "==== %d steps ====\n", mvstep); + chain_itr = front; +#endif + + // NOTE#1: the sq-tree (instances of class + // RelevantNoAtomicParent_TreeSerializer) doesn't contain children + // of an atomic node or any skippable nodes. +#if 0 + while ( chain_itr!=tail /* hack to avoid seg faults: && chain_itr!=NULL */ ) { + if ( vecGen_config.isSkippable(chain_itr)==false ) { + Tree * mergeable_ancestor = TreeAccessor::get_greatest_mergeable_ancestor_inrange(chain_itr, *this); + if ( mergeable_ancestor!=NULL ) { + chain_itr = mergeable_ancestor; // may be still the same chain_itr; + } + + mv->operator+=(*TreeAccessor::get_node_vector(chain_itr)); + mergeable_counts += chain_itr->terminal_number; + } // end if unskippable + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } +#endif // Old implementation. + while ( chain_itr!=tail ) { + Tree* mergeable_ancestor = NULL; + if ( vecGen_config.isAtomic(chain_itr)==true || TreeAccessor::is_tree_inrange_complete(chain_itr, *this)==true ) { + mergeable_ancestor = TreeAccessor::get_greatest_complete_ancestor_inrange(chain_itr, *this); + assert ( mergeable_ancestor!=NULL ); // In general, mergeable_ancestor may be NULL, but not here. + + mv->operator+=(*TreeAccessor::get_node_vector(mergeable_ancestor)); + mergeable_counts += mergeable_ancestor->terminal_number; + chain_itr = mergeable_ancestor; + } + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } + +#ifdef checkmergedtokens + fprintf(stderr, "Merging %d tokens\n", mergeable_counts); +#endif + return true; + } else + return false; +} + +inline int VectorMergerOnTokens:: +enoughForMerge() /* decide whether the sliding window contains enough stuff for merging. */ +{ + int mergeable_counts = 0; + Tree * chain_itr = front; + +#if 0 + while ( chain_itr!=tail /* hack to avoid seg faults: && chain_itr==NULL */ ) { + if ( vecGen_config.isSkippable(chain_itr)==false ) { + Tree* mergeable_ancestor = TreeAccessor::get_greatest_mergeable_ancestor_inrange(chain_itr, *this); + // Here, it seems need to use get_greatest_atomic_ancestor_in_parsetree first (TODO). + // However, thanks to moveForwardOneStep, the greatest_atomic_ancestor_in_parsetree + // should be (almost) always within range. + + if ( mergeable_ancestor!=NULL ) + chain_itr = mergeable_ancestor; + //assert ( chain_itr!=NULL ); + mergeable_counts += chain_itr->terminal_number; + + if ( mergeable_counts >= mergeUnit ) { + ok_for_merge = true; + return mergeable_counts-mergeUnit; + } + } // end if unskippable + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } +#endif // old implementation. + while ( chain_itr!=tail ) { + Tree* mergeable_ancestor = NULL; + if ( vecGen_config.isAtomic(chain_itr)==true || TreeAccessor::is_tree_inrange_complete(chain_itr, *this)==true ) { + mergeable_ancestor = TreeAccessor::get_greatest_complete_ancestor_inrange(chain_itr, *this); + assert ( mergeable_ancestor!=NULL ); // In general, mergeable_ancestor may be NULL, but not here. + + mergeable_counts += mergeable_ancestor->terminal_number; + if ( mergeable_counts >= mergeUnit ) { + ok_for_merge = true; + return mergeable_counts-mergeUnit; + } + chain_itr = mergeable_ancestor; + } + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } + + // assert (mergeable_counts < mergeUnit); + ok_for_merge = false; + return mergeable_counts-mergeUnit; // return the number of token lacked +} + +inline bool VectorMergerOnTokens:: +moveForwardOneStep(Tree* & t) /* definition of "one step" is on a piece of paper. */ +// cf. NOTE#1. Return false iff t is not really moved forward; return +// true iff t is moved forward at least one token. Also, need to +// guarantee tail is always after front. +{ + assert ( t==front || t==tail ); + if ( t==NULL ) + return false; + + bool flag = false; + + // logical programming here ;-) + if ( t==front ) { + // assert ( t!=NULL && front!=NULL ); + Tree* chain_itr = TreeAccessor::get_serialized_next_neighbor(t); + while ( chain_itr!=NULL ) { + if ( chain_itr->terminal_number>0 && + (vecGen_config.isAtomic(chain_itr)==true || chain_itr->children.size()==0) ) { + t = chain_itr; + flag = true; + goto check_tail_is_after_front; + } + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } + + t = NULL; + goto check_tail_is_after_front; + + } else if ( t==tail ) { + // assert ( t!=NULL && tail!=NULL ); + Tree* chain_itr = TreeAccessor::get_serialized_next_neighbor(t); + while ( chain_itr!=NULL ) { + if ( chain_itr->terminal_number>0 && (vecGen_config.isAtomic(chain_itr)==true || + TreeAccessor::get_serialized_low_id(chain_itr)>TreeAccessor::get_serialized_id(t)) ) { + t = chain_itr; + flag = true; + goto check_tail_is_after_front; + } + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } + + if ( t->terminal_number>0 && ( vecGen_config.isAtomic(t)==true || t->isTerminal()==true ) ) + // The condition is used to prevent certain duplicate vectors (whose differences are only at nonterminals) and could be omitted. + flag = true; // the node pointed to by the tail is not counted, but it will be counted after the following assignment. + t = NULL; + goto check_tail_is_after_front; + } + + // check whether tail is after front: + check_tail_is_after_front: + if ( tail==NULL ) + return flag; + else if ( front==NULL ) { + tail = NULL; + return false; // no longer moveable; + } else if ( TreeAccessor::get_serialized_id(front) < TreeAccessor::get_serialized_id(tail) ) + return flag; + else { + tail = TreeAccessor::get_serialized_next_neighbor(front); + return true; + } +} + +inline bool VectorMergerOnTokens:: +moveBackwardOneStep(Tree* & t) +{ + // logic is not easy unless I simplify the serialized tree to + // contain leafNodes only (Ghassan's). + return false; +} + +bool VectorMergerOnTokens:: +moveForward() +// move forward until the next mergeable sliding window. +{ + Tree* chain_itr = NULL; + + if ( ok_for_merge==true ) { + // move front forward one step: + moveForwardOneStep(front); + } + +#ifdef outputmovementofslidingwindow + fprintf(stdout, "SW from id %d to id %d\n", front==NULL ? -1 : TreeAccessor::get_serialized_id(front), tail==NULL ? -1 : TreeAccessor::get_serialized_id(tail)); +#endif + // move tail forward until the next mergeable sliding window or end. + while ( enoughForMerge()<0 ) { + if ( moveForwardOneStep(tail)==false ) + return false; +#ifdef outputmovementofslidingwindow + fprintf(stdout, "SW from id %d to id %d\n", front==NULL ? -1 : TreeAccessor::get_serialized_id(front), tail==NULL ? -1 : TreeAccessor::get_serialized_id(tail)); +#endif + } + + if ( ok_for_merge==true ) + return true; + else + return false; +} + +bool VectorMergerOnTokens:: +moveBackward() // TODO: logic is not right yet. +{ + Tree* chain_itr = NULL; + int lackoftokens = enoughForMerge(); + + if ( lackoftokens>=0 ) { + // move tail/front backward one token: + chain_itr = tail; + if ( chain_itr==NULL ) + chain_itr = serialized_tree.chain_tail; + + while ( chain_itr!=NULL && ( vecGen_config.isSkippable(chain_itr)==true || chain_itr->isNonTerminal()==true ) ) { + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + } + tail == chain_itr; + + chain_itr = front; // assert ( front!=NULL ); + while ( vecGen_config.isSkippable(chain_itr)==true || chain_itr->isNonTerminal()==true ) { + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + + if ( chain_itr==NULL ) { // couldn't move backward anymore + ok_for_merge = false; + return false; + } + } + front = chain_itr; + + ok_for_merge = true; + return true; + } else { + // move front backward until enough for merging or return false + int forward_tokens = 0; + lackoftokens = -lackoftokens; + + chain_itr = front; // assert ( front!=NULL ); + while ( chain_itr!=NULL && forward_tokensisTerminal()==true ) + forward_tokens++; + + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + } + + if ( chain_itr==NULL ) + front = serialized_tree.chain_header; + else + front = chain_itr; + + if ( forward_tokens>=lackoftokens ) { + ok_for_merge = true; + return true; + } else { + ok_for_merge = false; + return false; + } + } +} + +bool VectorMergerOnTokens:: +reset() /* re-initialize front/tail, and mergeUnit. */ +{ + VectorMerger::reset(); + mergeUnit = vecGen_config.mergeTokens; + + return true; +} + + + +/****************************************** + * Implementation for VectorMergerOnLists + * + *****************************************/ +VectorMergerOnLists:: +VectorMergerOnLists(SerializedTree & head, TraGenConfiguration & cfg) + : VectorMerger(head, cfg), list_for_merge() +{ + mergeUnit = cfg.mergeLists; + + if ( front!=NULL && vecGen_config.isMergeable(front)==true && vecGen_config.isSkippable(front)==false ) + list_for_merge.push_back(front); +} + +bool VectorMergerOnLists:: +mergeTreeVectors(TreeVector * mv) /* merge tree vectors into mv */ +{ + assert ( mv!=NULL ); + + if ( ok_for_merge==true || /* at the end */(front==NULL && list_for_merge.size()>0) ) { + int mergeable_counts = 0; + for ( list::iterator itr = list_for_merge.begin(); itr!=list_for_merge.end(); ++itr) { + mv->operator+=(*(TreeAccessor::get_node_vector(*itr))); + mergeable_counts += (*itr)->terminal_number; + } + +#ifdef checkmergedtokens + fprintf(stderr, "Merging %d tokens\n", mergeable_counts); +#endif + + return true; + } else + return false; +} + +Tree* VectorMergerOnLists:: +get_previous_mergeable_node(Tree* t) +// could return NULL or t itself +{ + Tree* chain_itr = t; + + // search for a missed node backward. Again, a logic programming ;-) + while ( chain_itr!=NULL ) { + if ( vecGen_config.isMergeable(chain_itr)==true ) { + if ( vecGen_config.isSkippable(chain_itr)==false ) + break; + else if ( vecGen_config.isAtomic(chain_itr)==true ) { + chain_itr = TreeAccessor::get_youngest_unskippable_child(chain_itr, *this); + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + } + } else if ( vecGen_config.isAtomic(chain_itr)==true ) { + chain_itr = TreeAccessor::get_youngest_unskippable_child(chain_itr, *this); + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + } else + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + } + + return chain_itr; +} + +int VectorMergerOnLists:: +add_mergeable_nodes_before(Tree* t, int missed_nodes, list & node_store) +// assert ( t==front && node_store==list_for_merge ); +{ + if ( t==NULL || missed_nodes<=0 ) + return -1; + + Tree* chain_itr = t; + int added_nodes = 0; + + do { + chain_itr = get_previous_mergeable_node(chain_itr); + + if ( chain_itr==NULL ) + return added_nodes; + + node_store.push_front(chain_itr); + missed_nodes--; + added_nodes++; + + chain_itr = TreeAccessor::get_youngest_unskippable_child(chain_itr, *this); // assert ( chain_itr!=NULL && chain_itr is in the ST); + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + } while ( missed_nodes>0 && chain_itr!=NULL ); + + return added_nodes; +} + +Tree* VectorMergerOnLists:: +get_next_mergeable_node(Tree* t) +{ + Tree* chain_itr = t; + + // logic programming again. Getting systematic but bored ;-) + while ( chain_itr!=NULL ) { + if ( vecGen_config.isMergeable(chain_itr)==true ) { + if ( vecGen_config.isSkippable(chain_itr)==true ) + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + // didn't try to move to chain_itr's greatest atomic ancestor + // because atomic nodes are few, thus no too much time saving. + else { + Tree* ancestor = TreeAccessor::get_greatest_atomic_relevant_ancestor_in_parsetree(chain_itr, this->vecGen_config); + if ( ancestor==NULL || ancestor==chain_itr ) + return chain_itr; + else + chain_itr = ancestor; + } + } else + chain_itr = TreeAccessor::get_serialized_next_neighbor(chain_itr); + } + + return chain_itr; +} + +inline int VectorMergerOnLists:: +enoughForMerge() /* decide whether the sliding window contains enough stuff for merging. */ +{ + int listsize = list_for_merge.size(); + + if ( listsize < mergeUnit ) { + ok_for_merge = false; + return listsize-mergeUnit; + } else { + int tokens = 0; + for ( list::iterator itr = list_for_merge.begin(); itr!=list_for_merge.end(); ++itr) + tokens += (*itr)->terminal_number; + + if ( tokens < vecGen_config.mergeTokens ) { + // even all the lists are too small + ok_for_merge = false; + return -1-mergeUnit; + } else { + ok_for_merge = true; + return listsize-mergeUnit; + } + } +} + +bool VectorMergerOnLists:: +reset() /* re-initialize front/tail, list_for_merge, and mergeUnit. */ +{ + list_for_merge.clear(); + VectorMerger::reset(); + mergeUnit = vecGen_config.mergeLists; + + if ( front!=NULL && vecGen_config.isMergeable(front)==true && vecGen_config.isSkippable(front)==false ) { + list_for_merge.push_back(front); + } + + return true; +} + +inline bool VectorMergerOnLists:: +moveForwardOneStep(Tree* & t) +/* definition of "one step" is one mergeableNodes (be careful with "parents"). */ +// return false iff t is not really moved forward. +// The sliding window is ended at t, inclusively (different from the tail in [front, tail) ). +// assert ( t==front ); +{ + if ( t==NULL ) + return false; + + Tree* chain_itr; + if ( t==serialized_tree.chain_header && ok_for_merge==false && list_for_merge.size()==0 ) + // this is the first time moveForwardOneStep is called. + chain_itr = t; + else + chain_itr = TreeAccessor::get_serialized_next_neighbor(t); + + chain_itr = get_next_mergeable_node(chain_itr); + + if ( chain_itr!=NULL ) { + bool flag_makeup_passed_nodes = false; + + for (list::iterator itr = list_for_merge.begin(); itr!=list_for_merge.end(); ++itr) { + // delete trees which are children of chain_itr: + if ( TreeAccessor::is_tree_in_subtree(*itr, chain_itr)==true ) { + // now, nodes in [itr, list_for_merge.end()) are all children of chain_itr: + + if ( (*itr)->terminal_numberterminal_number ) + flag_makeup_passed_nodes = true; // may need to make up some previously missed nodes. + + list_for_merge.erase(itr, list_for_merge.end()); + + break; + } + } + + t = chain_itr; tail = t; + // add the chain_itr into list_for_merge + list_for_merge.push_back(t); + + // try to make up the previously missed nodes: + int missed_nodes = list_for_merge.size()-mergeUnit; + // assert ( t==chain_itr && chain_itr!=NULL && list_for_merge.size()>=1 ); + if ( flag_makeup_passed_nodes==true && missed_nodes>0 ) { + chain_itr = TreeAccessor::get_youngest_unskippable_child(list_for_merge.front(), *this); + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + // add missed mergeable nodes from chain_itr (include it) into list_for_merge. + add_mergeable_nodes_before(chain_itr, missed_nodes, list_for_merge); + } + + return true; + } else { + t = NULL; tail = NULL; + return false; + } +} + +inline bool VectorMergerOnLists:: +moveBackwardOneStep(Tree* & t) // logic not meaningful +// assert ( t==front ) +{ + /* finishing... */ + if ( t==NULL ) { + t = serialized_tree.chain_tail; + if ( ok_for_merge==false ) { + int missed_nodes = mergeUnit-list_for_merge.size(); + if ( missed_nodes<=0 ) + missed_nodes = 1; + + Tree* chain_itr = TreeAccessor::get_youngest_unskippable_child(list_for_merge.front(), *this); + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + add_mergeable_nodes_before(chain_itr, missed_nodes, list_for_merge); + } + } else { + if ( ok_for_merge==false ) { + int missed_nodes = mergeUnit-list_for_merge.size(); + if ( missed_nodes<=0 ) + missed_nodes = 1; + + Tree* chain_itr = TreeAccessor::get_youngest_unskippable_child(list_for_merge.front(), *this); + chain_itr = TreeAccessor::get_serialized_previous_neighbor(chain_itr); + add_mergeable_nodes_before(chain_itr, missed_nodes, list_for_merge); + } else { + Tree* chain_itr = get_previous_mergeable_node(t); + if ( chain_itr==NULL ) + return false; + + list_for_merge.clear(); + add_mergeable_nodes_before(chain_itr, mergeUnit, list_for_merge); + t = chain_itr; + } + } + + return false; +} + +bool VectorMergerOnLists:: +moveForward() +// move forward until the next mergeable sliding window. +{ + Tree* chain_itr = NULL; + + // if already ok_for_merge==true, delete one node in list_for_merge + // to move sliding window forward: + if ( ok_for_merge==true ) + list_for_merge.pop_front(); + + while ( enoughForMerge()<0 ) { + if ( moveForwardOneStep(front)==false ) + return false; + } + + if ( ok_for_merge==true ) + return true; + else + return false; +} + +bool VectorMergerOnLists:: +moveBackward() +{ + return false; +} + diff --git a/src/vgen/treeTra/vector-merger.h b/src/vgen/treeTra/vector-merger.h new file mode 100755 index 0000000..537ff3e --- /dev/null +++ b/src/vgen/treeTra/vector-merger.h @@ -0,0 +1,131 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _VECTOR_MERGER_H_ +#define _VECTOR_MERGER_H_ + +#include +#include "../../include/ptree.h" +#include "sq-tree.h" + +/* Interface for different vector merging algorithms based on + different sliding windows. They are based on serialized trees. */ +class VectorMerger { + /* This class implements vector merging based on token counts. */ + protected: + SerializedTree serialized_tree; + Tree *front, *tail; /* determine the boundary of the sliding window [front, tail). */ + int mergeUnit; + bool ok_for_merge; /* flag whether the number of trees reaches mergeUnit. */ + public: + TraGenConfiguration vecGen_config; + + public: + VectorMerger(SerializedTree & head, TraGenConfiguration & cfg); + + virtual bool mergeTreeVectors(TreeVector * mv)=0; /* merge tree vectors into mv */ + + /* make sure ok_for_merge is consistent with the mergeUnit and the silding window. */ + virtual int enoughForMerge()=0; /* decide whether the sliding window contains enough stuff for merging; return value: 1-enough, 0-just enough, -n-lack of n tokens. */ + virtual bool moveForwardOneStep(Tree* & t)=0; /* definition of "one step" is on a piece of paper. */ + virtual bool moveBackwardOneStep(Tree* & t)=0; + virtual bool moveForward()=0; /* move the sliding window forward to the next position ok for merging. */ + virtual bool moveBackward()=0; /* move the sliding window backward to the previous position ok for merging. */ + virtual bool reset(); /* re-initialize front/tail. */ + + virtual long outputAllMergedVectors(FILE * out); /* return number of outputed vectors */ + + friend class TreeAccessor; +}; + +class VectorMerger1 { + /* This class is basically a circulated queue. old interface, not flexible enough. Removed. */ + public: + int mergeUnit; /* >0 */ + Tree** subtrees; /* of length at most mergeUnit. */ + TreeVector* treevectors; /* of length the same as subtrees. */ + int front, tail; /* in the range of [0, mergeUnit) */ + bool ok_for_merge; /* flag whether the number of trees reaches mergeUnit. */ + + VectorMerger1(int mu); + + bool mergeTreeVectors(TreeVector * mv); /* merge treevectors into mv */ + bool addTree(Tree* t, TreeVector * v); /* add a tree into this merger. */ + bool removeTree(unsigned int n); /* remove the n oldest trees in the merger; n<=mergeUnit. */ + bool removeAllTrees(); /* empty the merger. */ +}; + +class VectorMergerOnTokens : public VectorMerger { + /* This class implements vector merging based on token counts. */ + public: + VectorMergerOnTokens(SerializedTree & head, TraGenConfiguration & cfg); + + virtual bool mergeTreeVectors(TreeVector * mv); /* merge tree vectors into mv */ + + /* make sure ok_for_merge is consistent with the mergeUnit and the silding window. */ + virtual int enoughForMerge(); /* decide whether the sliding window contains enough stuff for merging; return value: 1-enough, 0-just enough, -n-lack of n tokens. */ + virtual bool moveForwardOneStep(Tree* & t); /* definition of "one step" is on a piece of paper. */ + virtual bool moveBackwardOneStep(Tree* & t); + virtual bool moveForward(); /* move the sliding window forward to the next position ok for merging. */ + virtual bool moveBackward(); /* move the sliding window backward to the previous position ok for merging. */ + virtual bool reset(); /* re-initialize front/tail, and mergeUnit. */ + + friend class TreeAccessor; +}; + +class VectorMergerOnLists : public VectorMerger { + /* This class will implement vector merging for statement_list etc. TODO: not used yet */ + protected: + std::list list_for_merge; /* a list of nodes for merging, should be in the post-order because of the ordered identifiers used later for bug finding. */ + + public: + VectorMergerOnLists(SerializedTree & head, TraGenConfiguration & cfg); + + virtual bool mergeTreeVectors(TreeVector * mv); /* merge tree vectors into mv */ + + private: + Tree* get_previous_mergeable_node(Tree* t); /* go back from t (include t) to get the last mergeable node. could return NULL. */ + int add_mergeable_nodes_before(Tree* t, int missed_nodes, std::list & node_store); /* go back from t (include t) to get missed_nodes number mergeable nodes. */ + Tree* get_next_mergeable_node(Tree* t); /* go forward from t (include t) to get the next mergeable node. could return NULL. */ + public: + /* make sure ok_for_merge is consistent with the mergeUnit and the silding window. */ + virtual int enoughForMerge(); /* decide whether the sliding window contains enough stuff for merging; return value: 1-enough, 0-just enough, -n-lack of n tokens. */ + virtual bool moveForwardOneStep(Tree* & t); /* definition of "one step" is on a piece of paper. */ + virtual bool moveBackwardOneStep(Tree* & t); + virtual bool moveForward(); /* move the sliding window forward to the next position ok for merging. */ + virtual bool moveBackward(); /* move the sliding window backward to the previous position ok for merging. */ + virtual bool reset(); /* re-initialize front/tail, list_for_merge, and mergeUnit. */ + + friend class TreeAccessor; +}; + +#endif /* _VECTOR_MERGER_H_ */ diff --git a/src/vgen/treeTra/vector-output.C b/src/vgen/treeTra/vector-output.C new file mode 100755 index 0000000..4fb017f --- /dev/null +++ b/src/vgen/treeTra/vector-output.C @@ -0,0 +1,168 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "vector-output.h" +#include "tree-accessor.h" + +using namespace std; + +/****************************************** + * Implementation for TraVecOutput + * + *****************************************/ +TraVecOutput:: +TraVecOutput(TraGenConfiguration & cfg, FILE * out) + : vecGen_config(cfg), vecGen_outfile(out), vector_mergers(), + nNodeVectors(0), nMergedVectors(), + tokenSizeUsed1(30), tokenSizeUsed2(50), + tokenSizeBound(100), strideUsed(2) +{ +} + +bool TraVecOutput:: +skipNode(Tree* astNode, Tree* inh) +{ + return astNode->terminal_number < vecGen_config.mergeTokens + || vecGen_config.isSkippable(astNode)==true + || vecGen_config.isOutputtable(astNode)==false; +} + +bool TraVecOutput:: +skipSubTree(Tree* astNode, Tree* inh) +{ + return vecGen_config.isAtomic(astNode); +} + +Tree* TraVecOutput:: +evaluateInheritedAttribute(Tree* astNode, Tree* inh) +{ + TreeVector* tv = TreeAccessor::get_node_vector(astNode); + tv->output(vecGen_outfile); + nNodeVectors++; + + return astNode; +} + +long TraVecOutput:: +evaluateSynthesizedAttribute(Tree* node, Tree* in, + SynthesizedAttributesList& l) +{ + return 0; // no use here. +} + +bool TraVecOutput:: +addMerger(VectorMerger* vm) +{ + if ( vm!=NULL ) { + vector_mergers.push_back(vm); + return true; + } else + return false; +} + +long TraVecOutput:: +outputMergedVectors() +{ + long n = 0; + for (list::iterator vm_itr = vector_mergers.begin(); vm_itr!=vector_mergers.end(); ++vm_itr) { + long tmp = (*vm_itr)->outputAllMergedVectors(vecGen_outfile); + n += tmp; + nMergedVectors.push_back( tmp ); + } + + return n; +} + +long TraVecOutput:: +nAllOutputedVectors() +{ + long n = nNodeVectors; + for (vector::iterator i=nMergedVectors.begin(); i!=nMergedVectors.end(); ++i) + n += (*i); + + return n; +} + +/* traverse the tree several times using different configuration values: */ +long TraVecOutput:: +multipleTraverse(Tree* basenode, Tree* inheritedValue, + t_traverseOrder travOrder) +{ + int tokensize1=tokenSizeUsed1, tokensize2=tokenSizeUsed2, tokensize=tokensize1; + int oldtokensize = vecGen_config.mergeTokens; + long ret; + + // should only use to smallest token size to traverse once to avoid duplication: + vecGen_config.mergeTokens = tokensize; + ret = this->traverse(basenode, inheritedValue, travOrder); + vecGen_config.mergeTokens = oldtokensize; + + return ret; +} + +/* traverse the tree several times using different configuration values: */ +long TraVecOutput:: +multipleOutputMergedVectors() +{ + int tokensize1=tokenSizeUsed1, tokensize2=tokenSizeUsed2, tokensize=tokensize1; + int selfoldtokensize = vecGen_config.mergeTokens; // should have no effect on vectors from stride >0. + int selfoldstride = vecGen_config.moveStride; // should have no effect, coz internally use VectorMerger's configuration. + long n = 0; + + vecGen_config.moveStride = strideUsed; + while ( tokensize<=tokenSizeBound ) { + int oldtokensize; + int oldstride; + vecGen_config.mergeTokens = tokensize; + for (list::iterator vm_itr = vector_mergers.begin(); vm_itr!=vector_mergers.end(); ++vm_itr) { + long tmp; + oldtokensize = (*vm_itr)->vecGen_config.mergeTokens; + oldstride = (*vm_itr)->vecGen_config.moveStride; + (*vm_itr)->vecGen_config.mergeTokens = tokensize; + (*vm_itr)->vecGen_config.moveStride = strideUsed; + (*vm_itr)->reset(); + tmp = (*vm_itr)->outputAllMergedVectors(vecGen_outfile); + n += tmp; + nMergedVectors.push_back( tmp ); + (*vm_itr)->vecGen_config.mergeTokens = oldtokensize; + (*vm_itr)->vecGen_config.moveStride = oldstride; + } + tokensize1 = tokensize2; + tokensize2 = tokensize+tokensize2; + tokensize = tokensize1; + } + vecGen_config.moveStride = selfoldstride; + vecGen_config.mergeTokens = selfoldtokensize; + + return n; +} + diff --git a/src/vgen/treeTra/vector-output.h b/src/vgen/treeTra/vector-output.h new file mode 100755 index 0000000..47babe7 --- /dev/null +++ b/src/vgen/treeTra/vector-output.h @@ -0,0 +1,77 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _VECTOR_OUTPUT_H_ +#define _VECTOR_OUTPUT_H_ + +#include +#include +#include "../../include/ptree.h" +#include "tree-traversal.C" +#include "vgen-config.h" +#include "vector-merger.h" + +class TraVecOutput : public ParseTreeTraversal /* parent node and no use. */ +{ + public: + FILE * vecGen_outfile; + TraGenConfiguration vecGen_config; + + long nNodeVectors; + std::vector nMergedVectors; + std::list vector_mergers; /* helpers for merging vectors. */ + + int tokenSizeUsed1, tokenSizeUsed2, tokenSizeBound, strideUsed; + + public: + TraVecOutput(TraGenConfiguration & cfg, FILE * out); + + virtual bool skipNode(Tree* astNode, Tree* inh); + virtual bool skipSubTree(Tree* astNode, Tree* inh); + + virtual Tree* evaluateInheritedAttribute(Tree* astNode, Tree* inh); + virtual long evaluateSynthesizedAttribute(Tree* node, Tree* inh, + SynthesizedAttributesList& l); + /* "traverse" are used to output vectors for stride 0. */ + + bool addMerger(VectorMerger* vm); + long outputMergedVectors(); /* output vectors for stride >0 */ + long nAllOutputedVectors(); + + /* traverse the tree several times using different token sizes: */ + long multipleTraverse(Tree* basenode, Tree* inheritedValue, + t_traverseOrder travOrder=TT_PREORDER); + long multipleOutputMergedVectors(); /* output vectors for a list of strides >0 */ +}; + +#endif /* _VECTOR_OUTPUT_H_ */ + diff --git a/src/vgen/treeTra/vgen-config.C b/src/vgen/treeTra/vgen-config.C new file mode 100755 index 0000000..1e7d088 --- /dev/null +++ b/src/vgen/treeTra/vgen-config.C @@ -0,0 +1,192 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include "vgen-config.h" + +using namespace std; +extern string identifierTypeName; + +#define VGDEBUG + +/****************************************** + * Implementation for TraGenConfiguration + * + *****************************************/ +TraGenConfiguration:: +TraGenConfiguration(ParseTree* rt) + : parse_tree(rt), cfgFile(NULL), nodekinds(rt->typeCount()), + countedNodes(nodekinds, false), outputtedNodes(nodekinds, false), + atomicNodes(nodekinds, false), mergeableNodes(nodekinds, false) + // NOTE: the order of initialization is not guaranteed, but it doesn't matter here. +{ + // default parameters: + mergeTokens = 30; + moveStride = 1; + mergeLists = 3; + + init(); +} + +TraGenConfiguration:: +TraGenConfiguration(ParseTree* rt, int tokens, int strides, int lists) + : parse_tree(rt), cfgFile(NULL), nodekinds(rt->typeCount()), + countedNodes(nodekinds, false), outputtedNodes(nodekinds, false), + atomicNodes(nodekinds, false), mergeableNodes(nodekinds, false) +{ + mergeTokens = tokens; + moveStride = strides; + mergeLists = lists; + + init(); +} + +TraGenConfiguration:: +TraGenConfiguration(const char *fn) +{ + // TODO: use syntax similar to C/C++ to ease parsing. Ghassan's trick (but still requires recompilation) ;-) +} + +void TraGenConfiguration:: +init() +{ +#ifdef VGDEBUG + fprintf(stderr, "typeCount after init() = %d\n", parse_tree->typeCount()); +#endif + + // assert rt->relevantNodes[i] is in [0, nodekinds) + for ( int i=0; irelevantNodes.size(); i++ ) { + countedNodes[ parse_tree->relevantNodes[i] ] = true; +#if 0 + cerr << "RelevantNodes====" << parse_tree->relevantNodes[i] << ":" << countedNodes[ parse_tree->relevantNodes[i] ] << endl; +#endif + } + for ( int i=0; ileafNodes.size(); i++ ) + atomicNodes[ parse_tree->leafNodes[i] ] = true; + for ( int i=0; ivalidParents.size(); i++ ) + outputtedNodes[ parse_tree->validParents[i] ] = true; + for ( int i=0; imergeableNodes.size(); i++ ) + mergeableNodes[ parse_tree->mergeableNodes[i] ] = true; + + /***** this is grammar-specific *****/ + identid = parse_tree->getTypeID(identifierTypeName); +} + +bool TraGenConfiguration:: +isSkippable(Tree* astNode) +{ + assert( astNode->type>=0 && astNode->typetype]==true ) + return false; + else + return true; +} + +bool TraGenConfiguration:: +isOutputtable(Tree* astNode) +{ + assert( astNode->type>=0 && astNode->typetype]==true ) + return true; + else + return false; +} + +bool TraGenConfiguration:: +isAtomic(Tree* astNode) +{ + assert( astNode->type>=0 && astNode->typetype]==true ) + return true; + else + return false; +} + +bool TraGenConfiguration:: +isMergeable(Tree* node) +{ + assert( node->type>=0 && node->typetype]==true ) + return true; + else + return false; +} + +bool TraGenConfiguration:: +increaseVecCounters(Tree* n, TreeVector* tv) +{ + if ( n->type<0 || n->type>=tv->counters.size() ) + return false; + + // increase node counts: + tv->counters[n->type] += 1; + + // update token range: + map::iterator attr_itr = n->attributes.find(NODE_TOKEN_ID); + assert( attr_itr!=n->attributes.end() ); + if ( tv->token_begin_id<0 || ( ((pair*)(*attr_itr).second)->first>=0 && ((pair*)(*attr_itr).second)->firsttoken_begin_id ) ) + tv->token_begin_id = ((pair*)(*attr_itr).second)->first; + if ( tv->token_end_id<0 || ((pair*)(*attr_itr).second)->second>tv->token_end_id ) + tv->token_end_id = ((pair*)(*attr_itr).second)->second; + + if ( n->isTerminal()==true ) { + // update line counts: + Terminal* tn = n->toTerminal(); +// tv->lines.insert(tn->line); + if ( tv->minLine<=0 || (tn->line>0 && tn->lineminLine) ) + tv->minLine = tn->line; + if ( tv->maxLine<=0 || tn->line>tv->maxLine ) + tv->maxLine = tn->line; +#ifdef outputcountedlines + fprintf(stderr, "A line ---%d, id==%s\n", tn->line, n->type==identid? tn->value->c_str() : "NULL"); +#endif + } + + if ( n->type==identid ) { + // TODO: the condition may be not enough coz we may want to + // consider recording '+', '-' to help locating bugs. + // assert( n->isTerminal() ); + Terminal* tn = n->toTerminal(); + map::iterator id = tv->name_counters.find(*(tn->value)); + if ( id==tv->name_counters.end() ) + tv->name_counters[*(tn->value)] = 1; + else + (*id).second += 1; + tv->ordered_names.push_back(tn->value); + } + + return true; +} + diff --git a/src/vgen/treeTra/vgen-config.h b/src/vgen/treeTra/vgen-config.h new file mode 100755 index 0000000..7669301 --- /dev/null +++ b/src/vgen/treeTra/vgen-config.h @@ -0,0 +1,88 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#ifndef _VEC_GEN_CONFIG_H_ +#define _VEC_GEN_CONFIG_H_ + +#include +#include "../../include/ptree.h" +#include "tree-vector.h" + +/* Represent the configurable options from configuration files. */ +class TraGenConfiguration { + public: + ParseTree* parse_tree; /* for backup ;-) */ + + const char * cfgFile; /* Name of the configuration file. TODO: we don't have flexible syntax for configuration yet. */ + + int nodekinds; /* The number of kinds of nodes: 0<=type countedNodes; /* of length nodekinds, indicating the nodes should be counted. */ + std::vector outputtedNodes; /* of length nodekinds, indicating the nodes should be outputted. */ + std::vector atomicNodes; /* of length nodekinds, indicating the nodes should not be counted separately. */ + std::vector mergeableNodes; /* of length nodekinds, indicating the nodes should be considered for merging (specific to the VectorMergerOnLists???). */ + + /* parameters for vector merging. */ + int mergeTokens; /* The minimal number of tokens should be outputted. If <0, disable it; if ==0, use a list: 30 50 80 130 210 340 550 890 */ + int mergeLists; /* The maximal number of statements (or other kinds of lists) should be outputted. TODO: not used yet. */ + int moveStride; /* The minimal distance the sliding window is moved each time. If <=0, disable it (no meaning to run a list if we can use stride 1). If mergeTokens==0, use stride 2. */ + + public: + /* Helper functions for generating file names. TODO: not useful for now. */ + const char * getHeaderName( ); + const char * getCppName( ); + const char * getVecName( ); + + public: + TraGenConfiguration(const char * fn); /* Read in a configuration file. */ + TraGenConfiguration(ParseTree* rt); /* configuration based on a parse tree. */ + TraGenConfiguration(ParseTree* rt, int tokens, int strides, int lists); + private: + void init(); + + public: + bool isSkippable(Tree* node); + bool isOutputtable(Tree* node); + bool isAtomic(Tree* node); + bool isMergeable(Tree* node); + + bool increaseVecCounters(Tree* n, TreeVector* tv/*, ParseTree* pt=parse_tree not allowed in C++.*/); + + int identid; /* node type representing identifiers; it's grammar-specific */ + + friend class VectorMerger; + friend class VectorMergerOnTokens; + friend class VectorMergerOnLists; + friend class VecGenerator; + friend class TraVecOutput; +}; + +#endif /* _VEC_GEN_CONFIG_H_ */ diff --git a/src/vgen/treeTra/vgen-utils.c b/src/vgen/treeTra/vgen-utils.c new file mode 100755 index 0000000..10d1d3d --- /dev/null +++ b/src/vgen/treeTra/vgen-utils.c @@ -0,0 +1,59 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "vgen-utils.h" + +char *stringclone(const char *var) +{ + char * tmp = NULL; + assert( var!=NULL ); + tmp = (char *)malloc(sizeof(char)*(strlen(var)+1)); + strcpy(tmp, var); + return tmp; +} + +char *stringnclone(const char *var, unsigned int len) +{ + char *tmp = NULL; + assert( var!=NULL ); + tmp = (char *)malloc(sizeof(char)*(len+1)); + strncpy(tmp, var, len); + tmp[len] = '\0'; + return tmp; +} + +/* a wrapper of strcmp. */ +int compare_string(const void *c1, const void *c2) /* a wrapper of strcmp. */ +{ + return strcmp((const char *)c1, (const char *)c2); +} diff --git a/src/vgen/treeTra/vgen-utils.h b/src/vgen/treeTra/vgen-utils.h new file mode 100755 index 0000000..8f2428f --- /dev/null +++ b/src/vgen/treeTra/vgen-utils.h @@ -0,0 +1,57 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +/* This file contains extensions for C strings, etc. */ + +#ifndef _VGEN_UTILS_H_ +#define _VGEN_UTILS_H_ + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/****** String Utilities ******/ +char *stringclone(const char *var); +char *stringnclone(const char *var, unsigned int len); +int compare_string(const void *c1, const void *c2); /* a wrapper of strcmp. */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/vgen/vgrouping/Makefile b/src/vgen/vgrouping/Makefile new file mode 100755 index 0000000..b470f88 --- /dev/null +++ b/src/vgen/vgrouping/Makefile @@ -0,0 +1,45 @@ +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +CC=gcc +CFLAGS:=${CFLAGS} -O3 +EXEFILES=vectorsort dispatchvectors computeranges +SRCFILES=$(addsuffix .c, $(EXEFILES)) + +all: $(EXEFILES) + +%::%.c + ${CC} ${CFLAGS} -o $@ $^ -lm + +clean: + rm -f ${EXEFILES} *~ *.o + diff --git a/src/vgen/vgrouping/computeranges.c b/src/vgen/vgrouping/computeranges.c new file mode 100644 index 0000000..220e5a3 --- /dev/null +++ b/src/vgen/vgrouping/computeranges.c @@ -0,0 +1,145 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* This code is used to compute the grouping ranges for vectors. + * It does not need a vector file as inputs; its usage: + * ./thisfile [flag-output_range] + * + * So, it only generates a set of ranges. Pls use another code to dispatch + * the vectors so it's more flexible to how merge certain ranges together. + */ +#include +#include +#include +#include +#include +#include +#include + +typedef int bool; +#define FALSE 0 +#define TRUE 1 +const double epsilon = 1e-6; + +const int max_num_ranges=1024*1024*1024; // could the size of a vector be so huge? +int num_ranges=1024*2; // may be dynamically increased +void *enlarge_array(long* arr, int * cap) +{ + assert( *cap < *cap+1024 ); // help reduce integer overflow + *cap = *cap + 1024; + return realloc(arr, *cap*sizeof(long)); +} + +int main (int argc, char *argv[]) +{ + double d, c; + long lo, hi; + long *lows, *highs; + long ri = 0; + int i, j; + + if ( argc!=4 ) { + fprintf(stderr, "Usage: %s \n", argv[0]); + exit(127); + } + + d = atof(argv[1]); + lo = atol(argv[2]); + hi = atol(argv[3]); + assert( d>=0.0 ); + assert( lo>=1 ); // 0 is meaningless for vectors + assert( hi>=lo ); + + lows = (long *)malloc(sizeof(long)*num_ranges); + highs = (long *)malloc(sizeof(long)*num_ranges); + assert( lows!=NULL && highs!=NULL ); + + if ( fabs(d)2 ); + while ( ri= max_num_ranges-1 ) { + fprintf(stderr, "ERR: Reach %ld ranges. The last range is set to include all rest sizes.\n", max_num_ranges); + if ( c<0 ) + lows[ri] = (long)ceil(highs[ri-1]-c); + else + lows[ri] = (long)ceil(highs[ri-1]-c*lows[ri-1]/lo); + } else { + /* enlarge the arrays if they are too small: */ + if ( ri >= num_ranges ) { + lows = enlarge_array(lows, &num_ranges); + highs = realloc(highs, num_ranges*sizeof(long)); + assert( lows!=NULL && highs!=NULL ); + } + if ( ri == 1 ) { + /* begin the second range: */ + if ( c<0 ) { + /* do not to worry about false clone transitions or scalable "c": */ + lows[ri] = (long)ceil(lo+d-c); + highs[ri] = (long)ceil((lo+d)*lows[ri]/lo); + } else { + lows[ri] = (long)ceil(lo+d-c); + highs[ri] = (long)ceil(lo+2*d+d*d/lo-c*c/lo+c+1); + } + } else { + if ( c<0 ) { + lows[ri] = (long)ceil(highs[ri-1]-c); + highs[ri] = (long)ceil((lo+d)*lows[ri]/lo); + } else { + lows[ri] = (long)ceil(highs[ri-1]-c*lows[ri-1]/lo); + highs[ri] = (long)ceil((lo+d+c)*highs[ri-1]/lo-(d*c+c*c)*lows[ri-1]/lo/lo+1); + } + } + ri++; + } + } + highs[ri-1] = -1; // "-1" means all rest sizes + + /* output the ranges: + * first line (the parameters): + * other lines: */ + printf("%.8g\t%ld\t%ld\t%ld\t%ld\t%ld\n", d, lo, hi, ri, highs[0], lows[ri-1]); + printf("%6ld\t\t%ld\t%ld\t%g\n", 1, lows[0], highs[0], d); + for ( i=1; i + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include +#include +#include +#include +/* for stat(): */ +#include +#include +#include + +/* TODO: modularization: + * - separate the core of dispatch from I/O + * -- output file name generation; length limitation + * - separate "scaling" from dispatch + * - refactor common utilities (e.g., dir_exists) + * - refactor command line processing + * - scripts, instead of C, may be more flexible for changes and maintanence + * -- BSD license makes IP issues irrelevant + */ + +typedef int bool; +#define FALSE 0 +#define TRUE 1 + +typedef struct avector_tag { + int kind; /* experiments show that it's no use (only three kinds: 0 (unkonwn), 65 (stmt), 131 (extdef) */ + long size; + char * note; + char * vector; + bool scaled; +} avector; + +void initavector(avector * av) +{ + assert ( av!=NULL ); + av->kind = 0; + av->size = 0; + av->note = NULL; + av->vector = NULL; + av->scaled = FALSE; +} + +void clearavector(avector * av) +{ + if ( av==NULL ) + return; + av->kind = 0; + av->size = 0; + if ( av->note!=NULL ) { + free(av->note); + av->note = NULL; + } + if ( av->vector!=NULL ) { + free(av->vector); + av->vector = NULL; + } + av->scaled = FALSE; +} + +int compare_avector(const void * a, const void * b) +{ + // if ( ((avector*)a)->kind == ((avector*)b)->kind ) + return ((avector*)a)->size - ((avector*)b)->size; + // else + // return ((avector*)a)->kind - ((avector*)b)->kind; +} + +int pointsDimension = 0; +double * scaledfloatvec = NULL; +int dimensionofavector(const char * line) + /* assume input is a string of digits */ +{ + const char *p = line; int dim = 0; + + if ( p==NULL ) + return 0; + + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + while (*p != '\0') { + while (*p != ' ' && *p!='\t' && *p!='\r' && *p!='\n' && *p != '\0') p++; + dim ++; + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + } + + return dim; +} + +bool scaleavector(avector* av, long srange, long xrange) +{ + char * scaledvec = NULL; + int dim = 0; + char *t = av->vector; + + av->scaled = TRUE; + + if ( t==NULL ) + return FALSE; + + if ( pointsDimension==0 ) + pointsDimension = dimensionofavector(av->vector); + if ( pointsDimension==0 ) + return FALSE; + + if ( scaledfloatvec==NULL ) { + scaledfloatvec = (double*)malloc(sizeof(double)*pointsDimension); + assert( scaledfloatvec!=NULL ); + } + + for (dim=0, t=av->vector; *t != '\0' && dim < pointsDimension; dim++) { + while ( !isdigit(*t) && *t != '\0' && *t != '.') t++; /* skip over the leading blanks; */ + // scaledfloatvec[dim] = strtof(t, &t); /* why this doesn't work? Why worked in LSH? Because g++ works fine; it's a bug in gcc/glibc... */ + sscanf(t, "%lg", &scaledfloatvec[dim]); /* get the number; */ + while ( isdigit(*t) || *t=='.') t++; /* skip over the number; */ + scaledfloatvec[dim] = scaledfloatvec[dim]*srange/xrange; + } + +#define PRECISION_LEN 13 + scaledvec = (char*)malloc((1+pointsDimension)*PRECISION_LEN); /* may be too many... */ + if ( scaledvec==NULL ) { + fprintf(stderr, "Not enough memory to scale all vectors. Try to free some vectors of previous groups...\n"); + return FALSE; + } + + t = scaledvec; + for (dim=0; dim0 ) { + t += inc; + } else { + t[0] = '\t'; t++; + } + assert( t-scaledvec<=pointsDimension*PRECISION_LEN ); /* a little conservative. */ + } + t[0] = '\n'; t[1] = '\0'; + + free(av->vector); /* why this seems no use for saving memory? TOFIX. */ + av->vector=scaledvec; + return TRUE; +} + +void outputavector(FILE* out, const avector* av, long info) +{ + // fprintf(out, "%ld\t%ld\n%s%s", info, av->size, av->note, av->vector); + fprintf(out, "%s%s", av->note, av->vector); +} + +bool dir_exists(const char * path) +{ + struct stat mystat; + int res; + + res = stat(path, &mystat); + if (res != 0) { + fprintf(stderr, "Can't read %s.\n", path); + return FALSE; + } else if (S_ISDIR(mystat.st_mode)) { // or use mystat.st_mode & 0x4000 + return TRUE; + } else { + fprintf(stderr, "%s exists, but it's not a directory\n", path); + return FALSE; + } +} + +bool read_size_ranges(FILE* rf, long **lows, long ** highs, int* num_ranges, + double * d, long * lo, long * hi) +{ + long fhi = 0, llo = 0; + char * line=NULL; int llen1 = 0; + int rid = 0, rcounter = 0; + int negative_range = 0; + + assert( rf!=NULL ); + + // first line: + assert( getline(&line, &llen1, rf)>0 ); + assert( sscanf(line, "%lg\t%ld\t%ld\t%d\t%ld\t%ld", d, lo, hi, num_ranges, &fhi, &llo)==6 ); + assert( *d>=0. ); + assert( *lo>=1 ); + assert( *hi>=*lo ); + assert( *num_ranges>=2 ); + assert( fhi>=*lo ); + assert( llo+*d>=fhi ); + //fprintf(stdout, "%lg\t%ld\t%ld\t%d\t%ld\t%ld\n", *d, *lo, *hi, *num_ranges, fhi, llo); + + /* init lows and highs: */ + if ( *lows==NULL || *highs==NULL ) { + *lows = (long *)malloc(sizeof(long)*(*num_ranges)); + *highs = (long *)malloc(sizeof(long)*(*num_ranges)); + } else { + *lows = (long *)realloc(*lows, sizeof(long)*(*num_ranges)); + *highs = (long *)realloc(*highs, sizeof(long)*(*num_ranges)); + } + assert( *lows!=NULL && *highs!=NULL ); + + while ( getline(&line, &llen1, rf)>0 && rcounter<*num_ranges ) { + //fprintf(stdout, "==>%s", line); + assert( sscanf(line, "%d\t%ld\t%ld", &rid, &((*lows)[rcounter]), &((*highs)[rcounter]))==3 ); + assert( (*lows)[rcounter]>=0 ); + assert( (*highs)[rcounter]>=(*lows)[rcounter] || (*highs)[rcounter]<0 ); + if ( (*highs)[rcounter]<0 ) + negative_range++; + //fprintf(stdout, "%d\t%ld\t%ld\n", rid, (*lows)[rcounter], (*highs)[rcounter]); + assert( ++rcounter==rid ); + } + if ( rcounter!=*num_ranges ) { + fprintf(stderr, "ERROR in the range file: 'num_ranges' does not match line counters: %d vs. %d\n", num_ranges, rcounter); + return FALSE; + } else if ( negative_range<1 ) { + fprintf(stderr, "ERROR: no -1 in the ranges?\n"); + return FALSE; + } else if ( negative_range>1 ) { + fprintf(stderr, "ERROR: more than one -1 in the ranges?\n"); + return FALSE; + } else + return TRUE; +} + +int dispatch(const avector * vec, const long const * lows, const long const * highs, const int num_ranges, + const double d, const long lo, const long hi, const char * destdir, const char * fprefix) +{ + // group file names: destdir/fprefix_g_d_lo_hi +#define BASENAME_LEN 40 + char vecgroupname[FILENAME_MAX]; + char * psuffix, *pslash; + int rid, rcounter = 0; + bool last_range = FALSE; + + assert( vec!=NULL ); + assert( destdir!=NULL ); + + if ( ! dir_exists(destdir) ) { + fprintf(stderr, "ERROR: directory does not exist: %s\n", destdir); + return 0; + } + if ( vec->size<=0 ) + return 0; + + /* prepare for the group names: */ + strncpy(vecgroupname, destdir, FILENAME_MAX); + vecgroupname[FILENAME_MAX-1] = 0; + psuffix = vecgroupname + strlen(vecgroupname); + assert( psuffix>vecgroupname ); + assert( psuffix-vecgroupname < FILENAME_MAX-BASENAME_LEN ); // save space for actual names. + pslash = strrchr(vecgroupname, '/'); + if ( pslash==NULL || pslash+1!=psuffix ) + *psuffix++ = '/'; + *psuffix = 0; + + for ( rid=0; ridsize < lows[rid] ) + break; + else if ( vec->size <= highs[rid] || highs[rid]<0 ) { + // save the vector into this range: + FILE * vgfile = NULL; + snprintf(psuffix, BASENAME_LEN, "%s_g%d_%.8g_%ld_%ld", fprefix, rid+1, d, lo, hi); + vgfile = fopen(vecgroupname, "a"); // NOTE: old group files are kept. + if ( vgfile == NULL ) { + fprintf(stderr, "Can't append to vector group file '%s'\n", vecgroupname); + continue; + } + + outputavector(vgfile, vec, 0); + + fclose(vgfile); + rcounter++; + if ( highs[rid]<0 ) { + if ( last_range ) + fprintf(stderr, "WARN: more than one -1 in the ranges; current rid = %d\n", rid+1); + last_range = TRUE; + } + } else + continue; + } + + return rcounter; +} + +int main (int argc, char *argv[]) +{ + /* for reading the ranges: */ + FILE * rangefile = NULL; + long *lows=NULL, *highs=NULL; + int num_ranges = 0; + double d = 0.; + long lo = 0, hi = 0; + /* for reading vectors: */ + FILE * vecfile = NULL; + const char * outdir = "./vectorgroups/"; + const char * fileprefix = "vdb"; + long vid = 0, dispatched = 0; + size_t llen1 = 0, llen2 = 0; + avector onevector; + regex_t num_node, node_kind; + regmatch_t matches[1]; + + if ( argc!=3 && argc!=4 && argc!=5 ) { + fprintf(stderr, "Usage: %s [ [fileprefix]]\n", argv[0]); + exit(127); + } + + rangefile = fopen(argv[1], "r"); + if ( rangefile==NULL ) { + fprintf(stderr, "Can't open the range file: %s\n", argv[1]); + exit(127); + } + vecfile = fopen(argv[2], "r"); + if ( vecfile==NULL ) { + fprintf(stderr, "Can't open the vec file: %s\n", argv[2]); + exit(127); + } + if ( argc>=4 ) { + outdir = argv[3]; + } + if ( ! dir_exists(outdir) ) { + exit(127); + } + if ( argc>=5 ) { + fileprefix = argv[4]; + } + + assert(0 == regcomp(&num_node, "NUM_NODE:([0-9]+)", REG_EXTENDED) ); + assert(0 == regcomp(&node_kind, "NODE_KIND:([0-9]+)", REG_EXTENDED) ); + + read_size_ranges(rangefile, &lows, &highs, &num_ranges, &d, &lo, &hi); + fclose(rangefile); + rangefile = NULL; + + initavector(&onevector); + while ( getline(&(onevector.note), &llen1, vecfile)>0 ) { + char temp; + int start, end; + + if ( getline(&(onevector.vector), &llen2, vecfile)<=0 ) { + fprintf(stderr, "Reading stopped at line %ld in vector file %s. Should be checked there!\n", 2*(vid+1), argv[2]); + } + + /* get node kind */ + if (0 != regexec(&node_kind, onevector.note, 1, matches, 0) ) { + fprintf(stderr, "Line %ld has no node_kind in %s?\n", 2*(vid+1)-1, argv[2]); + } else { + start = matches[0].rm_so; + end = matches[0].rm_eo; + assert( start!=-1 ); + temp = onevector.note[end]; + onevector.note[end] = '\0'; + onevector.kind = atoi(onevector.note+start+10); + onevector.note[end] = temp; + } + + /* get num_node */ + if (0 != regexec(&num_node, onevector.note, 1, matches, 0) ) { + fprintf(stderr, "Why line %ld has no NUM_NODE in %s?\n", 2*(vid+1)-1, argv[2]); + continue; + } + start = matches[0].rm_so; + end = matches[0].rm_eo; + assert( start!=-1 ); + temp = onevector.note[end]; + onevector.note[end] = '\0'; + onevector.size = atoi(onevector.note+start+9); + if ( onevector.size<=0 ) { + fprintf(stderr, "Warning: why line %ld has NUM_NODE<=0? Skip the vector in %s.\n", 2*(vid+1)-1, argv[2]); + continue; + } + assert( onevector.size>0 ); + onevector.note[end] = temp; + + /* one vector is read in, now dispatch it into group(s): */ + vid++; + dispatched += dispatch(&onevector, lows, highs, num_ranges, d, lo, hi, outdir, fileprefix); + } + + /* finalize: */ + clearavector(&onevector); + fclose(vecfile); + + fprintf(stdout, "Total %ld vectors read in; %ld vectors dispatched into %d ranges (actual groups may be many fewer).\n", vid, dispatched, num_ranges); + return 0; +} + diff --git a/src/vgen/vgrouping/rundispatch b/src/vgen/vgrouping/rundispatch new file mode 100644 index 0000000..137a5ec --- /dev/null +++ b/src/vgen/vgrouping/rundispatch @@ -0,0 +1,97 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: thisfile [s d c] + +if [[ $# -ne 2 && $# -ne 4 && $# -ne 6 && $# -ne 8 && $# -ne 10 ]]; then + echo "Usage: $0 [-l ] [-h ] [-o ] [-p ]" + echo "sample: $0 linux-2.6.19 1 -l 50 -h 1000000 -o vectorgroups/ -p linux2619" + exit 127; +fi + +vecsrcdir=$1 +shift +vecdist=$1 +shift +# default values +low=50 +high=1000000 +outdir=./vectorgroups +nameprefix=vdb + +while [ $# -gt 0 ]; do + case $1 in + -l) + low=$2;; + -h) + high=$2;; + -o) + outdir=$2;; + -p) + nameprefix=$2;; + *) + echo "ERROR: wrong parameters" + exit 127;; + esac + shift 2 +done + +#echo $vecsrcdir +#echo $vecdist +#echo $low +#echo $high +#echo $outdir +#echo $nameprefix + +if [[ ! -d $vecsrcdir ]]; then + echo "Directory $vecsrcdir doesn't exist." + exit 127 +fi + +if [[ ! -d $outdir ]]; then + mkdir $outdir || (echo "Can't make directory $outdir"; exit 127) +fi + +# create the range file (once) +`dirname $0`/computeranges $vecdist $low $high > "${outdir}/ranges_${vecdist}_${low}_${high}" +if [[ ! -s ${outdir}/ranges_${vecdist}_${low}_${high} ]]; then + echo "Can't generate the range file: '${outdir}/ranges_${vecdist}_${low}_${high}'" + exit 1 +fi +# dispatch the vectors: +find ${vecsrcdir} -iname "*.vec" | while read fn; do + `dirname $0`/dispatchvectors "${outdir}/ranges_${vecdist}_${low}_${high}" "$fn" "${outdir}" "${nameprefix}" || (echo "Failed to dispatch '$fn'."; exit 1) +done + diff --git a/src/vgen/vgrouping/rundispatchonefile b/src/vgen/vgrouping/rundispatchonefile new file mode 100644 index 0000000..896b9c3 --- /dev/null +++ b/src/vgen/vgrouping/rundispatchonefile @@ -0,0 +1,97 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: thisfile [s d c] + +if [[ $# -ne 2 && $# -ne 4 && $# -ne 6 && $# -ne 8 && $# -ne 10 ]]; then + echo "Usage: $0 [-l ] [-h ] [-o ] [-p ]" + echo "sample: $0 linux-2.6.19 1 -l 50 -h 1000000 -o vectorgroups/ -p linux2619" + exit 127; +fi + +vecfile=$1 +shift +vecdist=$1 +shift +# default values +low=50 +high=1000000 +outdir=./vectorgroups +nameprefix=vdb + +while [ $# -gt 0 ]; do + case $1 in + -l) + low=$2;; + -h) + high=$2;; + -o) + outdir=$2;; + -p) + nameprefix=$2;; + *) + echo "ERROR: wrong parameters" + exit 127;; + esac + shift 2 +done + +#echo $vecfile +#echo $vecdist +#echo $low +#echo $high +#echo $outdir +#echo $nameprefix + +if [[ ! -s $vecfile ]]; then + echo "Vector file $vecfile doesn't exist." + exit 127 +fi + +if [[ ! -d $outdir ]]; then + mkdir $outdir || (echo "Can't make directory $outdir"; exit 127) +fi + +# create the range file (once) +if [[ ! -s ${outdir}/ranges_${vecdist}_${low}_${high} ]]; then + `dirname $0`/computeranges $vecdist $low $high > "${outdir}/ranges_${vecdist}_${low}_${high}" +fi +if [[ ! -s ${outdir}/ranges_${vecdist}_${low}_${high} ]]; then + echo "Can't generate the range file: '${outdir}/ranges_${vecdist}_${low}_${high}'" + exit 1 +fi +# dispatch the vectors: +`dirname $0`/dispatchvectors "${outdir}/ranges_${vecdist}_${low}_${high}" "${vecfile}" "${outdir}" "${nameprefix}" || (echo "Failed to dispatch '${vecfile}'."; exit 1) + diff --git a/src/vgen/vgrouping/runsplit b/src/vgen/vgrouping/runsplit new file mode 100755 index 0000000..a3193c4 --- /dev/null +++ b/src/vgen/vgrouping/runsplit @@ -0,0 +1,74 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: thisfile [s d c] + +if [ $# -ne 1 -a $# -ne 2 ] +then + if [ $# -ne 4 ] + then + if [ $# -ne 5 ] + then + echo "Usage: $0 [ flag | {s d c [<#g>]} ]" + exit 65; + fi + fi +fi + +lineno=`wc -l $1 | awk '{print $1;}'` +vecno=1000000 +if [[ -n $lineno ]] +then + if [ $(($lineno%2)) -ne 0 ] + then + vecno=`expr $lineno / 2 + 1` + else + vecno=`expr $lineno / 2` + fi +fi + +if [ $# -eq 1 ] +then + `dirname $0`/split $1 $vecno +elif [ $# -eq 2 ] +then + `dirname $0`/split $1 $vecno $2 +elif [ $# -eq 4 ] +then + `dirname $0`/split $1 $vecno $2 $3 $4 +else + `dirname $0`/split $1 $vecno $2 $3 $4 $5 +fi + diff --git a/src/vgen/vgrouping/runvectorsort b/src/vgen/vgrouping/runvectorsort new file mode 100755 index 0000000..389bcfe --- /dev/null +++ b/src/vgen/vgrouping/runvectorsort @@ -0,0 +1,74 @@ +#!/bin/bash + +# +# +# Copyright (c) 2007-2010, +# Lingxiao Jiang +# Ghassan Misherghi +# Zhendong Su +# Stephane Glondu +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# Usage: thisfile [s d c] + +if [ $# -ne 1 -a $# -ne 2 ] +then + if [ $# -ne 4 ] + then + if [ $# -ne 5 ] + then + echo "Usage: $0 [ flag | {s d c [flag]} ]" + exit 65; + fi + fi +fi + +lineno=`wc -l $1 | awk '{print $1;}'` +vecno=1000000 +if [[ -n $lineno ]] +then + if [ $(($lineno%2)) -ne 0 ] + then + vecno=`expr $lineno / 2 + 1` + else + vecno=`expr $lineno / 2` + fi +fi + +if [ $# -eq 1 ] +then + `dirname $0`/vectorsort $1 $vecno +elif [ $# -eq 2 ] +then + `dirname $0`/vectorsort $1 $vecno $2 +elif [ $# -eq 4 ] +then + `dirname $0`/vectorsort $1 $vecno $2 $3 $4 +else + `dirname $0`/vectorsort $1 $vecno $2 $3 $4 $5 +fi + diff --git a/src/vgen/vgrouping/split.c b/src/vgen/vgrouping/split.c new file mode 100755 index 0000000..868b70b --- /dev/null +++ b/src/vgen/vgrouping/split.c @@ -0,0 +1,490 @@ +/* + * + * Copyright (c) 2007-2010, + * Lingxiao Jiang + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include +#include +#include +#include + +typedef int bool; +#define FALSE 0 +#define TRUE 1 + +typedef struct avector_tag { + int kind; /* experiments show that it's no use (only three kinds: 0 (unkonwn), 65 (stmt), 131 (extdef) */ + long size; + char * note; + char * vector; + bool scaled; +} avector; + +void initavector(avector * av) +{ + assert ( av!=NULL ); + av->kind = 0; + av->size = 0; + av->note = NULL; + av->vector = NULL; + av->scaled = FALSE; +} + +void clearavector(avector * av) +{ + if ( av==NULL ) + return; + av->kind = 0; + av->size = 0; + if ( av->note!=NULL ) { + free(av->note); + av->note = NULL; + } + if ( av->vector!=NULL ) { + free(av->vector); + av->vector = NULL; + } + av->scaled = FALSE; +} + +int compare_avector(const void * a, const void * b) +{ + // if ( ((avector*)a)->kind == ((avector*)b)->kind ) + return ((avector*)a)->size - ((avector*)b)->size; + // else + // return ((avector*)a)->kind - ((avector*)b)->kind; +} + +int pointsDimension = 0; +double * scaledfloatvec = NULL; +int dimensionofavector(const char * line) + /* assume input is a string of digits */ +{ + const char *p = line; int dim = 0; + + if ( p==NULL ) + return 0; + + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + while (*p != '\0') { + while (*p != ' ' && *p!='\t' && *p!='\r' && *p!='\n' && *p != '\0') p++; + dim ++; + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + } + + return dim; +} + +bool scaleavector(avector* av, long srange, long xrange) +{ + char * scaledvec = NULL; + int dim = 0; + char *t = av->vector; + + av->scaled = TRUE; + + if ( t==NULL ) + return FALSE; + + if ( pointsDimension==0 ) + pointsDimension = dimensionofavector(av->vector); + if ( pointsDimension==0 ) + return FALSE; + + if ( scaledfloatvec==NULL ) { + scaledfloatvec = (double*)malloc(sizeof(double)*pointsDimension); + assert( scaledfloatvec!=NULL ); + } + + for (dim=0, t=av->vector; *t != '\0' && dim < pointsDimension; dim++) { + while ( !isdigit(*t) && *t != '\0' && *t != '.') t++; /* skip over the leading blanks; */ + // scaledfloatvec[dim] = strtof(t, &t); /* why this doesn't work? Why worked in LSH? Because g++ works fine; it's a bug in gcc/glibc... */ + sscanf(t, "%lg", &scaledfloatvec[dim]); /* get the number; */ + while ( isdigit(*t) || *t=='.') t++; /* skip over the number; */ + scaledfloatvec[dim] = scaledfloatvec[dim]*srange/xrange; + } + +#define PRECISION_LEN 13 + scaledvec = (char*)malloc((1+pointsDimension)*PRECISION_LEN); /* may be too many... */ + if ( scaledvec==NULL ) { + fprintf(stderr, "Not enough memory to scale all vectors. Try to free some vectors of previous groups...\n"); + return FALSE; + } + + t = scaledvec; + for (dim=0; dim0 ) { + t += inc; + } else { + t[0] = '\t'; t++; + } + assert( t-scaledvec<=pointsDimension*PRECISION_LEN ); /* a little conservative. */ + } + t[0] = '\n'; t[1] = '\0'; + + free(av->vector); /* why this seems no use for saving memory? TOFIX. */ + av->vector=scaledvec; + return TRUE; +} + +void outputavector(FILE* out, avector* av, long info) +{ + // fprintf(out, "%ld\t%ld\n%s%s", info, av->size, av->note, av->vector); + fprintf(out, "%s%s", av->note, av->vector); +} + +long bsearchinsertpos(register void *key, void *base0, size_t nmemb, register size_t size, + register int (*compar)(const void *, const void *), long * foradd) +{ + register char *base = base0; + register long lim, cmp; + register void *p; + + for (lim = nmemb; lim != 0; lim >>= 1) { + p = base + (lim >> 1) * size; + cmp = (*compar)(key, p); + if (cmp == 0) { + *foradd = -1; + return ( (int)p-(int)base0 ) / size; + } + if (cmp > 0) { /* key > p: move right */ + base = (char *)p + size; + lim--; + } /* else move left */ + } + *foradd = ( (int)base-(int)base0 ) / size; + return -1; +} + +int comparelong(const void * a, const void * b) +{ + return *((long*)a) - *((long*)b); +} +/* only valid for overlapping ranges. */ +long smallestrangecontainsid(long lows[], long uppers[], long len, long vecid, avector * vectors, long veclen) +{ + long id, pos; + assert( vecid>=0 && len>0 ); + + if ( vecid>=veclen ) + return len-1; + + pos = bsearchinsertpos(&(vectors[vecid].size), uppers, len, sizeof(long), comparelong, &id); + if ( pos>=0 ) + return pos; + else + return id; +} +long smallestvecidofsize(avector * vectors, long veclen, long size) +{ + avector temp; + long id, pos; + + temp.size = size; + pos = bsearchinsertpos(&temp, vectors, veclen, sizeof(avector), compare_avector, &id); + // fprintf(stderr, "look for size %ld at %ld\n", size, pos); + assert( pos>=-1 && pos=0; id--) { + if ( vectors[id].size=-1 && possize ) + break; + else + pos++; + } + + return pos; +} + + +long vectorno = 1000000; +int main (int argc, char *argv[]) +{ + avector *vectors = NULL; + FILE * vecfile = NULL; + long vid = 0; + size_t len = 0; + regex_t num_node, node_kind; + regmatch_t matches[1]; + + if ( argc!=3 && argc!=4 && argc!=6 && argc!=7 ) { + fprintf(stderr, "Usage: vectorsort <#vector> [ flag_dist | {srange dist orange [<#groups]} ]\n"); + exit(1); + } + + vectorno = atol(argv[2]); + assert(vectorno>0); + + vecfile = fopen(argv[1], "r"); + if ( vecfile==NULL ) { + fprintf(stderr, "Can't open the vec file: %s\n", argv[1]); + exit(1); + } + + vectors = (avector*)malloc(sizeof(avector)*(vectorno+1)); + if ( vectors==NULL ) { + fprintf(stderr, "Memory malloc error for %ld vectors.\n", vectorno); + exit(1); + } + + assert(0 == regcomp(&num_node, "NUM_NODE:([0-9]+)", REG_EXTENDED) ); + assert(0 == regcomp(&node_kind, "NODE_KIND:([0-9]+)", REG_EXTENDED) ); + + initavector(&vectors[vid]); + while ( getline(&(vectors[vid].note), &len, vecfile)>=0 ) { + char temp; + int start, end; + + if ( getline(&(vectors[vid].vector), &len, vecfile)<=0 ) { + fprintf(stderr, "Reading stopped at line %ld. Should be checked there!\n", 2*(vid+1)); + } + + /* get node kind */ + if (0 != regexec(&node_kind, vectors[vid].note, 1, matches, 0) ) { + fprintf(stderr, "Line %ld has no node_kind?\n", 2*(vid+1)-1); + } else { + start = matches[0].rm_so; + end = matches[0].rm_eo; + assert( start!=-1 ); + temp = vectors[vid].note[end]; + vectors[vid].note[end] = '\0'; + vectors[vid].kind = atoi(vectors[vid].note+start+10); + vectors[vid].note[end] = temp; + } + + /* get num_node */ + if (0 != regexec(&num_node, vectors[vid].note, 1, matches, 0) ) { + fprintf(stderr, "Why line %ld has no NUM_NODE?\n", 2*(vid+1)-1); + continue; + } + start = matches[0].rm_so; + end = matches[0].rm_eo; + assert( start!=-1 ); + temp = vectors[vid].note[end]; + vectors[vid].note[end] = '\0'; + vectors[vid].size = atoi(vectors[vid].note+start+9); + // fprintf(stderr, "No. %ld Vec of size %ld from %s\n", vid+1, vectors[vid].size, vectors[vid].note+start+9); + if ( vectors[vid].size<=0 ) { + fprintf(stderr, "Warning: why line %ld has NUM_NODE<=0? Skip the vector.\n", 2*(vid+1)-1); + clearavector(&vectors[vid]); + continue; + } + assert( vectors[vid].size>0 ); + vectors[vid].note[end] = temp; + + vid++; + if ( vid>vectorno ) { + fprintf(stderr, "ERR: maximal #vec (%ld) is reached. Stop reading here.\n", vectorno); + break; + } + initavector(&vectors[vid]); + } + + fclose(vecfile); + pointsDimension = dimensionofavector(vectors[0].vector); /* get the dimension of the vector for later use. */ + + qsort(vectors, vid, sizeof(avector), compare_avector); + + if ( argc==3 ) { + // output sorted vectors: + long groups = 0, i; + for(i = 0; i startsize ) { + // start a new size/#occurrence: + if ( nsize > 0 ) + fprintf(stdout, "%ld\t%ld\n", startsize, nsize); + startsize = vectors[i].size; + nsize = 1; + } else + nsize++; + } + if ( nsize > 0 ) // last size + fprintf(stdout, "%ld\t%ld\n", startsize, nsize); + } else { +#define NUM_GROUP_LIMIT 1024*2 // only need hundreds if d>0 and c>=0. And 1024*1024 may be too big for some machines, such as my C640 and msg. + long s; double d, c; + long al[NUM_GROUP_LIMIT], au[NUM_GROUP_LIMIT]; + long groups = 0, i; + assert ( argc==6 || argc==7 ); + s = atol(argv[3]); + d = atof(argv[4]); + c = atof(argv[5]); + assert(s>0); /* the upper bound of the first range */ + assert(d>=0 && d=s is not very meaningful */ + if ( !(d>0 && c=0) && !(d==0 && c==-1) ) { /* the overlapped range. c<-1 may skip some vectors. */ + fprintf(stderr, "Warning: c=%g is not good for this case, ", c); + if ( d==0 ) + fprintf(stderr, "better to use c==-1.\n"); + else + fprintf(stderr, "better to use c in [0,%g)\n", d); + } + + /* compute all the group ranges */ + al[0] = 0, au[0] = (long)ceil(s+d); + groups = 1; + while ( al[groups-1]<=vectors[vid-1].size && au[groups-1]<=vectors[vid-1].size ) { + /* compute the range for a new group */ + if ( groups >= NUM_GROUP_LIMIT ) { + fprintf(stderr, "ERR: Can't handle more than %ld groups. The last group is set to include all rest vectors.\n", groups); + au[groups-1] = vectors[vid-1].size+1; + break; + } + + if ( groups == 1 ) { + /* begin the second group: */ + if ( c<0 ) { + /* do not to worry about false clone transitions or scalable "c": */ + al[groups] = (long)ceil(s+d-c); + au[groups] = (long)ceil((s+d)*al[groups]/s); + } else { + al[groups] = (long)ceil(s+d-c); + au[groups] = (long)ceil(s+2*d+d*d/s-c*c/s+c+1); + } + } else { + if ( c<0 ) { + al[groups] = (long)ceil(au[groups-1]-c); + au[groups] = (long)ceil((s+d)*al[groups]/s); + } else { + al[groups] = (long)ceil(au[groups-1]-c*al[groups-1]/s); + au[groups] = (long)ceil((s+d+c)*au[groups-1]/s-(d*c+c*c)*al[groups-1]/s/s+1); + } + } + groups++; + } + + /* output the group ranges: */ + if ( argc==6 ) { + for ( i=0; i=1 );//&& MAX_NUM_GROUP<=NUM_GROUP_LIMIT ); + avg_group_size = floor(((double)vid) / MAX_NUM_GROUP); + + sprintf(groupfilename, "%s_ranges_%s_%s_%s", argv[1], argv[4], argv[6], argv[3]); + rangefile = fopen(groupfilename, "w"); + if ( rangefile == NULL ) { + fprintf(stderr, "Can't open range file `%s' for writing. Use stdout instead.\n", groupfilename); + rangefile = stdout; + } + + while ( endvecid < vid-1 && startrangeid < groups ) { + /* search for the next endrangeid: */ + long vecidforlocating = endvecid+1+avg_group_size; + endrangeid = smallestrangecontainsid (al, au, groups, vecidforlocating, vectors, vid); +/* if ( endrangeid > 0 ) */ +/* endrangeid --; */ + assert( startrangeid>=0 && startrangeid<=endrangeid && endrangeid + * Ghassan Misherghi + * Zhendong Su + * Stephane Glondu + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +#include +#include +#include +#include +#include + +typedef int bool; +#define FALSE 0 +#define TRUE 1 + +typedef struct avector_tag { + int kind; /* experiments show that it's no use (only three kinds: 0 (unkonwn), 65 (stmt), 131 (extdef) */ + long size; + char * note; + char * vector; + bool scaled; +} avector; + +void initavector(avector * av) +{ + assert ( av!=NULL ); + av->kind = 0; + av->size = 0; + av->note = NULL; + av->vector = NULL; + av->scaled = FALSE; +} + +void clearavector(avector * av) +{ + if ( av==NULL ) + return; + av->kind = 0; + av->size = 0; + if ( av->note!=NULL ) { + free(av->note); + av->note = NULL; + } + if ( av->vector!=NULL ) { + free(av->vector); + av->vector = NULL; + } + av->scaled = FALSE; +} + +int compare_avector(const void * a, const void * b) +{ + // if ( ((avector*)a)->kind == ((avector*)b)->kind ) + return ((avector*)a)->size - ((avector*)b)->size; + // else + // return ((avector*)a)->kind - ((avector*)b)->kind; +} + +int pointsDimension = 0; +double * scaledfloatvec = NULL; +int dimensionofavector(const char * line) + /* assume input is a string of digits */ +{ + const char *p = line; int dim = 0; + + if ( p==NULL ) + return 0; + + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + while (*p != '\0') { + while (*p != ' ' && *p!='\t' && *p!='\r' && *p!='\n' && *p != '\0') p++; + dim ++; + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + } + + return dim; +} + +bool scaleavector(avector* av, long srange, long xrange) +{ + char * scaledvec = NULL; + int dim = 0; + char *t = av->vector; + + av->scaled = TRUE; + + if ( t==NULL ) + return FALSE; + + if ( pointsDimension==0 ) + pointsDimension = dimensionofavector(av->vector); + if ( pointsDimension==0 ) + return FALSE; + + if ( scaledfloatvec==NULL ) { + scaledfloatvec = (double*)malloc(sizeof(double)*pointsDimension); + assert( scaledfloatvec!=NULL ); + } + + for (dim=0, t=av->vector; *t != '\0' && dim < pointsDimension; dim++) { + while ( !isdigit(*t) && *t != '\0' && *t != '.') t++; /* skip over the leading blanks; */ + // scaledfloatvec[dim] = strtof(t, &t); /* why this doesn't work? Why worked in LSH? Because g++ works fine; it's a bug in gcc/glibc... */ + sscanf(t, "%lg", &scaledfloatvec[dim]); /* get the number; */ + while ( isdigit(*t) || *t=='.') t++; /* skip over the number; */ + scaledfloatvec[dim] = scaledfloatvec[dim]*srange/xrange; + } + +#define PRECISION_LEN 13 + scaledvec = (char*)malloc((1+pointsDimension)*PRECISION_LEN); /* may be too many... */ + if ( scaledvec==NULL ) { + fprintf(stderr, "Not enough memory to scale all vectors. Try to free some vectors of previous groups...\n"); + return FALSE; + } + + t = scaledvec; + for (dim=0; dim0 ) { + t += inc; + } else { + t[0] = '\t'; t++; + } + assert( t-scaledvec<=pointsDimension*PRECISION_LEN ); /* a little conservative. */ + } + t[0] = '\n'; t[1] = '\0'; + + free(av->vector); /* why this seems no use for saving memory? TOFIX. */ + av->vector=scaledvec; + return TRUE; +} + +void outputavector(FILE* out, avector* av, long info) +{ + // fprintf(out, "%ld\t%ld\n%s%s", info, av->size, av->note, av->vector); + fprintf(out, "%s%s", av->note, av->vector); +} + +long vectorno = 1000000; +int main (int argc, char *argv[]) +{ + avector *vectors = NULL; + FILE * vecfile = NULL; + long vid = 0; + size_t len = 0; + regex_t num_node, node_kind; + regmatch_t matches[1]; + + if ( argc!=3 && argc!=4 && argc!=6 && argc!=7 ) { + fprintf(stderr, "Usage: vectorsort <#vector> [ flag_dist | {srange dist orange [flag_tofile]} ]\n"); + exit(1); + } + + vectorno = atol(argv[2]); + assert(vectorno>0); + + vecfile = fopen(argv[1], "r"); + if ( vecfile==NULL ) { + fprintf(stderr, "Can't open the vec file: %s\n", argv[1]); + exit(1); + } + + vectors = (avector*)malloc(sizeof(avector)*(vectorno+1)); + if ( vectors==NULL ) { + fprintf(stderr, "Memory malloc error for %ld vectors.\n", vectorno); + exit(1); + } + + assert(0 == regcomp(&num_node, "NUM_NODE:([0-9]+)", REG_EXTENDED) ); + assert(0 == regcomp(&node_kind, "NODE_KIND:([0-9]+)", REG_EXTENDED) ); + + initavector(&vectors[vid]); + while ( getline(&(vectors[vid].note), &len, vecfile)>=0 ) { + char temp; + int start, end; + + if ( getline(&(vectors[vid].vector), &len, vecfile)<=0 ) { + fprintf(stderr, "Reading stopped at line %ld. Should be checked there!\n", 2*(vid+1)); + } + + /* get node kind */ + if (0 != regexec(&node_kind, vectors[vid].note, 1, matches, 0) ) { + fprintf(stderr, "Line %ld has no node_kind?\n", 2*(vid+1)-1); + } else { + start = matches[0].rm_so; + end = matches[0].rm_eo; + assert( start!=-1 ); + temp = vectors[vid].note[end]; + vectors[vid].note[end] = '\0'; + vectors[vid].kind = atoi(vectors[vid].note+start+10); + vectors[vid].note[end] = temp; + } + + /* get num_node */ + if (0 != regexec(&num_node, vectors[vid].note, 1, matches, 0) ) { + fprintf(stderr, "Why line %ld has no NUM_NODE?\n", 2*(vid+1)-1); + continue; + } + start = matches[0].rm_so; + end = matches[0].rm_eo; + assert( start!=-1 ); + temp = vectors[vid].note[end]; + vectors[vid].note[end] = '\0'; + vectors[vid].size = atoi(vectors[vid].note+start+9); + // fprintf(stderr, "No. %ld Vec of size %ld from %s\n", vid+1, vectors[vid].size, vectors[vid].note+start+9); + if ( vectors[vid].size<=0 ) { + fprintf(stderr, "Warning: why line %ld has NUM_NODE<=0? Skip the vector.\n", 2*(vid+1)-1); + clearavector(&vectors[vid]); + continue; + } + assert( vectors[vid].size>0 ); + vectors[vid].note[end] = temp; + + vid++; + if ( vid>vectorno ) { + fprintf(stderr, "ERR: maximal #vec (%ld) is reached. Stop reading here.\n", vectorno); + break; + } + initavector(&vectors[vid]); + } + + fclose(vecfile); + pointsDimension = dimensionofavector(vectors[0].vector); /* get the dimension of the vector for later use. */ + + qsort(vectors, vid, sizeof(avector), compare_avector); + + if ( argc==3 ) { + // output sorted vectors: + long groups = 0, i; + for(i = 0; i startsize ) { + // start a new size/#occurrence: + if ( nsize > 0 ) + fprintf(stdout, "%ld\t%ld\n", startsize, nsize); + startsize = vectors[i].size; + nsize = 1; + } else + nsize++; + } + if ( nsize > 0 ) // last size + fprintf(stdout, "%ld\t%ld\n", startsize, nsize); + } else { +#define MAX_NUM_GROUP 1024*2 // only need hundreds if d>0 and c>=0. And 1024*1024 may be too big for some machines, such as my C640 and msg. + long s; double d, c; + long al[MAX_NUM_GROUP], au[MAX_NUM_GROUP]; + long groups = 0, i; + assert ( argc==6 || argc==7 ); + s = atol(argv[3]); + d = atof(argv[4]); + c = atof(argv[5]); + assert(s>0); /* the upper bound of the first range */ + assert(d>=0 && d=s is not very meaningful */ + if ( !(d>0 && c=0) && !(d==0 && c==-1) ) { /* the overlapped range. c<-1 may skip some vectors. */ + fprintf(stderr, "Warning: c=%g is not good for this case, ", c); + if ( d==0 ) + fprintf(stderr, "better to use c==-1.\n"); + else + fprintf(stderr, "better to use c in [0,%g)\n", d); + } + + /* compute all the group ranges */ + al[0] = 0, au[0] = (long)ceil(s+d); + groups = 1; + while ( al[groups-1]<=vectors[vid-1].size && au[groups-1]<=vectors[vid-1].size ) { + /* compute the range for a new group */ + if ( groups >= MAX_NUM_GROUP ) { + fprintf(stderr, "ERR: Can't handle more than %ld groups. The last group is set to include all rest vectors.\n", groups); + au[groups-1] = vectors[vid-1].size+1; + break; + } + + if ( groups == 1 ) { + /* begin the second group: */ + if ( c<0 ) { + /* do not to worry about false clone transitions or scalable "c": */ + al[groups] = (long)ceil(s+d-c); + au[groups] = (long)ceil((s+d)*al[groups]/s); + } else { + al[groups] = (long)ceil(s+d-c); + au[groups] = (long)ceil(s+2*d+d*d/s-c*c/s+c+1); + } + } else { + if ( c<0 ) { + al[groups] = (long)ceil(au[groups-1]-c); + au[groups] = (long)ceil((s+d)*al[groups]/s); + } else { + al[groups] = (long)ceil(au[groups-1]-c*al[groups-1]/s); + au[groups] = (long)ceil((s+d+c)*au[groups-1]/s-(d*c+c*c)*al[groups-1]/s/s+1); + } + } + groups++; + } + + /* output the group ranges: */ + if ( argc==6 ) { + for ( i=0; ivectors[i-1].size) ) { + /* record the starting position of the next group: */ + nexti = i; + } + if ( vectors[i].size<=currau ) { + if ( vectors[i].size>=curral ) { +// comment the scaling out to make it more natural (use different distances during clustering later on) . +/* if ( vectors[i].scaled==FALSE && curral>0 ) */ +/* scaleavector(&vectors[i], s, curral); */ + outputavector(groupfile, &vectors[i], i); + groupsize++; + } /* here may skip some vectors when c<-1. */ + /* An optimization to save some memory: */ + if ( vectors[i].size