From 30118e907f474e7d3161084cb7ce7cb7a1a433c0 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 2 Jun 2015 11:56:34 +0200 Subject: [PATCH] Dev Testable version of participant import. --- application/components/Batch.php | 3 + application/controllers/Controller.php | 2 +- application/controllers/DevController.php | 779 ++++++++++++++++++ .../controllers/ParticipantsController.php | 121 +-- .../controllers/TemplatesController.php | 11 + application/helpers/Html.php | 20 + .../themes/default/assets/css/csvimport.css | 56 +- application/views/participants/import.php | 94 ++- 8 files changed, 998 insertions(+), 88 deletions(-) create mode 100644 application/controllers/DevController.php create mode 100644 application/controllers/TemplatesController.php create mode 100644 application/helpers/Html.php diff --git a/application/components/Batch.php b/application/components/Batch.php index 179f86bed4f..3372281f5de 100644 --- a/application/components/Batch.php +++ b/application/components/Batch.php @@ -12,6 +12,7 @@ class Batch { protected $defaultCategory; public $batchSize; + public $commitCount = 0; protected $data = []; public function __construct(Closure $callback, $batchSize = 5000, $defaultCategory = 'default') { @@ -41,8 +42,10 @@ public function add($elements, $category = null) { } public function commitCategory($category) { + $callback = $this->callback; $callback($this->data[$category], $category); + $this->commitCount++; unset($this->data[$category]); } public function commit() { diff --git a/application/controllers/Controller.php b/application/controllers/Controller.php index de8ced94297..788ef1f5f6c 100644 --- a/application/controllers/Controller.php +++ b/application/controllers/Controller.php @@ -188,7 +188,7 @@ protected function loadModel($id) { public function getActionParams() { - return array_merge($_GET, $_POST); + return App()->request->psr7->getParsedBody(); } diff --git a/application/controllers/DevController.php b/application/controllers/DevController.php new file mode 100644 index 00000000000..dc76b1add37 --- /dev/null +++ b/application/controllers/DevController.php @@ -0,0 +1,779 @@ +'; + $copy = $_SESSION; + unset($copy['LSWebUsermodel']); +// unset($copy['SSM']); + unset($copy['fieldmap-652359en']); + var_dump($copy); + + } + + protected function parseExpression(array &$tokens, array &$operandStack, array &$operatorStack) + { + /** + * Check production rules: + * LP => LP EXPR RP + * NUM => NUM + * NAME => NAME + * OP => OP EXPR + */ + $token = array_shift($tokens); + echo $token[2] . '
'; + switch($token[2]) { + case 'LP': + + while($this->parseExpression($tokens, $operandStack, $operatorStack)) {} + $rp = array_shift($tokens); + if ($rp[2] != 'RP'){ + die("Expected RP, got {$rp[2]}"); + } + break; + case 'NUMBER': + case 'WORD': + array_push($operandStack, $token); + break; + case 'AND_OR': + case 'COMPARE': + case 'BINARYOP': + $operand1 = array_pop($operandStack); + array_push($operatorStack, $token); + $this->parseExpression($tokens, $operandStack, $operatorStack); + $operand2 = array_pop($operandStack); + array_push($operandStack, [$token, $operand1, $operand2]); + break; + default: + array_unshift($tokens, $token); + return false; + } + return true; + } + + protected function dumpAST($tree, $expression = null) { + /* @var \ls\expressionmanager\Token $token */ + if (is_array($tree)) { + $token = array_shift($tree); + + if ($token->context == 'FUNC') { + $subtree = $tree[0]; + } else { + $subtree = $tree; + } + $subContent = \CHtml::openTag('ul'); + foreach($subtree as $child) { + $subContent .= \CHtml::tag('li', [], $this->dumpAST($child, $expression)); + } + + $subContent .= \CHtml::closeTag('ul'); +// die(htmlentities($subContent)); + return \CHtml::tag('ul', [], \CHtml::tag('li', [], $this->dumpAST($token, $expression) . $subContent)); + } else { + if (is_bool($tree->value)) { + $value = $tree->value ? 'true' : 'false'; + } elseif ($tree->value === "") { + $value = "EMPTY STRING"; + } else { + $value = $tree->value; + } + if (strlen($value) == 0) { + var_dump($tree); + die('no'); + } + return \CHtml::link($value); + } + + } + + public function actionIndex() { + echo '
';
+        var_dump(get_declared_classes()); die();
+        $parser = new Parser();
+
+        $expressions = [
+            'abs.test',
+            '5 + 3',
+            '4 * -2',
+            'abs..'
+        ];
+        $tests  = <<Empty Vs. Empty~"Empty Vs. Empty"
+1~'' == ''
+0~'' != ''
+0~'' > ''
+1~'' >= ''
+0~'' < ''
+1~'' <= ''
+1~!''
+~('' and '')
+~('' or '')
+Empty Vs. Zero~"Empty Vs. Zero"
+0~'' == 0
+1~'' != 0
+0~'' > 0
+0~'' >= 0
+0~'' < 0
+0~'' <= 0
+1~!''
+1~!0
+0~('' and 0)
+0~('' or 0)
+Empty Vs. Constant~"Empty Vs. Constant"
+0~'' == 3
+1~'' != 3
+0~'' > 3
+0~'' >= 3
+0~'' < 3
+0~'' <= 3
+1~!''
+0~!3
+0~('' and 3)
+1~('' or 3)
+Empty Vs. Empty_Var~"Empty Vs. Empty_Var"
+1~'' == empty
+0~'' != empty
+0~'' > empty
+1~'' >= empty
+0~'' < empty
+1~'' <= empty
+1~!''
+1~!empty
+~('' and empty)
+~('' or empty)
+Empty_Var Vs. Zero~"Empty_Var Vs. Zero"
+0~empty == 0
+1~empty != 0
+0~empty > 0
+0~empty >= 0
+0~empty < 0
+0~empty <= 0
+1~!empty
+1~!0
+0~(empty and 0)
+0~(empty or 0)
+Empty_Var Vs. Zero~"Empty_Var Vs. Zero"
+0~empty == zero
+1~empty != zero
+0~empty > zero
+0~empty >= zero
+0~empty < zero
+0~empty <= zero
+1~!empty
+1~!zero
+0~(empty and zero)
+0~(empty or zero)
+Empty_Var Vs. Constant~"Empty_Var Vs. Constant"
+0~empty == 3
+1~empty != 3
+0~empty > 3
+0~empty >= 3
+0~empty < 3
+0~empty <= 3
+1~!empty
+0~!3
+0~(empty and 3)
+1~(empty or 3)
+Solution: Empty_Var Vs. Zero~"Solution: Empty_Var Vs. Zero"
+0~!is_empty(empty) && (empty == 0)
+0~!is_empty(five) && (five == 0)
+1~!is_empty(zero) && (zero == 0)
+0~!is_empty(empty) && (empty > 0)
+0~!is_empty(empty) && (empty >= 0)
+0~!is_empty(empty) && (empty < 0)
+0~!is_empty(empty) && (empty <= 0)
+0~!is_empty(empty) && ((empty and 0))
+0~!is_empty(empty) && ((empty or 0))
+Solution: Empty_Var Vs. Zero~"Solution: Empty_Var Vs. Zero"
+0~!is_empty(empty) && (empty == zero)
+0~!is_empty(five) && (five == zero)
+1~!is_empty(zero) && (zero == zero)
+0~!is_empty(empty) && (empty > zero)
+0~!is_empty(empty) && (empty >= zero)
+0~!is_empty(empty) && (empty < zero)
+0~!is_empty(empty) && (empty <= zero)
+0~!is_empty(empty) && ((empty and zero))
+0~!is_empty(empty) && ((empty or zero))
+Solution: Empty_Var Vs. Constant~"Solution: Empty_Var Vs. Constant"
+0~!is_empty(empty) && (empty < 3)
+0~!is_empty(empty) && (empty <= 3)
+Solution: Empty_Var Vs. Variable~"Solution: Empty_Var Vs. Variable"
+0~!is_empty(empty) && (empty < five)
+0~!is_empty(empty) && (empty <= five)
+Solution: The Hard One is Empty_Var != 0~"Solution: The Hard One is Empty_Var != 0"
+1~(empty != 0)
+1~!is_empty(empty) && (empty != 0)
+1~is_empty(empty) || (empty != 0)
+1~is_empty(empty) || (empty != zero)
+0~is_empty(zero) || (zero != 0)
+1~is_empty(five) || (five != 0)
+SETUP~'SETUP'
+"Can strings contain embedded \"quoted passages\" (and parentheses + other characters?)?"~a=htmlspecialchars(ASSESSMENT_HEADING)
+"can single quoted strings" . 'contain nested 'quoted sections'?~b=htmlspecialchars(QUESTIONHELP)
+Can strings have embedded <tags> like <html>, or even unbalanced "quotes or entities without terminal semicolons like &amp and  &lt?~c=htmlspecialchars(QUESTION_HELP)
+Hi there!~d='Hi there!'
+FUNCTIONS~'FUNCTIONS'
+5~abs(five)
+5~abs(-five)
+0.2~acos(cos(0.2))
+0~acos(cos(pi()))-pi()
+"Can strings contain embedded \\"quoted passages\\" (and parentheses + other characters?)?"~addslashes(a)
+"can single quoted strings" . 'contain nested 'quoted sections'?~addslashes(b)
+Can strings have embedded <tags> like <html>, or even unbalanced "quotes or entities without terminal semicolons like &amp and  &lt?~addslashes(c)
+0.2~asin(sin(0.2))
+0.2~atan(tan(0.2))
+0~atan2(0,1)
+1~ceil(0.3)
+1~ceil(0.7)
+0~ceil(-0.3)
+0~ceil(-0.7)
+10~ceil(9.1)
+1~checkdate(1,29,1967)
+0~checkdate(2,29,1967)
+0.2~cos(acos(0.2))
+5~count(1,2,3,4,5)
+0~count()
+5~count(one,two,three,four,five)
+2~count(a,'',c)
+NULL~date('F j, Y, g:i a',time())
+April 5, 2006, 1:02 am~date('F j, Y, g:i a',mktime(1,2,3,4,5,6))
+20~floor(exp(3))
+0~floor(asin(sin(pi())))
+9~floor(9.9)
+3~floor(pi())
+January 12, 2012, 5:27 pm~date('F j, Y, g:i a',1326410867)
+January 12, 2012, 11:27 pm~gmdate('F j, Y, g:i a',1326410867)
+"Can strings contain embedded \"quoted passages\" (and parentheses + other characters?)?"~html_entity_decode(a)
+"can single quoted strings" . 'contain nested 'quoted sections'?~html_entity_decode(b)
+Can strings have embedded  like , or even unbalanced "quotes or entities without terminal semicolons like & and  <?~html_entity_decode(c)
+"Can strings contain embedded \"quoted passages\" (and parentheses + other characters?)?"~htmlentities(a)
+"can single quoted strings" . 'contain nested 'quoted sections'?~htmlentities(b)
+Can strings have embedded <tags> like <html>, or even unbalanced "quotes or entities without terminal semicolons like &amp and &lt?~htmlentities(c)
+1~c==htmlspecialchars(htmlspecialchars_decode(c))
+1~b==htmlspecialchars(htmlspecialchars_decode(b))
+1~a==htmlspecialchars(htmlspecialchars_decode(a))
+"Can strings contain embedded \"quoted passages\" (and parentheses + other characters?)?"~htmlspecialchars_decode(a)
+"can single quoted strings" . 'contain nested 'quoted sections'?~htmlspecialchars_decode(b)
+Can strings have embedded like , or even unbalanced "quotes or entities without terminal semicolons like & and  like , or even unbalanced "quotes or entities without terminal semicolons like & and <?~htmlspecialchars(c)
+9~idate('B',1326410867)
+0~if('0',1,0)
+0~if(0,1,0)
+1~if(!0,1,0)
+0~if(!(!0),1,0)
+1~if('true',1,0)
+1~if('false',1,0)
+1~if('00',1,0)
+0~if('',1,0)
+1~if('A',1,0)
+0~if(empty,1,0)
+4~if(5 > 7,2,4)
+1~if(' ',1,0)
+there~if((one > two),'hi','there')
+64~if((one < two),pow(2,6),pow(6,2))
+H e l l o~implode(' ','H','e','l','l','o')
+1|2|3|4|5~implode('|',one,two,three,four,five)
+123~join(1,2,3)
+123 5~join(one,2,three," ",five)
+4~intval('4')
+4~intval('100',2)
+5~intval(5.7)
+0~is_empty(four)
+1~is_empty(empty)
+1~is_empty('')
+0~is_empty(0)
+0~is_empty('0')
+0~is_empty('false')
+0~is_empty('NULL')
+0~is_empty(1)
+1~is_empty(one==two)
+0~!is_empty(one==two)
+1~is_float(half)
+0~is_float(one)
+1~is_float(pi())
+0~is_float(5)
+0~is_int(half)
+1~is_int(one)
+0~is_nan(half)
+1~is_nan(WELCOME)
+1~is_null(sdfjskdfj)
+0~is_null(four)
+0~is_numeric(empty)
+1~is_numeric('1')
+1~is_numeric(four)
+0~is_numeric('hi')
+1~is_numeric(five)
+0~is_numeric(hi)
+0~is_string(four)
+1~is_string('hi')
+1~is_string(hi)
+1, 2, 3, 4, 5~list(one,two,three,min(four,five,six),max(three,four,five))
+11, 12~list(eleven,twelve)
+0, 1, 3, 5~list(0,one,'',three,'',five)
+1~log(exp(1))
+2~log(exp(2))
+I was trimmed   ~ltrim('     I was trimmed   ')
+10~max(5,6,10,-20)
+6~max(five,(one + (two * four)- three))
+6~max((one + (two * four)- three))
+212~5 + max(1,(2+3),(4 + (5 + 6)),((7 + 8) + 9),((10 + 11), 12),(13 + (14 * 15) - 16))
+29~five + max(one, (two + three), (four + (five + six)),((seven + eight) + nine),((ten + eleven), twelve),(one + (two * three) - four))
+1024~max(one,(two*three),pow(four,five),six)
+2~max(one,two)
+5~max(one,two,three,four,five)
+-5~min(-5,10,15,12,-3)
+1~min(five,four,one,two,three)
+1344765967~mktime(5,6,7,8)
+1144191723~mktime(1,2,3,4,5,6)
+1,000~number_format(1000)
+1,000.23~number_format(1000.23)
+1,234,567~number_format(1234567)
+315~ceil(100*pi())
+1~pi() == pi() * 2 - pi()
+4~pow(2,2)
+27~pow(3,3)
+=~quoted_printable_decode(quoted_printable_encode('='))
+\\$~quotemeta('$')
+IGNORE THIS ERROR~rand(3,5)
+0~(a=rand())-a
+1~regexMatch('/embedded/',c)
+1~regexMatch('/^.*embedded.*$/',c)
+0~regexMatch('/joe/',c)
+1~regexMatch('/(?:dog|cat)food/','catfood stinks')
+1~regexMatch('/(?:dog|cat)food/','catfood stinks')
+1~regexMatch('/[0-9]{3}-[0-9]{2}-[0-9]{4}/','123-45-6789')
+1~regexMatch('/\d{3}-\d{2}-\d{4}/','123-45-6789')
+1~regexMatch('/(?:\(\d{3}\))\s*\d{3}-\d{4}/','(212) 555-1212')
+0~round(0.2)
+1~round(.8)
+0.07~0.01 + 0.06
+0.07~round(0.01 + 0.06,10)
+     I was trimmed~rtrim('     I was trimmed   ')
+0.2~sin(asin(0.2))
+1~sin(pi()/2)
+1~sin(pi()/2) == sin(.5 * pi())
+1~sin(0.5 * pi())
+hello,5~sprintf('%s,%d','hello',5)
+2~sqrt(4)
+158~round(stddev(4,5,6,7,8)*100)
+hello-----~str_pad('hello',10,'-')
+hello     ~str_pad('hello',10)
+hello~str_pad('hello',3)
+testtesttest~str_repeat('test',3)
+I am awesome~str_replace('You are','I am','You are awesome')
+I love LimeSurvey~str_replace('like','love','I like LimeSurvey')
+1~0==strcasecmp('Hello','hello')
+0~0==strcasecmp('Hello','hi')
+1~0==strcmp('Hello','Hello')
+0~0==strcmp('Hello','hi')
+Hi there!~c=strip_tags(d)
+hello~strip_tags('hello')
+5~stripos('ABCDEFGHI','f')
+hi~stripslashes('\\h\\i')
+FGHI~stristr('ABCDEFGHI','fg')
+5~strlen('12345')
+5~strlen(hi)
+0~strpos('ABCDEFGHI','f')
+5~strpos('ABCDEFGHI','F')
+2~strpos('I like LimeSurvey','like')
+54321~strrev('12345')
+0~strstr('ABCDEFGHI','fg')
+FGHI~strstr('ABCDEFGHI','FG')
+hi there!~strtolower(c)
+HI THERE!~strtoupper(c)
+3600~strtotime("27 Mar 1976 8:20")-strtotime("1976/03/27 7:20")
+10~(strtotime("13 Apr 2013")-strtotime("2013-04-03"))/60/60/24
+1985-11-05 00:00:00~date("Y-m-d H:i:s",strtotime("05 Nov 1985"))
+HOURS PASSED SINCE 1970~round(strtotime("now")/60/60)
+~""
+1985-11-05 00:00:00~date("Y-m-d H:i:s",strtotime("11/5/85"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("8/9/10"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("8/9/2010"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("2010/8/9"))
+~""
+1985-11-05 00:00:00~date("Y-m-d H:i:s",strtotime("85-11-5"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("10-8-9"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("9-8-2010"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("2010-8-9"))
+~""
+1985-11-05 00:53:20~date("Y-m-d H:i:s",strtotime("85-11-5 0:53:20"))
+2010-08-09 00:53:20~date("Y-m-d H:i:s",strtotime("10-8-9 0:53:20"))
+2010-08-09 11:12:13~date("Y-m-d H:i:s",strtotime("9-8-2010 11:12:13"))
+2010-08-09 11:12:13~date("Y-m-d H:i:s",strtotime("2010-8-9 11:12:13"))
+~""
+Today 11:11:59~date("Y-m-d H:i:s",strtotime("11.11.59"))
+Today 9:08:10~date("Y-m-d H:i:s",strtotime("9.8.10"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("9.8.2010"))
+~""
+1985-11-05 00:53:20~date("Y-m-d H:i:s",strtotime("5.11.85 0:53:20"))
+2010-08-09 11:12:13~date("Y-m-d H:i:s",strtotime("9.8.2010 11:12:13"))
+~""
+1970-01-01 00:00:00~date("Y-m-d H:i:s",strtotime("70-01-01"))
+1999-01-01 00:00:00~date("Y-m-d H:i:s",strtotime("99-01-01"))
+2001-01-01 00:00:00~date("Y-m-d H:i:s",strtotime("01-01-01"))
+1902-01-01 00:00:00~date("Y-m-d H:i:s",strtotime("1902-01-01"))
+~""
+today 2:15:00~date("Y-m-d H:i:s",strtotime("2:15:00"))
+Some dates that are not (correctly) parsed:~"Some dates that are not (correctly) parsed:"
+1969-01-19 00:00:00~date("Y-m-d H:i:s",strtotime("69-01-19"))
+1985-11-05 00:00:00~date("Y-m-d H:i:s",strtotime("85/11/5"))
+1985-11-05 00:00:00~date("Y-m-d H:i:s",strtotime("5-11-85"))
+2010-08-09 00:00:00~date("Y-m-d H:i:s",strtotime("2010.8.9"))
+1985-11-05 00:00:00~date("Y-m-d H:i:s",strtotime("85.11.5"))
+1985-11-05 00:53:20~date("Y-m-d H:i:s",strtotime("85.11.5 0:53:20"))
+2010-08-09 11:12:13~date("Y-m-d H:i:s",strtotime("9.8.10 11:12:13"))
+678~substr('1234567890',5,3)
+15~sum(1,2,3,4,5)
+15~sum(one,two,three,four,five)
+0.2~tan(atan(0.2))
+IGNORE THIS ERROR~time()
+I was trimmed~trim('     I was trimmed   ')
+Hi There You~ucwords('hi there you')
+EXPRESSIONS~'EXPRESSIONS'
+1~!'0'
+1~0 eq '0'
+0~0 ne '0'
+0~0 eq empty
+1~0 ne empty
+0~0 eq ''
+1~0 ne ''
+0~'' < 10
+0~0 < empty
+1~0 <= empty
+0~0 > empty
+1~0 >= empty
+0~'0' eq empty
+1~'0' ne empty
+0~'0' < empty
+1~'0' <= empty
+0~'0' > empty
+1~'0' >= empty
+1~empty eq empty
+0~empty ne empty
+0~'' > 0
+0~' ' > 0
+1~!0
+0~!' '
+0~!'A'
+0~!1
+0~!'1'
+1~!''
+1~!empty
+1~'0'==0
+0~'A'>0
+0~'A'<0
+0~'A'==0
+0~'A'>=0
+0~'A'<=0
+0~0>'A'
+0~0>='B'
+0~0=='C'
+0~0<'D'
+0~0<='E'
+1~0!='F'
+1~'A' or 'B'
+1~'A' and 'B'
+0~'A' eq 'B'
+1~'A' ne 'B'
+1~'A' < 'B'
+1~'A' <= 'B'
+0~'A' > 'B'
+0~'A' >= 'B'
+AB~'A' + 'B'
+NAN~'A' - 'B'
+NAN~'A' * 'B'
+NAN~'A' / 'B'
+1~'A' or empty
+0~'A' and empty
+0~'A' eq empty
+1~'A' ne empty
+0~'A' < empty
+0~'A' <= empty
+1~'A' > empty
+1~'A' >= empty
+A~'A' + empty
+NAN~'A' - empty
+NAN~'A' * empty
+NAN~'A' / empty
+0~0 or empty
+0~0 and empty
+0~0 + empty
+0~0 - empty
+0~0 * empty
+NAN~0 / empty
+0~(-1 > 0)
+0~zero
+~empty
+1~five > zero
+1~five > empty
+1~empty < 16
+1~zero == empty
+3~q5pointChoice.code
+5~q5pointChoice.type
+(question for q5pointChoice)~q5pointChoice.question
+1~q5pointChoice.relevance
+4~q5pointChoice.NAOK + 1
+NULL~q5pointChoice.bogus
+14~q5pointChoice.qid
+7~qArrayNumbers_ls1_min.code
+1~(one * (two + (three - four) + five) / six)
+2.4~(one  * two) + (three * four) / (five * six)
+50~12X34X56 * 12X3X5lab1_ber
+1~c == 'Hi there!'
+1~c == "Hi there!"
+3~a=three
+3~c=a
+12~c*=four
+15~c+=a
+5~c/=a
+-1~c-=six
+24~one * two * three * four
+-4~five - four - three - two
+0~two * three - two - two - two
+4~two * three - two
+105~5 + 1, 7 * 15
+7~7
+15~10 + 5
+24~12 * 2
+10~13 - 3
+3.5~14 / 4
+5~3 + 1 * 2
+1~one
+there~hi
+6.25~one * two - three / four + five
+1~one + hi
+1~two > one
+1~two gt one
+1~three >= two
+1~three ge  two
+0~four < three
+0~four lt three
+0~four <= three
+0~four le three
+0~four == three
+0~four eq three
+1~four != three
+0~four ne four
+NAN~one * hi
+0~a='hello',b='',c=0
+hello~a
+0~c
+0~one && 0
+0~two and 0
+1~five && 6
+1~seven && eight
+1~one or 0
+1~one || 0
+1~(one and 0) || (two and three)
+value for {QID}~QID
+"Can strings contain embedded \"quoted passages\" (and parentheses + other characters?)?"~ASSESSMENT_HEADING
+"can single quoted strings" . 'contain nested 'quoted sections'?~QUESTIONHELP
+Can strings have embedded  like , or even unbalanced "quotes or entities without terminal semicolons like & and  <?~QUESTION_HELP
+value for {TOKEN:FIRSTNAME}~TOKEN:FIRSTNAME
+value for {THEREAREXQUESTIONS}~THEREAREXQUESTIONS
+15~12X3X5lab1_ber#1
+1~three == three
+1~three == 3
+11~eleven
+144~twelve * twelve
+0~!three
+8~five + + three
+2~five + - three
+SYNTAX ERRORS~'SYNTAX ERRORS'
+NULL~*
+NULL~three +
+NULL~four * / seven
+NULL~(five - three
+NULL~five + three)
+NULL~seven + = four
+NULL~>
+NULL~five > > three
+NULL~seven > = four
+NULL~seven >=
+NULL~three &&
+NULL~three ||
+NULL~three +
+NULL~three >=
+NULL~three +=
+NULL~three !
+NULL~three *
+NULL~five ! three
+NULL~(5 + 7) = 8
+NULL~&& four
+NULL~min(
+NULL~max three, four, five)
+NULL~three four
+NULL~max(three,four,five) six
+NULL~WELCOME='Good morning'
+NULL~TOKEN:FIRSTNAME='Tom'
+NULL~NUMBEROFQUESTIONS+=3
+NULL~NUMBEROFQUESTIONS*=4
+NULL~NUMBEROFQUESTIONS/=5
+NULL~NUMBEROFQUESTIONS-=6
+NULL~'Tom'='tired'
+NULL~max()
+NULL~convert_value( 10, 1, '0,5,10,15,20', '0,5,10,15')
+100~convert_value( 10, 1, '0,5,10,15,20', '0,50,100,150,200')
+NULL~convert_value( 10, 0, '0,5,10,15,20', '0,50,100,150,200')
+100~convert_value( 8, 0, '0,5,10,15,20', '0,50,100,150,200')
+100~convert_value( 12, 0, '0,5,10,15,20', '0,50,100,150,200')
+0~convert_value( 0, 0, '0,5,10,15,20', '0,50,100,150,200')
+0~convert_value( -10000, 0, '0,5,10,15,20', '0,50,100,150,200')
+NULL~convert_value( -10000, 1, '0,5,10,15,20', '0,50,100,150,200')
+200~convert_value( 20, 0, '0,5,10,15,20', '0,50,100,150,200')
+200~convert_value( 20, 1, '0,5,10,15,20', '0,50,100,150,200')
+200~convert_value( 30, 0, '0,5,10,15,20', '0,50,100,150,200')
+NULL~convert_value( 30, 1, '0,5,10,15,20', '0,50,100,150,200')
+EOD;
+
+
+//        die();
+        $expressions = [];
+        foreach(explode("\n", $tests) as $testDef) {
+            if (false === $pos = strpos($testDef, '~')) {
+                echo htmlentities($testDef) . '
';
+                continue;
+            } elseif ($pos > 0) {
+                list($expected, $expressions[]) = explode('~', $testDef, 2);
+            } else {
+                $expressions[] = substr($testDef, 1);
+            }
+        }
+//        $expressions = [];
+        $expressions = array_merge($expressions, [
+            '5 + 87',
+
+            'test(3, 5)',
+            'test(3, 4 + 5)'
+//            'max(1,(2+3),(4 + (5 + 6)),((7 + 8) + 9))',
+//            'max(1,(2+3),(4 + (5 + 6)),((7 + 8) + 9),((10 + 11), 12),(13 + (14 * 15) - 16))',
+//            '5 + max(1,(2+3),(4 + (5 + 6)),((7 + 8) + 9),(10 + 11), 12,(13 + (14 * 15) - 16))',
+//            '!false',
+//            '(-five)',
+//            'abs(-five)',
+//            'acos(cos(pi()))-pi()',
+
+        ]);
+
+        $outputs = [];
+
+        foreach($expressions as $expression) {
+            ob_start();
+            echo "******************************************
";
+            echo $expression ."
";
+            echo "------------------------------------------
";
+            echo '
';
+            $ast = $parser->parse($expression);
+            echo '
';
+            $outputs[] = [
+                'debug' => ob_get_clean(),
+                'tree' => isset($ast) ? \CHtml::tag('div', ['class' => 'tree'], $this->dumpAST($ast, $expression)) : 'no tree'
+            ];
+        }
+        ob_get_clean();
+        App()->getClientScript()->registerCssFile('/styles/ast.css');
+        $result = '';
+        foreach($outputs as $output) {
+            $result .= $output['debug'] . $output['tree'];
+        }
+        $this->renderText($result);
+        die();
+
+
+    }
+
+    public function actionMigrateTest() {
+        App()->loadHelper('globalsettings');
+        var_dump(getUpdateInfo());
+        $zip = new \ZipArchive();
+        $zip->open('/home/sam/Downloads/limesurvey205plus-build150211.zip');
+        $zip2 = new \ZipArchive();
+        $zip2->open('/tmp/new.zip');
+        
+        // Hashes:
+        $hashesFrom = [];
+        $hashesTo = [];
+        $count = $zip->numFiles;
+        $start = microtime(true);
+        for ($i = 0; $i < $count; $i++) {
+            $hashesFrom[$zip->getNameIndex($i)] = md5($zip->getFromIndex($i));
+        }
+        $count2 = $zip2->numFiles;
+        
+        var_dump($count2);
+        for ($i = 0; $i < $count2; $i++) {
+            $hashesTo[$zip2->getNameIndex($i)] = md5($zip2->getFromIndex($i));
+        }
+        
+        // Deleted files:
+        $deleted = array_diff_key($hashesFrom, $hashesTo);
+        
+        // Created files:
+        $created = array_diff_key($hashesTo, $hashesFrom);
+        
+        // Changed files:
+        $changed = [];
+        foreach($hashesFrom as $file => $hash) {
+            if (isset($hashesTo[$file]) && $hashesTo[$file] != $hash) {
+                $changed[$file] = $hash;
+            }
+        }
+         
+        echo "Deleted " . count($deleted) ." files.
";
+//        var_dump($deleted);
+        echo "Created " . count($created) ." files.
";
+//        var_dump($created);
+        echo "Changed " . count($changed) ." files.
";
+//        var_dump($changed);
+        echo "Source files: " . $zip->numFiles;
+        echo "Target files: " . $zip2->numFiles;
+        
+        $end = microtime(true) - $start;
+        var_dump($end);
+
+        
+    }
+    
+    public function actionModule() {
+        $id = array_keys(App()->modules)[0];
+//        die(App()->createUrl("$id/dashboard"));
+    }
+}
\ No newline at end of file
diff --git a/application/controllers/ParticipantsController.php b/application/controllers/ParticipantsController.php
index 1b5d723df0f..ae3570bcd0e 100644
--- a/application/controllers/ParticipantsController.php
+++ b/application/controllers/ParticipantsController.php
@@ -45,12 +45,47 @@ public function actionAttributes($id) {
     }
 
 
-    public function actionImport() {
-        $start = microtime(true);
+    public function actionImport()
+    {
+        $this->render('import');
+    }
+
+
+    public function actionAjaxImport(array $items, $querySize = 1000)
+    {
+        header('Content-Type: application/json');
+        // Set response code so on errors (max execution time, memory limit) we don't get http 200.
+        http_response_code(501);
+        set_time_limit(3);
+        ini_set('memory_limit', '92M');
+        $return_bytes = function($val) {
+            $val = trim($val);
+            $last = strtolower($val[strlen($val)-1]);
+            switch($last) {
+                // The 'G' modifier is available since PHP 5.1.0
+                case 'g':
+                    $val *= 1024;
+                case 'm':
+                    $val *= 1024;
+                case 'k':
+                    $val *= 1024;
+            }
+
+            return $val;
+        };
+        $start = App()->request->psr7->getServerParams()['REQUEST_TIME_FLOAT'];
+        $memoryLimit = $return_bytes(ini_get('memory_limit'));
+
+
+
+
+
         $participant = new Participant();
         $regularFields = $participant->safeAttributeNames;
         $tableName = $participant->tableName();
         $attributeTableName = \ParticipantAttribute::model()->tableName();
+
+        $fields = array_flip($regularFields);
         $batchInserter = new \Batch(function(array $batch, $category = null) {
             if (!empty($batch)) {
                 \Yii::beginProfile('query');
@@ -65,66 +100,40 @@ public function actionImport() {
             }
         }, 1000, $tableName);
 
-        $fields = array_flip($regularFields);
-        if (isset(App()->request->psr7->getParsedBody()['items'])) {
-            if (isset(App()->request->psr7->getParsedBody()['querySize'])) {
-                $batchInserter->batchSize = App()->request->psr7->getParsedBody()['querySize'];
-            }
-            \Yii::beginProfile('import');
-            $count = 0;
-            // Custom validation for better performance.
-//            $validators = $participant->getValidators();
-            $initialAttributes = $participant->getAttributes();
-            array_map(function($row) use ($batchInserter, $attributeTableName, $participant, $initialAttributes) {
-                \Yii::beginProfile('row');
-              $participant->setAttributes($initialAttributes, false);
-//                 Manual for better performance.
-                \Yii::beginProfile('alternative');
-                foreach($row as $key => $value) {
-                    if (isset($fields[$key])) {
-                        $participant->$key = $value;
-                    }
+
+        $initialAttributes = $participant->getAttributes();
+        array_map(function($row) use ($batchInserter, $attributeTableName, $participant, $initialAttributes) {
+            \Yii::beginProfile('row');
+            $participant->setAttributes($initialAttributes, false);
+            \Yii::beginProfile('alternative');
+            foreach($row as $key => $value) {
+                if (isset($fields[$key])) {
+                    $participant->$key = $value;
                 }
-                \Yii::endProfile('alternative');
+            }
+            \Yii::endProfile('alternative');
 
-                if ($participant->validate()) {
-                    $batchInserter->add($participant->getAttributes());
-                    $batchInserter->add($participant->getNewCustomAttributes(), $attributeTableName);
-                } else {
-                    var_dump($participant->errors);
+            if ($participant->validate()) {
+                $batchInserter->add($participant->getAttributes());
+                $batchInserter->add($participant->getNewCustomAttributes(), $attributeTableName);
+            } else {
+                var_dump($participant->errors);
 
-                }
+            }
 
-                \Yii::endProfile('row');
-            }, App()->request->psr7->getParsedBody()['items']);
-            unset($batchInserter);
-            \Yii::endProfile('import');
-
-            $return_bytes = function($val) {
-                $val = trim($val);
-                $last = strtolower($val[strlen($val)-1]);
-                switch($last) {
-                    // The 'G' modifier is available since PHP 5.1.0
-                    case 'g':
-                        $val *= 1024;
-                    case 'm':
-                        $val *= 1024;
-                    case 'k':
-                        $val *= 1024;
-                }
+            \Yii::endProfile('row');
+        }, $items);
+        \Yii::endProfile('import');
 
-                return $val;
-            };
-            header('Content-Type: application/json');
-            echo json_encode([
-                'memory' => memory_get_peak_usage() / $return_bytes(ini_get('memory_limit')),
-                'time' => microtime(true) - $start
-            ]);
 
 
-        } else {
-            $this->render('import');
-        }
-    }
+        http_response_code(200);
+        echo json_encode([
+            'memory' => memory_get_peak_usage() / $memoryLimit,
+            'time' => (microtime(true) - $start) / ini_get('max_execution_time'),
+            'queries' => $batchInserter->commitCount
+        ]);
+
 
+    }
 }
\ No newline at end of file
diff --git a/application/controllers/TemplatesController.php b/application/controllers/TemplatesController.php
new file mode 100644
index 00000000000..22b41bff56c
--- /dev/null
+++ b/application/controllers/TemplatesController.php
@@ -0,0 +1,11 @@
+ $button) {
+            $buttonHtml[] = self::btn(TbArray::popValue('type', $button, self::BUTTON_TYPE_SUBMIT), $label, $button);
+
+        }
+        return implode(' ', $buttonHtml);
+    }
+
+}
\ No newline at end of file
diff --git a/application/themes/default/assets/css/csvimport.css b/application/themes/default/assets/css/csvimport.css
index e22431c64f4..a73f66ed198 100644
--- a/application/themes/default/assets/css/csvimport.css
+++ b/application/themes/default/assets/css/csvimport.css
@@ -19,19 +19,24 @@
     height: auto;
 }
 
-.memoryUsage {
+.resourceUsage {
     height: 100px;
     display: flex;
     border: 1px solid black;
     align-items: flex-end;
 }
 
-.memoryUsage > div {
+.resourceUsage > div {
     display: flex;
-    border: 1px solid #dddddd;
-    background-color: #11ff11;
 }
 
+.resourceUsage > div.time {
+    background-color: #681818;
+}
+
+.resourceUsage > div.memory {
+    background-color: #6ce26c;
+}
 #importForm.busy #participantForm, #importForm.busy #mapForm {
     display: none;
 }
@@ -41,4 +46,45 @@
 }
 #importForm.busy #progress {
     display: inherit;
-}
\ No newline at end of file
+}
+
+#importForm {
+    position: relative;
+
+}
+.overlay {
+    background-color: rgba(255,0,0,0.5);
+    top: 0;
+    bottom: 0;
+    position: absolute;
+    left: 0px;
+    right: 0px;
+    z-index: 10;
+}
+
+#importForm .overlay {
+    display: none;
+    text-align: center;
+    font-size: 2em;
+    font-weight: bold;
+    line-height: 4em;
+}
+
+#importForm.aborted .overlay {
+    display: inline-block;
+}
+
+#importForm.busy .busy,
+#importForm.stopped .stopped,
+#importForm .not-busy{
+    display: inline-block;;
+}
+
+#importForm .busy,
+#importForm .stopped,
+#importForm.busy .not-busy,
+#importForm.stopped .not-stopped{
+    display: none;
+}
+
+
diff --git a/application/views/participants/import.php b/application/views/participants/import.php
index 1d3af3ec05b..54c5b7cb96f 100644
--- a/application/views/participants/import.php
+++ b/application/views/participants/import.php
@@ -4,11 +4,13 @@
     $cs->registerCssFile(App()->theme->baseUrl . '/css/csvimport.css');
     echo TbHtml::beginFormTb(TbHtml::FORM_LAYOUT_VERTICAL, ['participants/import', 'step' => 'map'], 'post', ['enctype' => 'multipart/form-data', 'id' => 'importForm']);
 ?>
+

+    Please wait for the worker thread to end. This can take a while depending on your configured chunk size.
+

 

     

         clientScript->registerScriptFile(App()->params['bower-asset'] . '/papaparse/papaparse.js');
         echo TbHtml::fileFieldControlGroup('file', null, [
             'label' => gT("CSV File"),
@@ -37,28 +39,28 @@
             'controlWidthClass' => 'col-sm-6',
         ]);
 
-        echo TbHtml::numberFieldControlGroup('batchSize', 20000, [
+        echo TbHtml::numberFieldControlGroup('batchSize', 5000, [
             'label' => gT("Batch size for uploading"),
             'required' => true,
             'formLayout' => TbHtml::FORM_LAYOUT_HORIZONTAL,
             'labelWidthClass' => 'col-sm-6',
             'controlWidthClass' => 'col-sm-6',
-            'help' => gT("Bigger chunks will give less frequent status updates but have a (slightly) better performance."),
+//            'help' => gT("Bigger chunks will give less frequent status updates but have a (slightly) better performance."),
 
         ]);
-        echo TbHtml::numberFieldControlGroup('querySize', 20000, [
+        echo TbHtml::numberFieldControlGroup('querySize', 2500, [
             'label' => gT("Batch size for queries"),
             'required' => true,
             'formLayout' => TbHtml::FORM_LAYOUT_HORIZONTAL,
             'labelWidthClass' => 'col-sm-6',
             'controlWidthClass' => 'col-sm-6',
-            'help' => gT("Bigger batches will increase memory usage for better performance."),
+//            'help' => gT("Bigger batches will increase memory usage for better performance."),
         ]);
         echo TbHtml::numberFieldControlGroup('chunkSize', 1024*1024, [
             'label' => gT("Chunk size for reading"),
             'required' => true,
             'formLayout' => TbHtml::FORM_LAYOUT_HORIZONTAL,
-            'help' => gT("Bigger chunks will give less frequent status updates but have a (slightly) better performance."),
+//            'help' => gT("Bigger chunks will give less frequent status updates but have a (slightly) better performance."),
             'labelWidthClass' => 'col-sm-6',
             'controlWidthClass' => 'col-sm-6',
         ]);
@@ -94,15 +96,36 @@
 renderPartial('map'); ?>
 

     

-        

+        

          'primary'
-        ]);
-        echo TbHtml::button('Stop import', [
-            'id' => 'stop',
-            'color' => 'danger'
+
+        echo Html::buttonRow([
+            gT('Import participants') => [
+                'color' => 'primary',
+                'class' => 'not-busy'
+            ],
+            gT('Restart import') => [
+                'type' => Html::BUTTON_TYPE_SUBMIT,
+                'id' => 'restart',
+                'class' => 'stopped'
+
+            ],
+            gT('Reconfigure') => [
+                'id' => 'reconfigure',
+                'type' => Html::BUTTON_TYPE_HTML,
+                'class' => 'stopped',
+                'color' => 'primary'
+            ],
+
+            gT('Stop import') => [
+                'type' => Html::BUTTON_TYPE_HTML,
+                'color' => 'danger',
+                'id' => 'stop',
+                'class' => 'busy not-stopped'
+            ],
+
         ]);
+
         ?>
         

     

@@ -121,10 +144,10 @@
             'formLayout' => TbHtml::FORM_LAYOUT_HORIZONTAL
         ]);
         echo TbHtml::customControlGroup(TbHtml::tag('div', [
-            'id' => 'memory',
-            'class' => 'memoryUsage'
+            'id' => 'resourceUsage',
+            'class' => 'resourceUsage'
         ], ''), '', [
-            'label' => gT("PHP Memory usage (%)"),
+            'label' => gT("PHP Resource usage (memory / time)"),
             'formLayout' => TbHtml::FORM_LAYOUT_HORIZONTAL
         ]);
         ?>
@@ -152,16 +175,28 @@
     });
 
     $('#stop').on('click', function(e) {
+        console.log('aborted');
         $.ajaxq.abort('csvimport');
-        $('#importForm').addClass('aborted');
+        if ($('#readProgress').data('progress') < 100) {
+            $('#importForm').addClass('aborted');
+        }
+        $('#importForm').addClass('stopped')
+
     });
 
+    $('#reconfigure').on('click', function(e) {
+        $('#importForm').removeClass('busy').removeClass('stopped');
+    })
+
     function runImport() {
         // Get map.
         var map = {};
-        $('#importForm').addClass('busy');
+        $('#importForm').addClass('busy').removeClass('aborted').removeClass('stopped');
+        $('#resourceUsage').empty();
         $('#sendProgress').data('progress', 0);
+        $('#sendProgress').css('width', 0);
         $('#readProgress').css('width', 0);
+        $('body').animate({scrollTop: 0}, 'slow');
         var batchSize = $('#batchSize').val();
         $('.csvColumn').filter(function(i, elem) { return $(elem).find('input').val() != ''; }).each(function(i, elem) {
             map[$(elem).attr('data-column')] = $(elem).find('input').val();
@@ -171,14 +206,18 @@ function runImport() {
 
         var config = getConfig();
         var queue = [];
-        var sendData = function(data, i, progress) {
+        var sendData = function(data, progress) {
             var $progress = $('#sendProgress');
             $.ajaxq('csvimport', {
-                url: "createUrl('participants/import');?>",
+                url: "createUrl('participants/ajaximport');?>",
                 data: data,
                 method: 'post',
                 timeout: 0,
                 contentType: 'application/json',
+                error: function(jqXHR, textStatus, errorThrown) {
+                    $.ajaxq.abort('csvimport');
+
+                },
                 success: function(data) {
                     if (typeof progress == 'undefined') {
                         console.log('Done!');
@@ -186,9 +225,9 @@ function runImport() {
                     } else {
                         $progress.data('progress', $progress.data('progress') + 100 * progress);
                     }
-                    var memoryPercent = (data.memory * 100).toPrecision(2);
-                    $('
').css('height', memoryPercent).appendTo($('#memory'));
-                    $('#memory').children().css('width', ($progress.data('progress') / 100) * (100 / $('#memory').children().length) + '%');
+                    $('
').addClass('memory').css('height', (data.memory * 100).toPrecision(2)).appendTo($('#resourceUsage'));
+                    $('
').addClass('time').css('height', (data.time * 100).toPrecision(2)).appendTo($('#resourceUsage'));
+                    $('#resourceUsage').children().css('width', (100 / $('#resourceUsage').children().length) + '%');
 
                     $progress.css('width', $progress.data('progress') + '%');
                 }
@@ -201,9 +240,12 @@ function runImport() {
         config.chunk = function(result, reader) {
             if ($('#importForm').is('.aborted')) {
                 reader.abort();
+                $('#importForm').removeClass('aborted');
                 return;
             }
-            $('#readProgress').css('width', (result.meta.cursor / fileSize * 100) + '%');
+            var progress = (result.meta.cursor / fileSize * 100);
+            $('#readProgress').data('progress', progress).css('width', progress + '%');
+
             var rows = [];
             for (var i = 0; i < result.data.length; i++) {
                 var row = {};
@@ -223,7 +265,7 @@ function runImport() {
                         'YII_CSRF_TOKEN': $('input[name=YII_CSRF_TOKEN]').val()
                     });
                     rows = [];
-                    sendData(data, i, batchSize / result.data.length * (config.chunkSize / fileSize));
+                    sendData(data, batchSize / result.data.length * (config.chunkSize > fileSize ? 1 : config.chunkSize / fileSize));
                     i++;
                 }
             }
@@ -232,7 +274,7 @@ function runImport() {
                 'map' : map,
                 'YII_CSRF_TOKEN': $('input[name=YII_CSRF_TOKEN]').val()
             });
-            sendData(data, i, batchSize / result.meta.cursor * (config.chunkSize / fileSize));
+            sendData(data, rows.length / result.data.length * (config.chunkSize > fileSize ? 1 : config.chunkSize / fileSize));
         }