Item2174: Encode TML newlines and sequences of spaces when converting…

… to HTML, and encode the number of dashes in the TML for a horizontal rule.

I also cleaned up some of the translator tests - some had finaltml that was the same as the input tml.

git-svn-id: http://svn.foswiki.org/branches/Release01x01@10485 0b4bb1d4-4e5a-0410-9cc4-b2b747904278

WysiwygPlugin/data/System/WysiwygPlugin.txt

@@ -162,8 +162,6 @@ If %SYSTEMWEB%.TablePlugin is *not* enabled, then TML table cells containing onl
 
 ---+++ TinyMCEPlugin integration
 %STARTSECTION{"tinymceintegration"}%
-   * Foswikitask:Item2174 - WysiwygPlugin removes line breaks
-      * Work-around: Use the =* Set NOWYSIWYG = 1= setting to disable WYSIWYG on line-sensitive topics
    * Foswikitask:Item1396 - Anchors are not handled by WysiwygPlugin
    * Foswikitask:Item5955 - WysiwygPlugin fails to roundtrip tables with align="center", border attributes, etc.
       * Description: Sometimes tables will fail to be converted to TML syntax (will stay as HTML) because there are attributes on the table (such as alignment or border decorations) that WysiwygPlugin does not know how to preserve. If such attributes are necessary, please use [[VarTABLE]] instead.
@@ -190,6 +188,7 @@ Many thanks to the following sponsors for supporting this work:
 |  Release: | %$RELEASE% |
 |  Change History: | |
 |  ?           | Foswikitask:Item10048: Try to use Macros in the src URLs of images with titles |
+|  ?           | Foswikitask:Item2174: Protect line breaks within paragraphs in TML |
 |  04 Oct 2010 | Foswikitask:Item1391: Protect div and span tags with =style= attributes |
 |  28 Jun 2010 | Foswikitask:Item761, Foswikitask:Item2311, Foswikitask:Item5990, Foswikitask:Item9170: Fix conversion between character encodings. Any characters may be entered in the WYSIWYG editor, regardless of the site's encoding. Where possible, they are converted to the site encoding, otherwise they become entities. <br />\
                  Foswikitask:Item2254: Fix cursor-movement problems on Mozilla browsers (introduced by Foswikitask:Item1798)<br/>\

WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin/Constants.pm

@@ -15,8 +15,8 @@ our %ALWAYS_BLOCK = map { $_ => 1 }
   HR ISINDEX MENU NOFRAMES NOSCRIPT OL P PRE TABLE UL );
 our $ALWAYS_BLOCK_S = join( '|', keys %ALWAYS_BLOCK );
 
-our $STARTWW  = qr/^|(?<=[ \t\n\(\!])/om;
-our $ENDWW    = qr/$|(?=[ \t\n\,\.\;\:\!\?\)])/om;
+our $STARTWW  = qr/^|(?<=[ \t\n\(\!])|(?<=<p>)|(?<= <\/span>)/om;
+our $ENDWW    = qr/$|(?=[ \t\n\,\.\;\:\!\?\)])|(?=<\/p>)|(?=<span\b[^>]*> )/om;
 our $PROTOCOL = qr/^(file|ftp|gopher|https?|irc|news|nntp|telnet|mailto):/;
 
 # Colours with colour settings in DefaultPreferences.

WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin/HTML2TML/Node.pm

@@ -38,7 +38,7 @@ my %jqueryChiliClass = map { $_ => 1 }
 
 my %tml2htmlClass = map { $_ => 1 }
     qw( WYSIWYG_PROTECTED WYSIWYG_STICKY TMLverbatim WYSIWYG_LINK
-        TMLhtml );
+        TMLhtml WYSIWYG_HIDDENWHITESPACE );
 
 =pod
 
@@ -1513,7 +1513,15 @@ sub _handleHR {
 
     my ( $f, $kids ) = $this->_flatten($options);
     return ( $f, '<hr />' . $kids ) if ( $options & $WC::NO_BLOCK_TML );
-    return ( $f | $WC::BLOCK_TML, $WC::CHECKn . '---' . $WC::CHECKn . $kids );
+
+    my $dashes = 3;
+    if ( $this->{attrs}->{style} and
+         $this->{attrs}->{style} =~ s/\bnumdashes\s*:\s*(\d+)\b// ) {
+        $dashes = $1;
+        $dashes = 3 if $dashes < 3;
+        $dashes = 160 if $dashes > 160; # Filter out probably-bad data
+    }
+    return ( $f | $WC::BLOCK_TML, $WC::CHECKn . ('-' x $dashes) . $WC::CHECKn . $kids );
 }
 
 sub _handleHTML { return _flatten(@_); }
@@ -1678,6 +1686,46 @@ sub _handleSPAN {
         }
     }
 
+    if ( _removeClass( \%atts, 'WYSIWYG_HIDDENWHITESPACE' ) ) {
+        # This regular expression ensures the encoded whitespace is valid.
+        # The limit on the number of digits will ensure that the numbers are reasonable.
+        if ( $atts{style} and $atts{style} =~ s/\bencoded\s*:\s*(['"])((?:b|n|t\d{1,2}|s\d{1,3})+)\1;?// ) {
+            my $whitespace = $2;
+            #print STDERR "'$whitespace' -> ";
+            $whitespace =~ s/b/\\/g;
+            $whitespace =~ s/n/$WC::NBBR/g;
+            $whitespace =~ s/t(\d+)/'\t' x $1/ge;
+            $whitespace =~ s/s(\d+)/$WC::NBSP x $1/ge;
+            #print STDERR "'$whitespace'\n";
+            #require Data::Dumper;
+            my ( $f, $kids ) = $this->_flatten($options | $WC::KEEP_WS | $WC::KEEP_ENTITIES);
+            #die Data::Dumper::Dumper($kids); 
+            if ( $kids eq ' ' ) {
+                # The space was not changed
+                # So restore the encoded whitespace
+                return ( $f, $whitespace );
+            }
+            elsif ( length($kids) == 0 ) {
+                # The user deleted the space
+                # So return blank
+                return ( 0, '' );
+            }
+            #else {die "'".ord($kids)."'";}if(1){}
+            elsif ( 0 and ($kids eq '&nbsp;' or $kids eq chr(160)) ) { # SMELL: Firefox-specific
+                # This was probably inserted by Firefox after the user deleted the space.
+                # So return blank
+                return ( 0, '' );
+            }
+            else {
+                # The user entered some new text
+                # Return the combination.
+                # Assume that a leading space corresponds to the encoded whitespace
+                $kids =~ s/^ //;
+                return ( $f, $whitespace . $kids );
+            }
+        }
+    }
+
     # Remove all other (non foswiki) classes
     if ( defined $atts{class} && $atts{class} !~ /foswiki/ ) {
         delete $atts{class};

WysiwygPlugin/lib/Foswiki/Plugins/WysiwygPlugin/TML2HTML.pm

@@ -378,8 +378,8 @@ sub _getRenderedVersion {
 
     $text = $this->_takeOutCustomTags($text);
 
-    $text =~ s/\\\n/ /g;
     $text =~ s/\t/   /g;
+    $text =~ s/( +\\\n)/$this->_hideWhitespace($1)/ge;
 
     # Remove PRE to prevent TML interpretation of text inside it
     $text = $this->_liftOutBlocks( $text, 'pre', {} );
@@ -446,8 +446,7 @@ s/((^|(?<=[-*\s(]))$Foswiki::regex{linkProtocolPattern}:[^\s<>"]+[^\s*.,!?;:)<])
     $text =~ s/$TT0([$Foswiki::regex{mixedAlphaNum}]+;)/&$1/go;
 
     # Horizontal rule
-    my $hr = CGI::hr( { class => 'TMLhr' } );
-    $text =~ s/^---+$/$hr/gm;
+    $text =~ s/^(---+)$/_encodeHr($1)/gme;
 
     # Wrap tables with macros before or after them in a <div>,
     # together with the macros,
@@ -624,12 +623,14 @@ s/((^|(?<=[-*\s(]))$Foswiki::regex{linkProtocolPattern}:[^\s<>"]+[^\s*.,!?;:)<])
             $line =~ s/^(<li\Q$ot\E>)\s*$/$1&nbsp;/;
 
         }
-        elsif ( $inList && $line =~ /^[ \t]/ ) {
+        elsif ( $inList && $line =~ s/^([ \t]+)/$this->_hideWhitespace("\n$1")/e ) {
 
             # Extend text of previous list item by dropping through
+            $result[-1] .= $line;
+            $line = '';
 
         }
-        elsif ( $line eq $hr ) {
+        elsif ( $line =~ /^<hr class="TMLhr"/) {
             push( @result, '</p>' ) if $inParagraph;
             $inParagraph = 0;
         }
@@ -654,13 +655,25 @@ s/((^|(?<=[-*\s(]))$Foswiki::regex{linkProtocolPattern}:[^\s<>"]+[^\s*.,!?;:)<])
             # Other line
             $this->_addListItem( \@result, '', '', '' ) if $inList;
             $inList = 0;
+            if ( $inParagraph and @result and $result[-1] !~ /<p>$/ ) {
+                # This is the second (or later) line of a paragraph
+
+                my $whitespace = "\n";
+                if ($line =~ s/^(\s+)//) {
+                    $whitespace .= $1;
+                }
+                $line = $this->_hideWhitespace($whitespace) . $line;
+            }
             unless ( $inParagraph or $inDiv ) {
                 push( @result, '<p>' );
                 $inParagraph = 1;
             }
+            $line =~ s/(\s\s+)/$this->_hideWhitespace($1)/ge;
+            $result[-1] .= $line;
+            $line = '';
         }
 
-        push( @result, $line );
+        push( @result, $line ) if length($line) > 0;
     }
 
     if ($inTable) {
@@ -721,6 +734,34 @@ s/$WC::STARTWW(($Foswiki::regex{webNameRegex}\.)?$Foswiki::regex{wikiWordRegex}(
     return $text;
 }
 
+sub _encodeHr {
+    my $dashes = shift;
+    my $style = '';
+    if ( length($dashes) > 3 ) {
+        $style = ' style="{numdashes:' . length($dashes) . '}"';
+    }
+    return '<hr class="TMLhr"' . $style . ' />';
+}
+
+sub _hideWhitespace {
+    my $this = shift;
+    my $whitespace = shift;
+
+    $whitespace =~ s/\\/b/g;
+    $whitespace =~ s/\n/n/g;
+    $whitespace =~ s/(\t+)/'t' . length($1)/ge;
+    $whitespace =~ s/( +)/'s' . length($1)/ge;
+
+    return $this->_liftOutGeneral(
+        " ",
+        {
+            tag => 'span',
+            class => "WYSIWYG_HIDDENWHITESPACE",
+            params => "style=\"{encoded:'$whitespace'}\"",
+        }
+    );
+}
+
 sub _appendClassToTag {
     my $this = shift;
     my $tagWithAttrs = shift;

WysiwygPlugin/test/unit/WysiwygPlugin/BrowserEditorInterface.pm

@@ -10,12 +10,13 @@ use Scalar::Util;
 
 sub _DEBUG {0};
 
-my $editFrameLocator        = "css=iframe#topic_ifr";
-my $wikitextLocator         = "css=a#topic_hide";
-my $wysiwygLocator          = "css=input#topic_2WYSIWYG";
-my $editTextareaLocator     = "css=textarea#topic";
-my $editCancelButtonLocator = "css=input#cancel";
-my $editSaveButtonLocator   = "css=input#save";
+my $editFrameLocator                = "css=iframe#topic_ifr";
+my $wikitextLocator                 = "css=a#topic_hide";
+my $wysiwygLocator                  = "css=input#topic_2WYSIWYG";
+my $editTextareaLocator             = "css=textarea#topic";
+my $editCancelButtonLocator         = "css=input#cancel";
+my $editSaveButtonLocator           = "css=input#save";
+my $editSaveContinueButtonLocator   = "css=input#checkpoint";
 
 # This must match the text in foswiki_tiny.js
 my $waitForServerMessage = "Please wait... retrieving page from server.";
@@ -181,6 +182,25 @@ sub save {
     delete $this->{_editorModeForBrowser}->{ $this->{_test}->browserName() };
 }
 
+sub saveAndContinue {
+    my $this = shift;
+    print STDERR "BrowserEditorInterface::saveAndContinue()\n" if _DEBUG;
+
+    $this->{_test}->assert(0, "editor not open")
+      unless exists $this->{_editorModeForBrowser}
+          ->{ $this->{_test}->browserName() };
+
+    $this->selectTopFrame();
+    $this->{_test}->selenium->click_ok($editSaveContinueButtonLocator);
+
+# The editor can take a while to open, and has to do another server request to convert TML2HTML, so use a longer timeout
+    $this->{_test}->selenium->wait_for_element_present( $editFrameLocator,
+        2 * $this->{_test}->timeout() );
+    $this->{_test}->selenium->pause()
+      ;    # Breathe for a moment; let TMCE settle before doing anything else
+
+}
+
 sub selectWysiwygEditorFrame {
     my $this = shift;
     print STDERR "BrowserEditorInterface::selectWysiwygEditorFrame()\n" if _DEBUG;

WysiwygPlugin/test/unit/WysiwygPlugin/BrowserTranslatorTests.pm

@@ -476,7 +476,7 @@ sub verify_editSaveTopicWithUnnamedUnicodeEntity {
 
     $this->{editor}->init();
 
-    # Close the editor because this tests uses a different topic
+    # Close the editor because this test uses a different topic
     if ( $this->{editor}->editorMode() ) {
         $this->{editor}->cancelEdit();
     }
@@ -501,15 +501,26 @@ sub verify_editSaveTopicWithUnnamedUnicodeEntity {
         $topicName,
         "Before${testText}After\n");
     $topicObject->save();
+    $topicObject->finish();
+
+    # Reload the topic and note the topic date
+    $topicObject = Foswiki::Meta->new(
+        $this->{session},
+        $this->{test_web},
+        $topicName);
+    $topicObject->load();
+    my $topicinfo = $topicObject->get( 'TOPICINFO' );
+    my $dateBeforeSaveFromEditor = $topicinfo->{date};
+    $this->assert($dateBeforeSaveFromEditor, "Date from topic info before saving from editor");
+    $topicObject->finish();
 
     # Open the test topic in the wysiwyg editor
     $this->{editor}
       ->openWysiwygEditor( $this->{test_web}, $topicName );
 
-    # Write rubbish over the topic, which will be overwritten on save
-    $topicObject->text("Rubbish");
-    $topicObject->save();
-    undef $topicObject;
+    # Make sure the topic timestamp is different, 
+    # so that we can confirm that the save did write to the file
+    sleep(1);
 
     # Save from the editor
     $this->{editor}->save();
@@ -519,6 +530,13 @@ sub verify_editSaveTopicWithUnnamedUnicodeEntity {
         $this->{session},
         $this->{test_web},
         $topicName);
+    $topicObject->load();
+
+    # Make sure the topic really was saved
+    $topicinfo = $topicObject->get( 'TOPICINFO' );
+    my $dateAfterSaveFromEditor = $topicinfo->{date};
+    $this->assert($dateAfterSaveFromEditor, "Date from topic info after saving from editor");
+    $this->assert_num_not_equals($dateBeforeSaveFromEditor, $dateAfterSaveFromEditor);
 
     my $text = $topicObject->text();
 

WysiwygPlugin/test/unit/WysiwygPlugin/ExtendedTranslatorTests.pm

@@ -100,12 +100,14 @@ my $data = [
           . $protecton
           . '<customtag>'
           . $protectoff
-          . 'some > text'
+          . 'some >'
+          . TranslatorTests::encodedWhitespace('s2')
+          . 'text'
           . $protecton
           . '</customtag>'
           . $protectoff . '</p>',
         tml      => '<customtag>some >  text</customtag>',
-        finaltml => '<customtag>some &gt; text</customtag>',
+        finaltml => '<customtag>some &gt;  text</customtag>',
     },
     {
         exec  => $TML2HTML | $ROUNDTRIP,
@@ -120,12 +122,14 @@ my $data = [
           . $protecton
           . '&lt;customtag&gt;'
           . $protectoff
-          . 'some &gt; text'
+          . 'some &gt;'
+          . TranslatorTests::encodedWhitespace('s2')
+          . 'text'
           . $protecton
           . '&lt;/customtag&gt;'
           . $protectoff . '</p>',
         tml      => '<customtag>some >  text</customtag>',
-        finaltml => '<customtag>some &gt; text</customtag>',
+        finaltml => '<customtag>some &gt;  text</customtag>',
     },
     {
         exec  => $TML2HTML | $ROUNDTRIP,