Skip to content
This repository has been archived by the owner on Sep 6, 2021. It is now read-only.

Commit

Permalink
Merge pull request #1753 from adobe/nj/factor-token-utils
Browse files Browse the repository at this point in the history
Factor out generic token-navigation routines from HTMLUtils
  • Loading branch information
peterflynn committed Oct 3, 2012
2 parents 943433f + 0285579 commit 04bcf56
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 105 deletions.
121 changes: 16 additions & 105 deletions src/language/HTMLUtils.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,102 +28,13 @@
define(function (require, exports, module) {
"use strict";

var TokenUtils = require("utils/TokenUtils");

//constants
var TAG_NAME = "tagName",
ATTR_NAME = "attr.name",
ATTR_VALUE = "attr.value";

/**
* @private
* moves the current context backwards by one token
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} ctx
* @return {boolean} whether the context changed
*/
function _movePrevToken(ctx) {
if (ctx.pos.ch <= 0 || ctx.token.start <= 0) {
//move up a line
if (ctx.pos.line <= 0) {
return false; //at the top already
}
ctx.pos.line--;
ctx.pos.ch = ctx.editor.getLine(ctx.pos.line).length;
} else {
ctx.pos.ch = ctx.token.start;
}
ctx.token = ctx.editor.getTokenAt(ctx.pos);
return true;
}

/**
* @private
* moves the current context forward by one token
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} ctx
* @return {boolean} whether the context changed
*/
function _moveNextToken(ctx) {
var eol = ctx.editor.getLine(ctx.pos.line).length;
if (ctx.pos.ch >= eol || ctx.token.end >= eol) {
//move down a line
if (ctx.pos.line >= ctx.editor.lineCount() - 1) {
return false; //at the bottom
}
ctx.pos.line++;
ctx.pos.ch = 0;
} else {
ctx.pos.ch = ctx.token.end + 1;
}
ctx.token = ctx.editor.getTokenAt(ctx.pos);
return true;
}

/**
* @private
* moves the current context in the given direction, skipping any whitespace it hits
* @param {function} moveFxn the funciton to move the context
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} ctx
* @return {boolean} whether the context changed
*/
function _moveSkippingWhitespace(moveFxn, ctx) {
if (!moveFxn(ctx)) {
return false;
}
while (!ctx.token.className && ctx.token.string.trim().length === 0) {
if (!moveFxn(ctx)) {
return false;
}
}
return true;
}

/**
* @private
* creates a context object
* @param {CodeMirror} editor
* @param {{ch:{string}, line:{number}} pos
* @return {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}}
*/
function _getInitialContext(editor, pos) {
return {
"editor": editor,
"pos": pos,
"token": editor.getTokenAt(pos)
};
}

/**
* @private
* in the given context, get the character offset of pos from the start of the token
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} context
* @return {number}
*/
function _offsetInToken(ctx) {
var offset = ctx.pos.ch - ctx.token.start;
if (offset < 0) {
console.log("CodeHintUtils: _offsetInToken - Invalid context: the pos what not in the current token!");
}
return offset;
}

/**
* @private
* Sometimes as attr values are getting typed, if the quotes aren't balanced yet
Expand All @@ -136,7 +47,7 @@ define(function (require, exports, module) {
var attrValue = ctx.token.string,
startChar = attrValue.charAt(0),
endChar = attrValue.charAt(attrValue.length - 1),
offset = _offsetInToken(ctx),
offset = TokenUtils.offsetInToken(ctx),
foundEqualSign = false;

//If this is a fully quoted value, return the whole
Expand Down Expand Up @@ -251,12 +162,12 @@ define(function (require, exports, module) {
}

//Move to the prev token, and check if it's "="
if (!_moveSkippingWhitespace(_movePrevToken, ctx) || ctx.token.string !== "=") {
if (!TokenUtils.moveSkippingWhitespace(TokenUtils.movePrevToken, ctx) || ctx.token.string !== "=") {
return createTagInfo();
}

//Move to the prev token, and check if it's an attribute
if (!_moveSkippingWhitespace(_movePrevToken, ctx) || ctx.token.className !== "attribute") {
if (!TokenUtils.moveSkippingWhitespace(TokenUtils.movePrevToken, ctx) || ctx.token.className !== "attribute") {
return createTagInfo();
}

Expand All @@ -283,13 +194,13 @@ define(function (require, exports, module) {

var tagName = _extractTagName(ctx);
var attrName = ctx.token.string;
var offset = _offsetInToken(ctx);
var offset = TokenUtils.offsetInToken(ctx);

if (!_moveSkippingWhitespace(_moveNextToken, ctx) || ctx.token.string !== "=") {
if (!TokenUtils.moveSkippingWhitespace(TokenUtils.moveNextToken, ctx) || ctx.token.string !== "=") {
return createTagInfo(ATTR_NAME, offset, tagName, attrName);
}

if (!_moveSkippingWhitespace(_moveNextToken, ctx)) {
if (!TokenUtils.moveSkippingWhitespace(TokenUtils.moveNextToken, ctx)) {
return createTagInfo(ATTR_NAME, offset, tagName, attrName);
}
//this should be the attrvalue
Expand Down Expand Up @@ -320,9 +231,9 @@ define(function (require, exports, module) {
// the pos the caller passed in so we use extend to make a safe copy of it.
// This is what pass by value in c++ would do.
var pos = $.extend({}, constPos),
ctx = _getInitialContext(editor._codeMirror, pos),
ctx = TokenUtils.getInitialContext(editor._codeMirror, pos),
tempCtx = null,
offset = _offsetInToken(ctx),
offset = TokenUtils.offsetInToken(ctx),
tagInfo,
tokenType;

Expand Down Expand Up @@ -366,15 +277,15 @@ define(function (require, exports, module) {
// use it to scan backwards if we don't find an equal sign here.
// Comment out this block to fix issue #1510.
// if (testToken.string.length > 0 && testToken.string.charAt(0) !== ">") {
// tempCtx = _getInitialContext(editor._codeMirror, pos);
// if (_moveSkippingWhitespace(_moveNextToken, tempCtx) && tempCtx.token.string === "=") {
// tempCtx = TokenUtils.getInitialContext(editor._codeMirror, pos);
// if (TokenUtils.moveSkippingWhitespace(TokenUtils.moveNextToken, tempCtx) && tempCtx.token.string === "=") {
// // Return an empty tag info since we're between an atribute name and the equal sign.
// return createTagInfo();
// }
// }

// next, see what's before pos
if (!_movePrevToken(ctx)) {
if (!TokenUtils.movePrevToken(ctx)) {
return createTagInfo();
}

Expand Down Expand Up @@ -438,7 +349,7 @@ define(function (require, exports, module) {
if (ctx.token.string === "=") {
// We could be between the attr and the value
// Step back and check
if (!_moveSkippingWhitespace(_movePrevToken, ctx) || ctx.token.className !== "attribute") {
if (!TokenUtils.moveSkippingWhitespace(TokenUtils.movePrevToken, ctx) || ctx.token.className !== "attribute") {
return createTagInfo();
}

Expand Down Expand Up @@ -484,13 +395,13 @@ define(function (require, exports, module) {
*/
function findStyleBlocks(editor) {
// Start scanning from beginning of file
var ctx = _getInitialContext(editor._codeMirror, {line: 0, ch: 0});
var ctx = TokenUtils.getInitialContext(editor._codeMirror, {line: 0, ch: 0});

var styleBlocks = [];
var currentStyleBlock = null;
var inStyleBlock = false;

while (_moveNextToken(ctx)) {
while (TokenUtils.moveNextToken(ctx)) {
if (inStyleBlock) {
// Check for end of this <style> block
if (ctx.token.state.mode !== "css") {
Expand Down
128 changes: 128 additions & 0 deletions src/utils/TokenUtils.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/*
* Copyright (c) 2012 Adobe Systems Incorporated. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
*/


/*jslint vars: true, plusplus: true, devel: true, nomen: true, indent: 4, maxerr: 50 */
/*global define, $ */

/**
* Functions for iterating through tokens in the current editor buffer. Useful for doing
* light parsing that can rely purely on information gathered by the code coloring mechanism.
*/

define(function (require, exports, module) {
"use strict";

/**
* Creates a context object for the given editor and position, suitable for passing to the
* move functions.
* @param {CodeMirror} editor
* @param {{ch:{string}, line:{number}} pos
* @return {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}}
*/
function getInitialContext(editor, pos) {
return {
"editor": editor,
"pos": pos,
"token": editor.getTokenAt(pos)
};
}

/**
* Moves the given context backwards by one token.
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} ctx
* @return {boolean} whether the context changed
*/
function movePrevToken(ctx) {
if (ctx.pos.ch <= 0 || ctx.token.start <= 0) {
//move up a line
if (ctx.pos.line <= 0) {
return false; //at the top already
}
ctx.pos.line--;
ctx.pos.ch = ctx.editor.getLine(ctx.pos.line).length;
} else {
ctx.pos.ch = ctx.token.start;
}
ctx.token = ctx.editor.getTokenAt(ctx.pos);
return true;
}

/**
* Moves the given context forward by one token.
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} ctx
* @return {boolean} whether the context changed
*/
function moveNextToken(ctx) {
var eol = ctx.editor.getLine(ctx.pos.line).length;
if (ctx.pos.ch >= eol || ctx.token.end >= eol) {
//move down a line
if (ctx.pos.line >= ctx.editor.lineCount() - 1) {
return false; //at the bottom
}
ctx.pos.line++;
ctx.pos.ch = 0;
} else {
ctx.pos.ch = ctx.token.end + 1;
}
ctx.token = ctx.editor.getTokenAt(ctx.pos);
return true;
}

/**
* Moves the given context in the given direction, skipping any whitespace it hits.
* @param {function} moveFxn the funciton to move the context
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} ctx
* @return {boolean} whether the context changed
*/
function moveSkippingWhitespace(moveFxn, ctx) {
if (!moveFxn(ctx)) {
return false;
}
while (!ctx.token.className && ctx.token.string.trim().length === 0) {
if (!moveFxn(ctx)) {
return false;
}
}
return true;
}

/**
* In the given context, get the character offset of pos from the start of the token.
* @param {editor:{CodeMirror}, pos:{ch:{string}, line:{number}}, token:{object}} context
* @return {number}
*/
function offsetInToken(ctx) {
var offset = ctx.pos.ch - ctx.token.start;
if (offset < 0) {
console.log("CodeHintUtils: _offsetInToken - Invalid context: the pos what not in the current token!");
}
return offset;
}

exports.movePrevToken = movePrevToken;
exports.moveNextToken = moveNextToken;
exports.moveSkippingWhitespace = moveSkippingWhitespace;
exports.getInitialContext = getInitialContext;
exports.offsetInToken = offsetInToken;
});

0 comments on commit 04bcf56

Please sign in to comment.