Skip to content

Commit

Permalink
Some lexer improvement, mostly by avoiding QChar::isWhatever()
Browse files Browse the repository at this point in the history 
Change-Id: I330a45a2f611a594d9b0e9bb24ebef028654a8d8
Reviewed-on: http://codereview.qt.nokia.com/3756
Reviewed-by: Roberto Raggi <roberto.raggi@nokia.com>
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
  • Loading branch information
Martin Jones authored and Qt by Nokia committed Aug 30, 2011
1 parent 6f7b045 commit e17010e
Show file tree
Hide file tree
Showing 6 changed files with 199 additions and 85 deletions.
2 changes: 1 addition & 1 deletion src/declarative/qml/ftw/ftw.pri
View file
Expand Up @@ -9,11 +9,11 @@ HEADERS += \
$$PWD/qdeclarativerefcount_p.h \
$$PWD/qdeclarativepool_p.h \
$$PWD/qfieldlist_p.h \
$$PWD/qdeclarativeutils_p.h

SOURCES += \
$$PWD/qintrusivelist.cpp \
$$PWD/qmetaobjectbuilder.cpp \
$$PWD/qhashedstring.cpp \
$$PWD/qdeclarativerefcount.cpp \
$$PWD/qdeclarativepool.cpp \

91 changes: 91 additions & 0 deletions src/declarative/qml/ftw/qdeclarativeutils_p.h
View file
@@ -0,0 +1,91 @@
/****************************************************************************
**
** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
** All rights reserved.
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** This file is part of the QtDeclarative module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** No Commercial Usage
** This file contains pre-release code and may not be distributed.
** You may use this file in accordance with the terms and conditions
** contained in the Technology Preview License Agreement accompanying
** this package.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain additional
** rights. These rights are described in the Nokia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** If you have questions regarding the use of this file, please contact
** Nokia at qt-info@nokia.com.
**
**
**
**
**
**
**
**
** $QT_END_LICENSE$
**
****************************************************************************/

#ifndef QDECLARATIVEUTIL_P_H
#define QDECLARATIVEUTIL_P_H

#include <QtCore/QString>

QT_BEGIN_NAMESPACE

namespace QDeclarativeUtils {

inline bool isUpper(const QChar &qc)
{
ushort c = qc.unicode();
return ((c >= 'A' && c <= 'Z') || (c > 127 && QChar::category(c) == QChar::Letter_Uppercase));
}

inline bool isLower(const QChar &qc)
{
ushort c = qc.unicode();
return ((c >= 'a' && c <= 'z') || (c > 127 && QChar::category(c) == QChar::Letter_Lowercase));
}

inline bool isLetter(const QChar &qc)
{
ushort c = qc.unicode();
return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c > 127 && qc.isLetter()));
}

inline bool isDigit(const QChar &qc)
{
ushort c = qc.unicode();
return ((c >= '0' && c <= '9') || (c > 127 && qc.isDigit()));
}

inline bool isLetterOrNumber(const QChar &qc)
{
ushort c = qc.unicode();
return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || (c > 127 && qc.isLetterOrNumber()));
}

inline bool isSpace(const QChar &qc)
{
ushort c = qc.unicode();
return (c == 0x20 || (c >= 0x09 && c <= 0x0D) || c == 0x85 || (c > 127 && qc.isSpace()));
}

} // namespace QDeclarative

QT_END_NAMESPACE

#endif // QDECLARATIVEUTIL_P_H
100 changes: 53 additions & 47 deletions src/declarative/qml/parser/qdeclarativejslexer.cpp
View file
Expand Up @@ -42,7 +42,9 @@
#include "qdeclarativejslexer_p.h"
#include "qdeclarativejsengine_p.h"
#include "qdeclarativejsnodepool_p.h"
#include <private/qdeclarativeutils_p.h>
#include <QtCore/QCoreApplication>
#include <QtCore/QVarLengthArray>
#include <QtCore/QDebug>

QT_BEGIN_NAMESPACE
Expand Down Expand Up @@ -126,6 +128,7 @@ void Lexer::setCode(const QString &code, int lineno)

_code = code;
_tokenText.clear();
_tokenText.reserve(1024);
_errorMessage.clear();
_tokenSpell = QStringRef();

Expand Down Expand Up @@ -275,7 +278,7 @@ int Lexer::scanToken()
_validTokenText = false;
_tokenLinePtr = _lastLinePtr;

while (_char.isSpace()) {
while (QDeclarativeUtils::isSpace(_char)) {
if (_char == QLatin1Char('\n')) {
_tokenLinePtr = _codePtr;

Expand Down Expand Up @@ -404,43 +407,44 @@ int Lexer::scanToken()
return T_DIVIDE_;

case '.':
if (_char.isDigit()) {
QByteArray chars;
chars.reserve(32);
if (QDeclarativeUtils::isDigit(_char)) {
QVarLengthArray<char,32> chars;

chars += ch.unicode(); // append the `.'
chars.append(ch.unicode()); // append the `.'

while (_char.isDigit()) {
chars += _char.unicode();
while (QDeclarativeUtils::isDigit(_char)) {
chars.append(_char.unicode());
scanChar();
}

if (_char.toLower() == QLatin1Char('e')) {
if (_codePtr[0].isDigit() || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) &&
_codePtr[1].isDigit())) {
if (_char == QLatin1Char('e') || _char == QLatin1Char('E')) {
if (QDeclarativeUtils::isDigit(_codePtr[0]) || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) &&
QDeclarativeUtils::isDigit(_codePtr[1]))) {

chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume `e'

if (_char == QLatin1Char('+') || _char == QLatin1Char('-')) {
chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume the sign
}

while (_char.isDigit()) {
chars += _char.unicode();
while (QDeclarativeUtils::isDigit(_char)) {
chars.append(_char.unicode());
scanChar();
}
}
}

chars.append('\0');

const char *begin = chars.constData();
const char *end = 0;
bool ok = false;

_tokenValue = qstrtod(begin, &end, &ok);

if (end != chars.end()) {
if (end - begin != chars.size() - 1) {
_errorCode = IllegalExponentIndicator;
_errorMessage = QCoreApplication::translate("QDeclarativeParser", "Illegal syntax for exponential number");
return T_ERROR;
Expand Down Expand Up @@ -525,7 +529,7 @@ int Lexer::scanToken()
case '\'':
case '"': {
const QChar quote = ch;
_tokenText.clear();
_tokenText.resize(0);
_validTokenText = true;

bool multilineStringLiteral = false;
Expand Down Expand Up @@ -632,11 +636,11 @@ int Lexer::scanToken()
}

default:
if (ch.isLetter() || ch == QLatin1Char('$') || ch == QLatin1Char('_') || (ch == QLatin1Char('\\') && _char == QLatin1Char('u'))) {
if (QDeclarativeUtils::isLetter(ch) || ch == QLatin1Char('$') || ch == QLatin1Char('_') || (ch == QLatin1Char('\\') && _char == QLatin1Char('u'))) {
bool identifierWithEscapeChars = false;
if (ch == QLatin1Char('\\')) {
identifierWithEscapeChars = true;
_tokenText.clear();
_tokenText.resize(0);
bool ok = false;
_tokenText += decodeUnicodeEscapeCharacter(&ok);
_validTokenText = true;
Expand All @@ -647,15 +651,16 @@ int Lexer::scanToken()
}
}
while (true) {
if (_char.isLetterOrNumber() || _char == QLatin1Char('$') || _char == QLatin1Char('_')) {
if (QDeclarativeUtils::isLetterOrNumber(_char) || _char == QLatin1Char('$') || _char == QLatin1Char('_')) {
if (identifierWithEscapeChars)
_tokenText += _char;

scanChar();
} else if (_char == QLatin1Char('\\') && _codePtr[0] == QLatin1Char('u')) {
if (! identifierWithEscapeChars) {
identifierWithEscapeChars = true;
_tokenText = QString(_tokenStartPtr, _codePtr - _tokenStartPtr - 1);
_tokenText.resize(0);
_tokenText.insert(0, _tokenStartPtr, _codePtr - _tokenStartPtr - 1);
_validTokenText = true;
}

Expand Down Expand Up @@ -685,19 +690,18 @@ int Lexer::scanToken()
return kind;
}
}
} else if (ch.isDigit()) {
QByteArray chars;
chars.reserve(32);
chars += ch.unicode();
} else if (QDeclarativeUtils::isDigit(ch)) {
QVarLengthArray<char,32> chars;
chars.append(ch.unicode());

if (ch == QLatin1Char('0') && (_char == 'x' || _char == 'X')) {
// parse hex integer literal

chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume `x'

while (isHexDigit(_char)) {
chars += _char.unicode();
chars.append(_char.unicode());
scanChar();
}

Expand All @@ -706,64 +710,66 @@ int Lexer::scanToken()
}

// decimal integer literal
while (_char.isDigit()) {
chars += _char.unicode();
while (QDeclarativeUtils::isDigit(_char)) {
chars.append(_char.unicode());
scanChar(); // consume the digit
}

if (_char == QLatin1Char('.')) {
chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume `.'

while (_char.isDigit()) {
chars += _char.unicode();
while (QDeclarativeUtils::isDigit(_char)) {
chars.append(_char.unicode());
scanChar();
}

if (_char.toLower() == QLatin1Char('e')) {
if (_codePtr[0].isDigit() || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) &&
_codePtr[1].isDigit())) {
if (_char == QLatin1Char('e') || _char == QLatin1Char('E')) {
if (QDeclarativeUtils::isDigit(_codePtr[0]) || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) &&
QDeclarativeUtils::isDigit(_codePtr[1]))) {

chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume `e'

if (_char == QLatin1Char('+') || _char == QLatin1Char('-')) {
chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume the sign
}

while (_char.isDigit()) {
chars += _char.unicode();
while (QDeclarativeUtils::isDigit(_char)) {
chars.append(_char.unicode());
scanChar();
}
}
}
} else if (_char.toLower() == QLatin1Char('e')) {
if (_codePtr[0].isDigit() || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) &&
_codePtr[1].isDigit())) {
} else if (_char == QLatin1Char('e') || _char == QLatin1Char('E')) {
if (QDeclarativeUtils::isDigit(_codePtr[0]) || ((_codePtr[0] == QLatin1Char('+') || _codePtr[0] == QLatin1Char('-')) &&
QDeclarativeUtils::isDigit(_codePtr[1]))) {

chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume `e'

if (_char == QLatin1Char('+') || _char == QLatin1Char('-')) {
chars += _char.unicode();
chars.append(_char.unicode());
scanChar(); // consume the sign
}

while (_char.isDigit()) {
chars += _char.unicode();
while (QDeclarativeUtils::isDigit(_char)) {
chars.append(_char.unicode());
scanChar();
}
}
}

chars.append('\0');

const char *begin = chars.constData();
const char *end = 0;
bool ok = false;

_tokenValue = qstrtod(begin, &end, &ok);

if (end != chars.end()) {
if (end - begin != chars.size() - 1) {
_errorCode = IllegalExponentIndicator;
_errorMessage = QCoreApplication::translate("QDeclarativeParser", "Illegal syntax for exponential number");
return T_ERROR;
Expand All @@ -780,7 +786,7 @@ int Lexer::scanToken()

bool Lexer::scanRegExp(RegExpBodyPrefix prefix)
{
_tokenText.clear();
_tokenText.resize(0);
_validTokenText = true;
_patternFlags = 0;

Expand Down

0 comments on commit e17010e

Please sign in to comment.