org.netbeans.lib.lexer.test.LexerTestUtilities.assertNextTokenEquals()

Here are the examples of the java api org.netbeans.lib.lexer.test.LexerTestUtilities.assertNextTokenEquals() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

87 Examples 7

19 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testMethodReferenceOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc.&");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.MEMBER_POINTER, ".&");
}

19 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testElvisOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc?:");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.ELVIS_OPERATOR, "?:");
}

19 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testJavaFieldOverrideOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc.@");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.DOT, ".");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.AT, "@");
}

19 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

public void testBlockComment() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("/*" + "Testing block comment\n" + ".. is it work?\n" + "*/");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.BLOCK_COMMENT, "/*" + "Testing block comment\n" + ".. is it work?\n" + "*/");
}

19 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

public void testDoreplacedentation() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("/**" + "Some method description.\n" + "@param whatever\n" + "@return whaaat?" + "*/");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.BLOCK_COMMENT, "/**" + "Some method description.\n" + "@param whatever\n" + "@return whaaat?" + "*/");
}

18 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testRegexpFindOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("\"123\" =~ \"1\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"123\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.REGEX_FIND, "=~");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"1\"");
}

18 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

// Interestingly enough no token is comming from Groovy Lexer for Spread Java Field
// Instead of such expected behavior there are two tokens SPREAD_DOT + AT
public void testSpreadJavaFieldOverrideOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc*.@");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.SPREAD_DOT, "*.");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.AT, "@");
}

18 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testSafeNavigationOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc?.");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.OPTIONAL_DOT, "?.");
}

18 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

public void testCharInDeclaration() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("a=\'x\'");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "a");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\'x\'");
}

18 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

// Groovy supports special line comment started with #!
public void testLineComment_SH_comment() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("#! def =\"string\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LINE_COMMENT, "#! def =\"string\"");
}

18 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

public void testLineComment() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("// def =\"string\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LINE_COMMENT, "// def =\"string\"");
}

17 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testIdenreplacedyOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("0 <=> 1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.NUM_INT, "0");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COMPARE_TO, "<=>");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.NUM_INT, "1");
}

17 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testRegexpMatchOperator() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("\"1\" ==~ /\\d+/");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"1\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.REGEX_MATCH, "==~");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    // Not sure if Lexer is returning correct token here, shouldn't this be something like REGEXP_LITERAL ??
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "/\\d+/");
}

17 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

public void testStringInDeclaration() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("a = \"string\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "a");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"string\"");
}

17 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

public void testMultilineString() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("def text = \"\"\"\\\n" + "hello there\n" + "how are you?\n" + "\"\"\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LITERAL_def, "def");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "text");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"\"\"\\\n" + "hello there\n" + "how are you?\n" + "\"\"\"");
}

15 Source : EmbeddedTokenListTest.java
with Apache License 2.0
from apache

public void testEmbeddingPresence() throws Exception {
    Doreplacedent d = new PlainDoreplacedent();
    d.putProperty(Language.clreplaced, TestEmbeddingTokenId.language());
    d.insertString(0, " acnacn", null);
    TokenHierarchy<?> h = TokenHierarchy.get(d);
    ((AbstractDoreplacedent) d).readLock();
    try {
        TokenSequence<TestEmbeddingTokenId> ts = h.tokenSequence(TestEmbeddingTokenId.language());
        TokenSequence<?> inner;
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.TEXT, " ");
        inner = ts.embedded();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.A, "a");
        inner = ts.embedded();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.C, "c");
        inner = ts.embedded();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.N, "n");
        inner = ts.embedded();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.A, "a");
        inner = ts.embedded();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.C, "c");
        inner = ts.embedded();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.N, "n");
        inner = ts.embedded();
        replacedertEquals(1, TestEmbeddingTokenId.cEmbeddingQueryCount);
        replacedertEquals(2, TestEmbeddingTokenId.aEmbeddingQueryCount);
    } finally {
        ((AbstractDoreplacedent) d).readUnlock();
    }
}

15 Source : LexerOperatorsTest.java
with Apache License 2.0
from apache

public void testSpreadOperatorOnMap() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("foo(1:\"a\", *:x)");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "foo");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LPAREN, "(");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.NUM_INT, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COLON, ":");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"a\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COMMA, ",");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    // Not sure why Star + Colon, I would expected Spread_Map_Arg token instead
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STAR, "*");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COLON, ":");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "x");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.RPAREN, ")");
}

15 Source : GroovyLexerTest.java
with Apache License 2.0
from apache

public void testGString() {
    TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("def text = \"hello there ${name}, how are you?\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LITERAL_def, "def");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "text");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"hello there $");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LBRACE, "{");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "name");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.RBRACE, "}");
    LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, ", how are you?\"");
}

15 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testStringLiteral03() throws Exception {
    String text = "\"string with quote \\\" \"";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "\"string with quote \\\" \"");
}

15 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testStringLiteral02() throws Exception {
    String text = "'string in apostrophes'";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'string in apostrophes'");
}

15 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testStringLiteral04() throws Exception {
    String text = "\"string with slashes \\\\ \"";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "\"string with slashes \\\\ \"");
}

15 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testStringLiteral01() throws Exception {
    String text = "\"string in quotes\"";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "\"string in quotes\"");
}

15 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testStringLiteral05() throws Exception {
    String text = "'string with apostrophes \\''";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'string with apostrophes \\''");
}

14 Source : DocumentUpdateTest.java
with Apache License 2.0
from apache

public void testUpdate3() throws Exception {
    Doreplacedent d = new ModificationTextDoreplacedent();
    d.putProperty(Language.clreplaced, TestTokenId.language());
    d.insertString(0, "\"t\"", null);
    TokenHierarchy<?> h = TokenHierarchy.get(d);
    ((AbstractDoreplacedent) d).readLock();
    try {
        h.tokenSequence().tokenCount();
        TokenSequence<?> s = h.tokenSequence();
        replacedertTrue(s.moveNext());
        replacedertNotNull(s.embedded());
    } finally {
        ((AbstractDoreplacedent) d).readUnlock();
    }
    d.insertString(1, "\\", null);
    ((AbstractDoreplacedent) d).readLock();
    try {
        LexerTestUtilities.replacedertNextTokenEquals(h.tokenSequence(), TestTokenId.STRING_LITERAL, "\"\\t\"");
        TokenSequence<?> s = h.tokenSequence();
        replacedertTrue(s.moveNext());
        TokenSequence<?> e = s.embedded();
        replacedertNotNull(e);
        replacedertTrue(e.moveNext());
        replacedertEquals(e.token().id(), TestStringTokenId.TAB);
    } finally {
        ((AbstractDoreplacedent) d).readUnlock();
    }
}

14 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testLogicalOperator01() throws Exception {
    String text = "true && true";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.TRUE_KEYWORD, "true");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.AND_AND, "&&");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.TRUE_KEYWORD, "true");
}

14 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testIdentifiers01() throws Exception {
    String text = "session";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "session");
    text = "myBean.property";
    ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "myBean");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.DOT, ".");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "property");
}

14 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testRelationalOperator05() throws Exception {
    String text = "1 > 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.GT, ">");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

14 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testMinusOperator01() throws Exception {
    String text = "x -";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "x");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MINUS, "-");
}

14 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testLogicalOperator03() throws Exception {
    String text = "!true";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.NOT, "!");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.TRUE_KEYWORD, "true");
}

14 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testTKeyword01() throws Exception {
    String text = "Boolean.TRUE";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "Boolean");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.DOT, ".");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "TRUE");
}

14 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testMathOperator02() throws Exception {
    String text = "1 - 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MINUS, "-");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : TextmateLexerTest.java
with Apache License 2.0
from apache

public void testNETBEANS2430() throws Exception {
    FileObject grammar = FileUtil.createData(FileUtil.getConfigRoot(), "Editors/text/test/grammar.json");
    try (OutputStream out = grammar.getOutputStream();
        Writer w = new OutputStreamWriter(out)) {
        w.write("{ \"scopeName\": \"test\", " + " \"patterns\": [\n" + "  { \"name\": \"string.test\",\n" + "    \"begin\": \"x([^x]+)x\",\n" + "    \"end\": \"x\\\\1x\"\n" + "   },\n" + "  { \"name\": \"whitespace.test\",\n" + "    \"match\": \" +\"\n" + "   }\n" + "]}\n");
    }
    grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test");
    Doreplacedent doc = new PlainDoreplacedent();
    doc.insertString(0, " xaax xbbx\nxccx xaax ", null);
    doc.putProperty(Language.clreplaced, new LanguageHierarchyImpl("text/test", "test").language());
    TokenHierarchy hi = TokenHierarchy.get(doc);
    TokenSequence<?> ts = hi.tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
    replacedertTokenProperties(ts, "test", "whitespace.test");
    // do not fully lex
    doc.insertString(3, "a", null);
    ts = hi.tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
    replacedertTokenProperties(ts, "test", "whitespace.test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaaax");
    replacedertTokenProperties(ts, "test", "string.test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbx\n");
    replacedertTokenProperties(ts, "test", "string.test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xccx xaax \n");
    replacedertFalse(ts.moveNext());
}

13 Source : TextmateLexerTest.java
with Apache License 2.0
from apache

public void testUTF8() throws Exception {
    clearWorkDir();
    FileObject grammar = FileUtil.createData(FileUtil.getConfigRoot(), "Editors/text/test/grammar.json");
    try (OutputStream out = grammar.getOutputStream();
        Writer w = new OutputStreamWriter(out)) {
        w.write("{ \"scopeName\": \"test\", " + " \"patterns\": [\n" + "  { \"name\": \"ident.test\",\n" + "    \"match\": \"[^ ]+\"\n" + "   }\n" + "]}\n");
    }
    grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test");
    Doreplacedent doc = new PlainDoreplacedent();
    doc.insertString(0, " večerníček večerníček  ", null);
    doc.putProperty(Language.clreplaced, new LanguageHierarchyImpl("text/test", "test").language());
    TokenHierarchy<Doreplacedent> hi = TokenHierarchy.get(doc);
    TokenSequence<?> ts = hi.tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
    replacedertTokenProperties(ts, "test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "večerníček");
    replacedertTokenProperties(ts, "test", "ident.test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
    replacedertTokenProperties(ts, "test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "večerníček");
    replacedertTokenProperties(ts, "test", "ident.test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "  ");
    replacedertTokenProperties(ts, "test");
    LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "\n");
    replacedertFalse(ts.moveNext());
}

13 Source : EmbeddedTokenListTest.java
with Apache License 2.0
from apache

public void testUpdateStartOffset() throws Exception {
    Doreplacedent d = new PlainDoreplacedent();
    d.putProperty(Language.clreplaced, TestTokenId.language());
    d.insertString(0, "ident ident /** @see X */", null);
    TokenHierarchy<?> h = TokenHierarchy.get(d);
    ((AbstractDoreplacedent) d).readLock();
    try {
        TokenSequence<?> ts = h.tokenSequence();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
        replacedertEquals(0, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
        replacedertEquals(5, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
        replacedertEquals(6, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
        replacedertEquals(11, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.JAVADOC_COMMENT, "/** @see X */");
        replacedertEquals(12, ts.offset());
        TokenSequence<?> inner = ts.embedded();
        replacedertNotNull(inner);
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
        replacedertEquals(15, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.TAG, "@see");
        replacedertEquals(16, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
        replacedertEquals(20, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.IDENT, "X");
        replacedertEquals(21, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
        replacedertEquals(22, inner.offset());
    } finally {
        ((AbstractDoreplacedent) d).readUnlock();
    }
}

13 Source : EmbeddedTokenListTest.java
with Apache License 2.0
from apache

public void testSnapshots() throws Exception {
    Doreplacedent d = new PlainDoreplacedent();
    d.putProperty(Language.clreplaced, TestTokenId.language());
    d.insertString(0, "ident ident /** @see X */", null);
    TokenHierarchy<?> h = TokenHierarchy.get(d);
    ((AbstractDoreplacedent) d).readLock();
    try {
        TokenSequence<?> ts = h.tokenSequence();
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
        replacedertEquals(0, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
        replacedertEquals(5, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
        replacedertEquals(6, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
        replacedertEquals(11, ts.offset());
        LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.JAVADOC_COMMENT, "/** @see X */");
        replacedertEquals(12, ts.offset());
        TokenSequence<?> inner = ts.embedded();
        replacedertNotNull(inner);
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
        replacedertEquals(15, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.TAG, "@see");
        replacedertEquals(16, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
        replacedertEquals(20, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.IDENT, "X");
        replacedertEquals(21, inner.offset());
        LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
        replacedertEquals(22, inner.offset());
    } finally {
        ((AbstractDoreplacedent) d).readUnlock();
    }
// h = TokenHierarchy.get(d).createSnapshot();
// ts = h.tokenSequence();
// 
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.IDENTIFIER, "ident");
// replacedertEquals(0, ts.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.WHITESPACE, " ");
// replacedertEquals(5, ts.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.IDENTIFIER, "ident");
// replacedertEquals(6, ts.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.WHITESPACE, " ");
// replacedertEquals(11, ts.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.JAVADOC_COMMENT, "/** @see X */");
// replacedertEquals(12, ts.offset());
// 
// inner = ts.embedded();
// 
// replacedertNotNull(inner);
// 
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.OTHER_TEXT, " ");
// replacedertEquals(15, inner.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.TAG, "@see");
// replacedertEquals(16, inner.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.OTHER_TEXT, " ");
// replacedertEquals(20, inner.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.IDENT, "X");
// replacedertEquals(21, inner.offset());
// 
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.OTHER_TEXT, " ");
// replacedertEquals(22, inner.offset());
}

13 Source : HtmlLexerBatchTest.java
with Apache License 2.0
from apache

public void testJspTags() {
    String text = "<jsp:useBean name=\"pkg.myBean\"/><!--comment-->abc>def<tag attr=\"value\"></tag>";
    TokenHierarchy<?> hi = TokenHierarchy.create(text, HTMLTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN_SYMBOL, "<");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN, "jsp:useBean");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.WS, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.ARGUMENT, "name");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.OPERATOR, "=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.VALUE, "\"pkg.myBean\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE_SYMBOL, "/>");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.BLOCK_COMMENT, "<!--comment-->");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TEXT, "abc");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.CHARACTER, ">");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TEXT, "def");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN_SYMBOL, "<");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN, "tag");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.WS, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.ARGUMENT, "attr");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.OPERATOR, "=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.VALUE, "\"value\"");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE_SYMBOL, ">");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN_SYMBOL, "</");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE, "tag");
    LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE_SYMBOL, ">");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testLambdaOperator02() throws Exception {
    String text = "x ->";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "x");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.LAMBDA, "->");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testNeKeyword01() throws Exception {
    String text = "1 ne 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.NE_KEYWORD, "ne");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testDivKeyword01() throws Exception {
    String text = "1 div 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.DIV_KEYWORD, "div");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testNumberLiteral07() throws Exception {
    String text = "1.5f + 1.7F";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.FLOAT_LITERAL, "1.5f");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.PLUS, "+");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.FLOAT_LITERAL, "1.7F");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testNumberLiteral01() throws Exception {
    String text = "15 + 18";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "15");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.PLUS, "+");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "18");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testRelationalOperator03() throws Exception {
    String text = "1 < 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.LT, "<");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testEqualOperator01() throws Exception {
    String text = "A ==";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "A");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.EQ_EQ, "==");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testMathOperator03() throws Exception {
    String text = "1 * 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MUL, "*");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testGtKeyword01() throws Exception {
    String text = "1 gt 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.GT_KEYWORD, "gt");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testMathOperator05() throws Exception {
    String text = "1 % 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MOD, "%");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testConcatOperator01() throws Exception {
    String text = "'a' += 'b'";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'a'");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.CONCAT, "+=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'b'");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testEqKeyword01() throws Exception {
    String text = "1 eq 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.EQ_KEYWORD, "eq");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testRelationalOperator06() throws Exception {
    String text = "1 >= 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.GT_EQ, ">=");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

13 Source : ELLexerBatchTest.java
with Apache License 2.0
from apache

public void testMathOperator01() throws Exception {
    String text = "1 + 1";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.PLUS, "+");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}

See More Examples