Here are the examples of the java api org.netbeans.lib.lexer.test.LexerTestUtilities.assertNextTokenEquals() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
87 Examples
19
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testMethodReferenceOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc.&");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.MEMBER_POINTER, ".&");
}
19
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testElvisOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc?:");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.ELVIS_OPERATOR, "?:");
}
19
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testJavaFieldOverrideOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc.@");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.DOT, ".");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.AT, "@");
}
19
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testBlockComment() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("/*" + "Testing block comment\n" + ".. is it work?\n" + "*/");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.BLOCK_COMMENT, "/*" + "Testing block comment\n" + ".. is it work?\n" + "*/");
}
19
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testDoreplacedentation() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("/**" + "Some method description.\n" + "@param whatever\n" + "@return whaaat?" + "*/");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.BLOCK_COMMENT, "/**" + "Some method description.\n" + "@param whatever\n" + "@return whaaat?" + "*/");
}
18
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testRegexpFindOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("\"123\" =~ \"1\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"123\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.REGEX_FIND, "=~");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"1\"");
}
18
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
// Interestingly enough no token is comming from Groovy Lexer for Spread Java Field
// Instead of such expected behavior there are two tokens SPREAD_DOT + AT
public void testSpreadJavaFieldOverrideOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc*.@");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.SPREAD_DOT, "*.");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.AT, "@");
}
18
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testSafeNavigationOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("abc?.");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "abc");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.OPTIONAL_DOT, "?.");
}
18
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testCharInDeclaration() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("a=\'x\'");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "a");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\'x\'");
}
18
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
// Groovy supports special line comment started with #!
public void testLineComment_SH_comment() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("#! def =\"string\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LINE_COMMENT, "#! def =\"string\"");
}
18
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testLineComment() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("// def =\"string\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LINE_COMMENT, "// def =\"string\"");
}
17
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testIdenreplacedyOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("0 <=> 1");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.NUM_INT, "0");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COMPARE_TO, "<=>");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.NUM_INT, "1");
}
17
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testRegexpMatchOperator() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("\"1\" ==~ /\\d+/");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"1\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.REGEX_MATCH, "==~");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
// Not sure if Lexer is returning correct token here, shouldn't this be something like REGEXP_LITERAL ??
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "/\\d+/");
}
17
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testStringInDeclaration() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("a = \"string\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "a");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"string\"");
}
17
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testMultilineString() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("def text = \"\"\"\\\n" + "hello there\n" + "how are you?\n" + "\"\"\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LITERAL_def, "def");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "text");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"\"\"\\\n" + "hello there\n" + "how are you?\n" + "\"\"\"");
}
15
Source : EmbeddedTokenListTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testEmbeddingPresence() throws Exception {
Doreplacedent d = new PlainDoreplacedent();
d.putProperty(Language.clreplaced, TestEmbeddingTokenId.language());
d.insertString(0, " acnacn", null);
TokenHierarchy<?> h = TokenHierarchy.get(d);
((AbstractDoreplacedent) d).readLock();
try {
TokenSequence<TestEmbeddingTokenId> ts = h.tokenSequence(TestEmbeddingTokenId.language());
TokenSequence<?> inner;
LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.TEXT, " ");
inner = ts.embedded();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.A, "a");
inner = ts.embedded();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.C, "c");
inner = ts.embedded();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.N, "n");
inner = ts.embedded();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.A, "a");
inner = ts.embedded();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.C, "c");
inner = ts.embedded();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestEmbeddingTokenId.N, "n");
inner = ts.embedded();
replacedertEquals(1, TestEmbeddingTokenId.cEmbeddingQueryCount);
replacedertEquals(2, TestEmbeddingTokenId.aEmbeddingQueryCount);
} finally {
((AbstractDoreplacedent) d).readUnlock();
}
}
15
Source : LexerOperatorsTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testSpreadOperatorOnMap() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("foo(1:\"a\", *:x)");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "foo");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LPAREN, "(");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.NUM_INT, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COLON, ":");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"a\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COMMA, ",");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
// Not sure why Star + Colon, I would expected Spread_Map_Arg token instead
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STAR, "*");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.COLON, ":");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "x");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.RPAREN, ")");
}
15
Source : GroovyLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testGString() {
TokenSequence<GroovyTokenId> ts = createTokenSequenceFor("def text = \"hello there ${name}, how are you?\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LITERAL_def, "def");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "text");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.replacedIGN, "=");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, "\"hello there $");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.LBRACE, "{");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.IDENTIFIER, "name");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.RBRACE, "}");
LexerTestUtilities.replacedertNextTokenEquals(ts, GroovyTokenId.STRING_LITERAL, ", how are you?\"");
}
15
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testStringLiteral03() throws Exception {
String text = "\"string with quote \\\" \"";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "\"string with quote \\\" \"");
}
15
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testStringLiteral02() throws Exception {
String text = "'string in apostrophes'";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'string in apostrophes'");
}
15
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testStringLiteral04() throws Exception {
String text = "\"string with slashes \\\\ \"";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "\"string with slashes \\\\ \"");
}
15
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testStringLiteral01() throws Exception {
String text = "\"string in quotes\"";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "\"string in quotes\"");
}
15
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testStringLiteral05() throws Exception {
String text = "'string with apostrophes \\''";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'string with apostrophes \\''");
}
14
Source : DocumentUpdateTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testUpdate3() throws Exception {
Doreplacedent d = new ModificationTextDoreplacedent();
d.putProperty(Language.clreplaced, TestTokenId.language());
d.insertString(0, "\"t\"", null);
TokenHierarchy<?> h = TokenHierarchy.get(d);
((AbstractDoreplacedent) d).readLock();
try {
h.tokenSequence().tokenCount();
TokenSequence<?> s = h.tokenSequence();
replacedertTrue(s.moveNext());
replacedertNotNull(s.embedded());
} finally {
((AbstractDoreplacedent) d).readUnlock();
}
d.insertString(1, "\\", null);
((AbstractDoreplacedent) d).readLock();
try {
LexerTestUtilities.replacedertNextTokenEquals(h.tokenSequence(), TestTokenId.STRING_LITERAL, "\"\\t\"");
TokenSequence<?> s = h.tokenSequence();
replacedertTrue(s.moveNext());
TokenSequence<?> e = s.embedded();
replacedertNotNull(e);
replacedertTrue(e.moveNext());
replacedertEquals(e.token().id(), TestStringTokenId.TAB);
} finally {
((AbstractDoreplacedent) d).readUnlock();
}
}
14
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testLogicalOperator01() throws Exception {
String text = "true && true";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.TRUE_KEYWORD, "true");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.AND_AND, "&&");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.TRUE_KEYWORD, "true");
}
14
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testIdentifiers01() throws Exception {
String text = "session";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "session");
text = "myBean.property";
ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "myBean");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.DOT, ".");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "property");
}
14
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testRelationalOperator05() throws Exception {
String text = "1 > 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.GT, ">");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
14
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testMinusOperator01() throws Exception {
String text = "x -";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "x");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MINUS, "-");
}
14
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testLogicalOperator03() throws Exception {
String text = "!true";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.NOT, "!");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.TRUE_KEYWORD, "true");
}
14
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testTKeyword01() throws Exception {
String text = "Boolean.TRUE";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "Boolean");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.DOT, ".");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "TRUE");
}
14
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testMathOperator02() throws Exception {
String text = "1 - 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MINUS, "-");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : TextmateLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testNETBEANS2430() throws Exception {
FileObject grammar = FileUtil.createData(FileUtil.getConfigRoot(), "Editors/text/test/grammar.json");
try (OutputStream out = grammar.getOutputStream();
Writer w = new OutputStreamWriter(out)) {
w.write("{ \"scopeName\": \"test\", " + " \"patterns\": [\n" + " { \"name\": \"string.test\",\n" + " \"begin\": \"x([^x]+)x\",\n" + " \"end\": \"x\\\\1x\"\n" + " },\n" + " { \"name\": \"whitespace.test\",\n" + " \"match\": \" +\"\n" + " }\n" + "]}\n");
}
grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test");
Doreplacedent doc = new PlainDoreplacedent();
doc.insertString(0, " xaax xbbx\nxccx xaax ", null);
doc.putProperty(Language.clreplaced, new LanguageHierarchyImpl("text/test", "test").language());
TokenHierarchy hi = TokenHierarchy.get(doc);
TokenSequence<?> ts = hi.tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
replacedertTokenProperties(ts, "test", "whitespace.test");
// do not fully lex
doc.insertString(3, "a", null);
ts = hi.tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
replacedertTokenProperties(ts, "test", "whitespace.test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaaax");
replacedertTokenProperties(ts, "test", "string.test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbx\n");
replacedertTokenProperties(ts, "test", "string.test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xccx xaax \n");
replacedertFalse(ts.moveNext());
}
13
Source : TextmateLexerTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testUTF8() throws Exception {
clearWorkDir();
FileObject grammar = FileUtil.createData(FileUtil.getConfigRoot(), "Editors/text/test/grammar.json");
try (OutputStream out = grammar.getOutputStream();
Writer w = new OutputStreamWriter(out)) {
w.write("{ \"scopeName\": \"test\", " + " \"patterns\": [\n" + " { \"name\": \"ident.test\",\n" + " \"match\": \"[^ ]+\"\n" + " }\n" + "]}\n");
}
grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test");
Doreplacedent doc = new PlainDoreplacedent();
doc.insertString(0, " večerníček večerníček ", null);
doc.putProperty(Language.clreplaced, new LanguageHierarchyImpl("text/test", "test").language());
TokenHierarchy<Doreplacedent> hi = TokenHierarchy.get(doc);
TokenSequence<?> ts = hi.tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
replacedertTokenProperties(ts, "test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "večerníček");
replacedertTokenProperties(ts, "test", "ident.test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
replacedertTokenProperties(ts, "test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "večerníček");
replacedertTokenProperties(ts, "test", "ident.test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
replacedertTokenProperties(ts, "test");
LexerTestUtilities.replacedertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "\n");
replacedertFalse(ts.moveNext());
}
13
Source : EmbeddedTokenListTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testUpdateStartOffset() throws Exception {
Doreplacedent d = new PlainDoreplacedent();
d.putProperty(Language.clreplaced, TestTokenId.language());
d.insertString(0, "ident ident /** @see X */", null);
TokenHierarchy<?> h = TokenHierarchy.get(d);
((AbstractDoreplacedent) d).readLock();
try {
TokenSequence<?> ts = h.tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
replacedertEquals(0, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
replacedertEquals(5, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
replacedertEquals(6, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
replacedertEquals(11, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.JAVADOC_COMMENT, "/** @see X */");
replacedertEquals(12, ts.offset());
TokenSequence<?> inner = ts.embedded();
replacedertNotNull(inner);
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
replacedertEquals(15, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.TAG, "@see");
replacedertEquals(16, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
replacedertEquals(20, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.IDENT, "X");
replacedertEquals(21, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
replacedertEquals(22, inner.offset());
} finally {
((AbstractDoreplacedent) d).readUnlock();
}
}
13
Source : EmbeddedTokenListTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testSnapshots() throws Exception {
Doreplacedent d = new PlainDoreplacedent();
d.putProperty(Language.clreplaced, TestTokenId.language());
d.insertString(0, "ident ident /** @see X */", null);
TokenHierarchy<?> h = TokenHierarchy.get(d);
((AbstractDoreplacedent) d).readLock();
try {
TokenSequence<?> ts = h.tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
replacedertEquals(0, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
replacedertEquals(5, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.IDENTIFIER, "ident");
replacedertEquals(6, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.WHITESPACE, " ");
replacedertEquals(11, ts.offset());
LexerTestUtilities.replacedertNextTokenEquals(ts, TestTokenId.JAVADOC_COMMENT, "/** @see X */");
replacedertEquals(12, ts.offset());
TokenSequence<?> inner = ts.embedded();
replacedertNotNull(inner);
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
replacedertEquals(15, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.TAG, "@see");
replacedertEquals(16, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
replacedertEquals(20, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.IDENT, "X");
replacedertEquals(21, inner.offset());
LexerTestUtilities.replacedertNextTokenEquals(inner, TestJavadocTokenId.OTHER_TEXT, " ");
replacedertEquals(22, inner.offset());
} finally {
((AbstractDoreplacedent) d).readUnlock();
}
// h = TokenHierarchy.get(d).createSnapshot();
// ts = h.tokenSequence();
//
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.IDENTIFIER, "ident");
// replacedertEquals(0, ts.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.WHITESPACE, " ");
// replacedertEquals(5, ts.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.IDENTIFIER, "ident");
// replacedertEquals(6, ts.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.WHITESPACE, " ");
// replacedertEquals(11, ts.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(ts,TestTokenId.JAVADOC_COMMENT, "/** @see X */");
// replacedertEquals(12, ts.offset());
//
// inner = ts.embedded();
//
// replacedertNotNull(inner);
//
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.OTHER_TEXT, " ");
// replacedertEquals(15, inner.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.TAG, "@see");
// replacedertEquals(16, inner.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.OTHER_TEXT, " ");
// replacedertEquals(20, inner.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.IDENT, "X");
// replacedertEquals(21, inner.offset());
//
// LexerTestUtilities.replacedertNextTokenEquals(inner,TestJavadocTokenId.OTHER_TEXT, " ");
// replacedertEquals(22, inner.offset());
}
13
Source : HtmlLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testJspTags() {
String text = "<jsp:useBean name=\"pkg.myBean\"/><!--comment-->abc>def<tag attr=\"value\"></tag>";
TokenHierarchy<?> hi = TokenHierarchy.create(text, HTMLTokenId.language());
TokenSequence<?> ts = hi.tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN_SYMBOL, "<");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN, "jsp:useBean");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.WS, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.ARGUMENT, "name");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.OPERATOR, "=");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.VALUE, "\"pkg.myBean\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE_SYMBOL, "/>");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.BLOCK_COMMENT, "<!--comment-->");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TEXT, "abc");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.CHARACTER, ">");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TEXT, "def");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN_SYMBOL, "<");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN, "tag");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.WS, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.ARGUMENT, "attr");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.OPERATOR, "=");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.VALUE, "\"value\"");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE_SYMBOL, ">");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_OPEN_SYMBOL, "</");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE, "tag");
LexerTestUtilities.replacedertNextTokenEquals(ts, HTMLTokenId.TAG_CLOSE_SYMBOL, ">");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testLambdaOperator02() throws Exception {
String text = "x ->";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "x");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.LAMBDA, "->");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testNeKeyword01() throws Exception {
String text = "1 ne 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.NE_KEYWORD, "ne");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testDivKeyword01() throws Exception {
String text = "1 div 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.DIV_KEYWORD, "div");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testNumberLiteral07() throws Exception {
String text = "1.5f + 1.7F";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.FLOAT_LITERAL, "1.5f");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.PLUS, "+");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.FLOAT_LITERAL, "1.7F");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testNumberLiteral01() throws Exception {
String text = "15 + 18";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "15");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.PLUS, "+");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "18");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testRelationalOperator03() throws Exception {
String text = "1 < 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.LT, "<");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testEqualOperator01() throws Exception {
String text = "A ==";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "A");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.EQ_EQ, "==");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testMathOperator03() throws Exception {
String text = "1 * 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MUL, "*");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testGtKeyword01() throws Exception {
String text = "1 gt 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.GT_KEYWORD, "gt");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testMathOperator05() throws Exception {
String text = "1 % 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.MOD, "%");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testConcatOperator01() throws Exception {
String text = "'a' += 'b'";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'a'");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.CONCAT, "+=");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.STRING_LITERAL, "'b'");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testEqKeyword01() throws Exception {
String text = "1 eq 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.EQ_KEYWORD, "eq");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testRelationalOperator06() throws Exception {
String text = "1 >= 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.GT_EQ, ">=");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
13
Source : ELLexerBatchTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public void testMathOperator01() throws Exception {
String text = "1 + 1";
TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.PLUS, "+");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
LexerTestUtilities.replacedertNextTokenEquals(ts, ELTokenId.INT_LITERAL, "1");
}
See More Examples