Here are the examples of the java api class org.apache.lucene.document.Document taken from open source projects.
1. TestBackwardsCompatibility#addDoc()
Project: lucene-solr
File: TestBackwardsCompatibility.java
File: TestBackwardsCompatibility.java
private void addDoc(IndexWriter writer, int id) throws IOException { Document doc = new Document(); doc.add(new TextField("content", "aaa", Field.Store.NO)); doc.add(new StringField("id", Integer.toString(id), Field.Store.YES)); FieldType customType2 = new FieldType(TextField.TYPE_STORED); customType2.setStoreTermVectors(true); customType2.setStoreTermVectorPositions(true); customType2.setStoreTermVectorOffsets(true); doc.add(new Field("autf8", "Lu??ce??ne ? ab??cd", customType2)); doc.add(new Field("utf8", "Lu??ce??ne ? ab??cd", customType2)); doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2)); doc.add(new Field("fie?ld", "field with non-ascii name", customType2)); // add numeric fields, to test if flex preserves encoding doc.add(new LegacyIntField("trieInt", id, Field.Store.NO)); doc.add(new LegacyLongField("trieLong", (long) id, Field.Store.NO)); // add docvalues fields doc.add(new NumericDocValuesField("dvByte", (byte) id)); byte bytes[] = new byte[] { (byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id }; BytesRef ref = new BytesRef(bytes); doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref)); doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref)); doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref)); doc.add(new SortedDocValuesField("dvBytesSortedVar", ref)); doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref)); doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref)); doc.add(new DoubleDocValuesField("dvDouble", (double) id)); doc.add(new FloatDocValuesField("dvFloat", (float) id)); doc.add(new NumericDocValuesField("dvInt", id)); doc.add(new NumericDocValuesField("dvLong", id)); doc.add(new NumericDocValuesField("dvPacked", id)); doc.add(new NumericDocValuesField("dvShort", (short) id)); doc.add(new SortedSetDocValuesField("dvSortedSet", ref)); doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id)); doc.add(new IntPoint("intPoint1d", id)); doc.add(new IntPoint("intPoint2d", id, 2 * id)); doc.add(new FloatPoint("floatPoint1d", (float) id)); doc.add(new FloatPoint("floatPoint2d", (float) id, (float) 2 * id)); doc.add(new LongPoint("longPoint1d", id)); doc.add(new LongPoint("longPoint2d", id, 2 * id)); doc.add(new DoublePoint("doublePoint1d", (double) id)); doc.add(new DoublePoint("doublePoint2d", (double) id, (double) 2 * id)); doc.add(new BinaryPoint("binaryPoint1d", bytes)); doc.add(new BinaryPoint("binaryPoint2d", bytes, bytes)); // a field with both offsets and term vectors for a cross-check FieldType customType3 = new FieldType(TextField.TYPE_STORED); customType3.setStoreTermVectors(true); customType3.setStoreTermVectorPositions(true); customType3.setStoreTermVectorOffsets(true); customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3)); // a field that omits only positions FieldType customType4 = new FieldType(TextField.TYPE_STORED); customType4.setStoreTermVectors(true); customType4.setStoreTermVectorPositions(false); customType4.setStoreTermVectorOffsets(true); customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS); doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4)); // TODO: // index different norms types via similarity (we use a random one currently?!) // remove any analyzer randomness, explicitly add payloads for certain fields. writer.addDocument(doc); }
2. TestJoinUtil#testSimpleWithScoring()
Project: lucene-solr
File: TestJoinUtil.java
File: TestJoinUtil.java
public void testSimpleWithScoring() throws Exception { final String idField = "id"; final String toField = "movieId"; Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); // 0 Document doc = new Document(); doc.add(new TextField("description", "A random movie", Field.Store.NO)); doc.add(new TextField("name", "Movie 1", Field.Store.NO)); doc.add(new TextField(idField, "1", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("1"))); w.addDocument(doc); // 1 doc = new Document(); doc.add(new TextField("subtitle", "The first subtitle of this movie", Field.Store.NO)); doc.add(new TextField(idField, "2", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("2"))); doc.add(new TextField(toField, "1", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("1"))); w.addDocument(doc); // 2 doc = new Document(); doc.add(new TextField("subtitle", "random subtitle; random event movie", Field.Store.NO)); doc.add(new TextField(idField, "3", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("3"))); doc.add(new TextField(toField, "1", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("1"))); w.addDocument(doc); // 3 doc = new Document(); doc.add(new TextField("description", "A second random movie", Field.Store.NO)); doc.add(new TextField("name", "Movie 2", Field.Store.NO)); doc.add(new TextField(idField, "4", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("4"))); w.addDocument(doc); w.commit(); // 4 doc = new Document(); doc.add(new TextField("subtitle", "a very random event happened during christmas night", Field.Store.NO)); doc.add(new TextField(idField, "5", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("5"))); doc.add(new TextField(toField, "4", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("4"))); w.addDocument(doc); // 5 doc = new Document(); doc.add(new TextField("subtitle", "movie end movie test 123 test 123 random", Field.Store.NO)); doc.add(new TextField(idField, "6", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("6"))); doc.add(new TextField(toField, "4", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("4"))); w.addDocument(doc); IndexSearcher indexSearcher = new IndexSearcher(w.getReader()); w.close(); // Search for movie via subtitle Query joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("subtitle", "random")), indexSearcher, ScoreMode.Max); TopDocs result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(0, result.scoreDocs[0].doc); assertEquals(3, result.scoreDocs[1].doc); // Score mode max. joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("subtitle", "movie")), indexSearcher, ScoreMode.Max); result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(3, result.scoreDocs[0].doc); assertEquals(0, result.scoreDocs[1].doc); // Score mode total joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("subtitle", "movie")), indexSearcher, ScoreMode.Total); result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(0, result.scoreDocs[0].doc); assertEquals(3, result.scoreDocs[1].doc); //Score mode avg joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("subtitle", "movie")), indexSearcher, ScoreMode.Avg); result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(3, result.scoreDocs[0].doc); assertEquals(0, result.scoreDocs[1].doc); indexSearcher.getIndexReader().close(); dir.close(); }
3. TestJoinUtil#testSimple()
Project: lucene-solr
File: TestJoinUtil.java
File: TestJoinUtil.java
public void testSimple() throws Exception { final String idField = "id"; final String toField = "productId"; Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); // 0 Document doc = new Document(); doc.add(new TextField("description", "random text", Field.Store.NO)); doc.add(new TextField("name", "name1", Field.Store.NO)); doc.add(new TextField(idField, "1", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("1"))); w.addDocument(doc); // 1 doc = new Document(); doc.add(new TextField("price", "10.0", Field.Store.NO)); doc.add(new TextField(idField, "2", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("2"))); doc.add(new TextField(toField, "1", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("1"))); w.addDocument(doc); // 2 doc = new Document(); doc.add(new TextField("price", "20.0", Field.Store.NO)); doc.add(new TextField(idField, "3", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("3"))); doc.add(new TextField(toField, "1", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("1"))); w.addDocument(doc); // 3 doc = new Document(); doc.add(new TextField("description", "more random text", Field.Store.NO)); doc.add(new TextField("name", "name2", Field.Store.NO)); doc.add(new TextField(idField, "4", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("4"))); w.addDocument(doc); w.commit(); // 4 doc = new Document(); doc.add(new TextField("price", "10.0", Field.Store.NO)); doc.add(new TextField(idField, "5", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("5"))); doc.add(new TextField(toField, "4", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("4"))); w.addDocument(doc); // 5 doc = new Document(); doc.add(new TextField("price", "20.0", Field.Store.NO)); doc.add(new TextField(idField, "6", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("6"))); doc.add(new TextField(toField, "4", Field.Store.NO)); doc.add(new SortedDocValuesField(toField, new BytesRef("4"))); w.addDocument(doc); IndexSearcher indexSearcher = new IndexSearcher(w.getReader()); w.close(); // Search for product Query joinQuery = JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name2")), indexSearcher, ScoreMode.None); TopDocs result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(4, result.scoreDocs[0].doc); assertEquals(5, result.scoreDocs[1].doc); joinQuery = JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name1")), indexSearcher, ScoreMode.None); result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(1, result.scoreDocs[0].doc); assertEquals(2, result.scoreDocs[1].doc); // Search for offer joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("id", "5")), indexSearcher, ScoreMode.None); result = indexSearcher.search(joinQuery, 10); assertEquals(1, result.totalHits); assertEquals(3, result.scoreDocs[0].doc); indexSearcher.getIndexReader().close(); dir.close(); }
4. TestMemoryIndex#test2DPoints()
Project: lucene-solr
File: TestMemoryIndex.java
File: TestMemoryIndex.java
public void test2DPoints() throws Exception { Document doc = new Document(); doc.add(new IntPoint("ints", 0, -100)); doc.add(new IntPoint("ints", 20, 20)); doc.add(new IntPoint("ints", 100, -100)); doc.add(new LongPoint("longs", 0L, -100L)); doc.add(new LongPoint("longs", 20L, 20L)); doc.add(new LongPoint("longs", 100L, -100L)); doc.add(new FloatPoint("floats", 0F, -100F)); doc.add(new FloatPoint("floats", 20F, 20F)); doc.add(new FloatPoint("floats", 100F, -100F)); doc.add(new DoublePoint("doubles", 0D, -100D)); doc.add(new DoublePoint("doubles", 20D, 20D)); doc.add(new DoublePoint("doubles", 100D, -100D)); MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer); IndexSearcher s = mi.createSearcher(); assertEquals(1, s.count(IntPoint.newRangeQuery("ints", new int[] { 10, 10 }, new int[] { 30, 30 }))); assertEquals(1, s.count(LongPoint.newRangeQuery("longs", new long[] { 10L, 10L }, new long[] { 30L, 30L }))); assertEquals(1, s.count(FloatPoint.newRangeQuery("floats", new float[] { 10F, 10F }, new float[] { 30F, 30F }))); assertEquals(1, s.count(DoublePoint.newRangeQuery("doubles", new double[] { 10D, 10D }, new double[] { 30D, 30D }))); }
5. ParentChildFieldDataTests#before()
Project: elasticsearch
File: ParentChildFieldDataTests.java
File: ParentChildFieldDataTests.java
@Before public void before() throws Exception { mapperService.merge(childType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), MapperService.MergeReason.MAPPING_UPDATE, false); mapperService.merge(grandChildType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(grandChildType, "_parent", "type=" + childType).string()), MapperService.MergeReason.MAPPING_UPDATE, false); Document d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); d.add(createJoinField(parentType, "1")); writer.addDocument(d); d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO)); d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); d.add(createJoinField(parentType, "1")); d.add(createJoinField(childType, "2")); writer.addDocument(d); writer.commit(); d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "3"), Field.Store.NO)); d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); d.add(createJoinField(parentType, "1")); d.add(createJoinField(childType, "3")); writer.addDocument(d); d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO)); d.add(createJoinField(parentType, "2")); writer.addDocument(d); d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "4"), Field.Store.NO)); d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO)); d.add(createJoinField(parentType, "2")); d.add(createJoinField(childType, "4")); writer.addDocument(d); d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "5"), Field.Store.NO)); d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); d.add(createJoinField(parentType, "1")); d.add(createJoinField(childType, "5")); writer.addDocument(d); writer.commit(); d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(grandChildType, "6"), Field.Store.NO)); d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO)); d.add(createJoinField(childType, "2")); writer.addDocument(d); d = new Document(); d.add(new StringField(UidFieldMapper.NAME, Uid.createUid("other-type", "1"), Field.Store.NO)); writer.addDocument(d); }
6. AllGroupHeadsCollectorTest#testBasic()
Project: lucene-solr
File: AllGroupHeadsCollectorTest.java
File: AllGroupHeadsCollectorTest.java
public void testBasic() throws Exception { final String groupField = "author"; Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); DocValuesType valueType = DocValuesType.SORTED; // 0 Document doc = new Document(); addGroupField(doc, groupField, "author1", valueType); doc.add(newTextField("content", "random text", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 1)); doc.add(new SortedDocValuesField("id_2", new BytesRef("1"))); w.addDocument(doc); // 1 doc = new Document(); addGroupField(doc, groupField, "author1", valueType); doc.add(newTextField("content", "some more random text blob", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 2)); doc.add(new SortedDocValuesField("id_2", new BytesRef("2"))); w.addDocument(doc); // 2 doc = new Document(); addGroupField(doc, groupField, "author1", valueType); doc.add(newTextField("content", "some more random textual data", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 3)); doc.add(new SortedDocValuesField("id_2", new BytesRef("3"))); w.addDocument(doc); // To ensure a second segment w.commit(); // 3 doc = new Document(); addGroupField(doc, groupField, "author2", valueType); doc.add(newTextField("content", "some random text", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 4)); doc.add(new SortedDocValuesField("id_2", new BytesRef("4"))); w.addDocument(doc); // 4 doc = new Document(); addGroupField(doc, groupField, "author3", valueType); doc.add(newTextField("content", "some more random text", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 5)); doc.add(new SortedDocValuesField("id_2", new BytesRef("5"))); w.addDocument(doc); // 5 doc = new Document(); addGroupField(doc, groupField, "author3", valueType); doc.add(newTextField("content", "random blob", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 6)); doc.add(new SortedDocValuesField("id_2", new BytesRef("6"))); w.addDocument(doc); // 6 -- no author field doc = new Document(); doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 6)); doc.add(new SortedDocValuesField("id_2", new BytesRef("6"))); w.addDocument(doc); // 7 -- no author field doc = new Document(); doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO)); doc.add(new NumericDocValuesField("id_1", 7)); doc.add(new SortedDocValuesField("id_2", new BytesRef("7"))); w.addDocument(doc); IndexReader reader = w.getReader(); IndexSearcher indexSearcher = newSearcher(reader); w.close(); int maxDoc = reader.maxDoc(); Sort sortWithinGroup = new Sort(new SortField("id_1", SortField.Type.INT, true)); AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup); indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector); assertTrue(arrayContains(new int[] { 2, 3, 5, 7 }, allGroupHeadsCollector.retrieveGroupHeads())); assertTrue(openBitSetContains(new int[] { 2, 3, 5, 7 }, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc)); allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup); indexSearcher.search(new TermQuery(new Term("content", "some")), allGroupHeadsCollector); assertTrue(arrayContains(new int[] { 2, 3, 4 }, allGroupHeadsCollector.retrieveGroupHeads())); assertTrue(openBitSetContains(new int[] { 2, 3, 4 }, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc)); allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup); indexSearcher.search(new TermQuery(new Term("content", "blob")), allGroupHeadsCollector); assertTrue(arrayContains(new int[] { 1, 5 }, allGroupHeadsCollector.retrieveGroupHeads())); assertTrue(openBitSetContains(new int[] { 1, 5 }, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc)); // STRING sort type triggers different implementation Sort sortWithinGroup2 = new Sort(new SortField("id_2", SortField.Type.STRING, true)); allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup2); indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector); assertTrue(arrayContains(new int[] { 2, 3, 5, 7 }, allGroupHeadsCollector.retrieveGroupHeads())); assertTrue(openBitSetContains(new int[] { 2, 3, 5, 7 }, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc)); Sort sortWithinGroup3 = new Sort(new SortField("id_2", SortField.Type.STRING, false)); allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup3); indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector); // 7 b/c higher doc id wins, even if order of field is in not in reverse. assertTrue(arrayContains(new int[] { 0, 3, 4, 6 }, allGroupHeadsCollector.retrieveGroupHeads())); assertTrue(openBitSetContains(new int[] { 0, 3, 4, 6 }, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc)); indexSearcher.getIndexReader().close(); dir.close(); }
7. LuceneSerializerTest#createDocument()
Project: querydsl
File: LuceneSerializerTest.java
File: LuceneSerializerTest.java
private Document createDocument() { Document doc = new Document(); doc.add(new Field("title", new StringReader("Jurassic Park"))); doc.add(new Field("author", new StringReader("Michael Crichton"))); doc.add(new Field("text", new StringReader("It's a UNIX system! I know this!"))); doc.add(new Field("rating", new StringReader("Good"))); doc.add(new Field("publisher", "", Store.YES, Index.ANALYZED)); doc.add(new IntField("year", 1990, Store.YES)); doc.add(new DoubleField("gross", 900.0, Store.YES)); doc.add(new LongField("longField", 1, Store.YES)); doc.add(new IntField("shortField", 1, Store.YES)); doc.add(new IntField("byteField", 1, Store.YES)); doc.add(new FloatField("floatField", 1, Store.YES)); return doc; }
8. LuceneSerializerTest#createDocument()
Project: querydsl
File: LuceneSerializerTest.java
File: LuceneSerializerTest.java
private Document createDocument() { Document doc = new Document(); doc.add(new Field("title", new StringReader("Jurassic Park"))); doc.add(new Field("author", new StringReader("Michael Crichton"))); doc.add(new Field("text", new StringReader("It's a UNIX system! I know this!"))); doc.add(new Field("rating", new StringReader("Good"))); doc.add(new Field("publisher", "", Store.YES, Index.ANALYZED)); doc.add(new NumericField("year", Store.YES, true).setIntValue(1990)); doc.add(new NumericField("gross", Store.YES, true).setDoubleValue(900.00)); doc.add(new NumericField("longField", Store.YES, true).setLongValue(1)); doc.add(new NumericField("shortField", Store.YES, true).setIntValue(1)); doc.add(new NumericField("byteField", Store.YES, true).setIntValue(1)); doc.add(new NumericField("floatField", Store.YES, true).setFloatValue(1)); return doc; }
9. TestParallelCompositeReader#getInvalidStructuredDir2()
Project: lucene-solr
File: TestParallelCompositeReader.java
File: TestParallelCompositeReader.java
// this dir has a different subreader structure (1,1,2); private Directory getInvalidStructuredDir2(Random random) throws IOException { Directory dir2 = newDirectory(); IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.INSTANCE)); Document d1 = new Document(); d1.add(newTextField("f3", "v1", Field.Store.YES)); d1.add(newTextField("f4", "v1", Field.Store.YES)); w2.addDocument(d1); w2.commit(); Document d2 = new Document(); d2.add(newTextField("f3", "v2", Field.Store.YES)); d2.add(newTextField("f4", "v2", Field.Store.YES)); w2.addDocument(d2); w2.commit(); Document d3 = new Document(); d3.add(newTextField("f3", "v3", Field.Store.YES)); d3.add(newTextField("f4", "v3", Field.Store.YES)); w2.addDocument(d3); Document d4 = new Document(); d4.add(newTextField("f3", "v4", Field.Store.YES)); d4.add(newTextField("f4", "v4", Field.Store.YES)); w2.addDocument(d4); w2.close(); return dir2; }
10. TestParallelCompositeReader#getDir2()
Project: lucene-solr
File: TestParallelCompositeReader.java
File: TestParallelCompositeReader.java
// subreader structure: (1,2,1) private Directory getDir2(Random random) throws IOException { Directory dir2 = newDirectory(); IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.INSTANCE)); Document d1 = new Document(); d1.add(newTextField("f3", "v1", Field.Store.YES)); d1.add(newTextField("f4", "v1", Field.Store.YES)); w2.addDocument(d1); w2.commit(); Document d2 = new Document(); d2.add(newTextField("f3", "v2", Field.Store.YES)); d2.add(newTextField("f4", "v2", Field.Store.YES)); w2.addDocument(d2); Document d3 = new Document(); d3.add(newTextField("f3", "v3", Field.Store.YES)); d3.add(newTextField("f4", "v3", Field.Store.YES)); w2.addDocument(d3); w2.commit(); Document d4 = new Document(); d4.add(newTextField("f3", "v4", Field.Store.YES)); d4.add(newTextField("f4", "v4", Field.Store.YES)); w2.addDocument(d4); w2.close(); return dir2; }
11. TestParallelCompositeReader#getDir1()
Project: lucene-solr
File: TestParallelCompositeReader.java
File: TestParallelCompositeReader.java
// subreader structure: (1,2,1) private Directory getDir1(Random random) throws IOException { Directory dir1 = newDirectory(); IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig(new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.INSTANCE)); Document d1 = new Document(); d1.add(newTextField("f1", "v1", Field.Store.YES)); d1.add(newTextField("f2", "v1", Field.Store.YES)); w1.addDocument(d1); w1.commit(); Document d2 = new Document(); d2.add(newTextField("f1", "v2", Field.Store.YES)); d2.add(newTextField("f2", "v2", Field.Store.YES)); w1.addDocument(d2); Document d3 = new Document(); d3.add(newTextField("f1", "v3", Field.Store.YES)); d3.add(newTextField("f2", "v3", Field.Store.YES)); w1.addDocument(d3); w1.commit(); Document d4 = new Document(); d4.add(newTextField("f1", "v4", Field.Store.YES)); d4.add(newTextField("f2", "v4", Field.Store.YES)); w1.addDocument(d4); w1.close(); return dir1; }
12. TestJoinUtil#testSimpleOrdinalsJoin()
Project: lucene-solr
File: TestJoinUtil.java
File: TestJoinUtil.java
public void testSimpleOrdinalsJoin() throws Exception { final String idField = "id"; final String productIdField = "productId"; // A field indicating to what type a document belongs, which is then used to distinques between documents during joining. final String typeField = "type"; // A single sorted doc values field that holds the join values for all document types. // Typically during indexing a schema will automatically create this field with the values final String joinField = idField + productIdField; Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)); // 0 Document doc = new Document(); doc.add(new TextField(idField, "1", Field.Store.NO)); doc.add(new TextField(typeField, "product", Field.Store.NO)); doc.add(new TextField("description", "random text", Field.Store.NO)); doc.add(new TextField("name", "name1", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("1"))); w.addDocument(doc); // 1 doc = new Document(); doc.add(new TextField(productIdField, "1", Field.Store.NO)); doc.add(new TextField(typeField, "price", Field.Store.NO)); doc.add(new TextField("price", "10.0", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("1"))); w.addDocument(doc); // 2 doc = new Document(); doc.add(new TextField(productIdField, "1", Field.Store.NO)); doc.add(new TextField(typeField, "price", Field.Store.NO)); doc.add(new TextField("price", "20.0", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("1"))); w.addDocument(doc); // 3 doc = new Document(); doc.add(new TextField(idField, "2", Field.Store.NO)); doc.add(new TextField(typeField, "product", Field.Store.NO)); doc.add(new TextField("description", "more random text", Field.Store.NO)); doc.add(new TextField("name", "name2", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("2"))); w.addDocument(doc); w.commit(); // 4 doc = new Document(); doc.add(new TextField(productIdField, "2", Field.Store.NO)); doc.add(new TextField(typeField, "price", Field.Store.NO)); doc.add(new TextField("price", "10.0", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("2"))); w.addDocument(doc); // 5 doc = new Document(); doc.add(new TextField(productIdField, "2", Field.Store.NO)); doc.add(new TextField(typeField, "price", Field.Store.NO)); doc.add(new TextField("price", "20.0", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("2"))); w.addDocument(doc); IndexSearcher indexSearcher = new IndexSearcher(w.getReader()); w.close(); IndexReader r = indexSearcher.getIndexReader(); SortedDocValues[] values = new SortedDocValues[r.leaves().size()]; for (int i = 0; i < values.length; i++) { LeafReader leafReader = r.leaves().get(i).reader(); values[i] = DocValues.getSorted(leafReader, joinField); } MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(r.getCoreCacheKey(), values, PackedInts.DEFAULT); Query toQuery = new TermQuery(new Term(typeField, "price")); Query fromQuery = new TermQuery(new Term("name", "name2")); // Search for product and return prices Query joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, ScoreMode.None, ordinalMap); TopDocs result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(4, result.scoreDocs[0].doc); assertEquals(5, result.scoreDocs[1].doc); fromQuery = new TermQuery(new Term("name", "name1")); joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, ScoreMode.None, ordinalMap); result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(1, result.scoreDocs[0].doc); assertEquals(2, result.scoreDocs[1].doc); // Search for prices and return products fromQuery = new TermQuery(new Term("price", "20.0")); toQuery = new TermQuery(new Term(typeField, "product")); joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, ScoreMode.None, ordinalMap); result = indexSearcher.search(joinQuery, 10); assertEquals(2, result.totalHits); assertEquals(0, result.scoreDocs[0].doc); assertEquals(3, result.scoreDocs[1].doc); indexSearcher.getIndexReader().close(); dir.close(); }
13. ClassificationTestBase#updateSampleIndex()
Project: lucene-solr
File: ClassificationTestBase.java
File: ClassificationTestBase.java
private void updateSampleIndex() throws Exception { String text; Document doc = new Document(); text = "Warren Bennis says John F. Kennedy grasped a key lesson about the presidency that few have followed."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Julian Zelizer says Bill Clinton is still trying to shape his party, years after the White House, while George W. Bush opts for a much more passive role."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Crossfire: Sen. Tim Scott passes on Sen. Lindsey Graham endorsement"; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Illinois becomes 16th state to allow same-sex marriage."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Apple is developing iPhones with curved-glass screens and enhanced sensors that detect different levels of pressure, according to a new report."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "technology", ft)); doc.add(new Field(booleanFieldName, "false", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "The Xbox One is Microsoft's first new gaming console in eight years. It's a quality piece of hardware but it's also noteworthy because Microsoft is using it to make a statement."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "technology", ft)); doc.add(new Field(booleanFieldName, "false", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Google says it will replace a Google Maps image after a California father complained it shows the body of his teen-age son, who was shot to death in 2009."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "technology", ft)); doc.add(new Field(booleanFieldName, "false", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "second unlabeled doc"; doc.add(new Field(textFieldName, text, ft)); indexWriter.addDocument(doc); indexWriter.commit(); }
14. ClassificationTestBase#getSampleIndex()
Project: lucene-solr
File: ClassificationTestBase.java
File: ClassificationTestBase.java
protected LeafReader getSampleIndex(Analyzer analyzer) throws IOException { indexWriter.close(); indexWriter = new RandomIndexWriter(random(), dir, newIndexWriterConfig(analyzer).setOpenMode(IndexWriterConfig.OpenMode.CREATE)); indexWriter.commit(); String text; Document doc = new Document(); text = "The traveling press secretary for Mitt Romney lost his cool and cursed at reporters " + "who attempted to ask questions of the Republican presidential candidate in a public plaza near the Tomb of " + "the Unknown Soldier in Warsaw Tuesday."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Mitt Romney seeks to assure Israel and Iran, as well as Jewish voters in the United" + " States, that he will be tougher against Iran's nuclear ambitions than President Barack Obama."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "And there's a threshold question that he has to answer for the American people and " + "that's whether he is prepared to be commander-in-chief,\" she continued. \"As we look to the past events, we " + "know that this raises some questions about his preparedness and we'll see how the rest of his trip goes.\""; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Still, when it comes to gun policy, many congressional Democrats have \"decided to " + "keep quiet and not go there,\" said Alan Lizotte, dean and professor at the State University of New York at " + "Albany's School of Criminal Justice."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "politics", ft)); doc.add(new Field(booleanFieldName, "true", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "Standing amongst the thousands of people at the state Capitol, Jorstad, director of " + "technology at the University of Wisconsin-La Crosse, documented the historic moment and shared it with the " + "world through the Internet."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "technology", ft)); doc.add(new Field(booleanFieldName, "false", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "So, about all those experts and analysts who've spent the past year or so saying " + "Facebook was going to make a phone. A new expert has stepped forward to say it's not going to happen."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "technology", ft)); doc.add(new Field(booleanFieldName, "false", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "More than 400 million people trust Google with their e-mail, and 50 million store files" + " in the cloud using the Dropbox service. People manage their bank accounts, pay bills, trade stocks and " + "generally transfer or store huge volumes of personal data online."; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(categoryFieldName, "technology", ft)); doc.add(new Field(booleanFieldName, "false", ft)); indexWriter.addDocument(doc); doc = new Document(); text = "unlabeled doc"; doc.add(new Field(textFieldName, text, ft)); indexWriter.addDocument(doc); indexWriter.commit(); indexWriter.forceMerge(1); return getOnlyLeafReader(indexWriter.getReader()); }
15. TestParallelLeafReader#single()
Project: lucene-solr
File: TestParallelLeafReader.java
File: TestParallelLeafReader.java
// Fields 1-4 indexed together: private IndexSearcher single(Random random) throws IOException { dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random))); Document d1 = new Document(); d1.add(newTextField("f1", "v1", Field.Store.YES)); d1.add(newTextField("f2", "v1", Field.Store.YES)); d1.add(newTextField("f3", "v1", Field.Store.YES)); d1.add(newTextField("f4", "v1", Field.Store.YES)); w.addDocument(d1); Document d2 = new Document(); d2.add(newTextField("f1", "v2", Field.Store.YES)); d2.add(newTextField("f2", "v2", Field.Store.YES)); d2.add(newTextField("f3", "v2", Field.Store.YES)); d2.add(newTextField("f4", "v2", Field.Store.YES)); w.addDocument(d2); w.close(); DirectoryReader ir = DirectoryReader.open(dir); return newSearcher(ir); }
16. GeoFieldDataTests#fillMultiValueAllSet()
Project: elasticsearch
File: GeoFieldDataTests.java
File: GeoFieldDataTests.java
@Override protected void fillMultiValueAllSet() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "2", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "3", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); }
17. TestJoinUtil#testInsideBooleanQuery()
Project: lucene-solr
File: TestJoinUtil.java
File: TestJoinUtil.java
/** LUCENE-5487: verify a join query inside a SHOULD BQ * will still use the join query's optimized BulkScorers */ public void testInsideBooleanQuery() throws Exception { final String idField = "id"; final String toField = "productId"; Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); // 0 Document doc = new Document(); doc.add(new TextField("description", "random text", Field.Store.NO)); doc.add(new TextField("name", "name1", Field.Store.NO)); doc.add(new TextField(idField, "7", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("7"))); w.addDocument(doc); // 1 doc = new Document(); doc.add(new TextField("price", "10.0", Field.Store.NO)); doc.add(new TextField(idField, "2", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("2"))); doc.add(new TextField(toField, "7", Field.Store.NO)); w.addDocument(doc); // 2 doc = new Document(); doc.add(new TextField("price", "20.0", Field.Store.NO)); doc.add(new TextField(idField, "3", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("3"))); doc.add(new TextField(toField, "7", Field.Store.NO)); w.addDocument(doc); // 3 doc = new Document(); doc.add(new TextField("description", "more random text", Field.Store.NO)); doc.add(new TextField("name", "name2", Field.Store.NO)); doc.add(new TextField(idField, "0", Field.Store.NO)); w.addDocument(doc); w.commit(); // 4 doc = new Document(); doc.add(new TextField("price", "10.0", Field.Store.NO)); doc.add(new TextField(idField, "5", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("5"))); doc.add(new TextField(toField, "0", Field.Store.NO)); w.addDocument(doc); // 5 doc = new Document(); doc.add(new TextField("price", "20.0", Field.Store.NO)); doc.add(new TextField(idField, "6", Field.Store.NO)); doc.add(new SortedDocValuesField(idField, new BytesRef("6"))); doc.add(new TextField(toField, "0", Field.Store.NO)); w.addDocument(doc); w.forceMerge(1); IndexSearcher indexSearcher = new IndexSearcher(w.getReader()); w.close(); // Search for product Query joinQuery = JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("description", "random")), indexSearcher, ScoreMode.Avg); BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(joinQuery, BooleanClause.Occur.SHOULD); bq.add(new TermQuery(new Term("id", "3")), BooleanClause.Occur.SHOULD); indexSearcher.search(bq.build(), new SimpleCollector() { boolean sawFive; @Override public void collect(int docID) { // appending to head of linked list): if (docID == 5) { sawFive = true; } else if (docID == 1) { assertFalse("optimized bulkScorer was not used for join query embedded in boolean query!", sawFive); } } @Override public boolean needsScores() { return false; } }); indexSearcher.getIndexReader().close(); dir.close(); }
18. TestJoinUtil#testOrdinalsJoinExplainNoMatches()
Project: lucene-solr
File: TestJoinUtil.java
File: TestJoinUtil.java
public void testOrdinalsJoinExplainNoMatches() throws Exception { final String idField = "id"; final String productIdField = "productId"; // A field indicating to what type a document belongs, which is then used to distinques between documents during joining. final String typeField = "type"; // A single sorted doc values field that holds the join values for all document types. // Typically during indexing a schema will automatically create this field with the values final String joinField = idField + productIdField; Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)); // 0 Document doc = new Document(); doc.add(new TextField(idField, "1", Field.Store.NO)); doc.add(new TextField(typeField, "product", Field.Store.NO)); doc.add(new TextField("description", "random text", Field.Store.NO)); doc.add(new TextField("name", "name1", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("1"))); w.addDocument(doc); // 1 doc = new Document(); doc.add(new TextField(idField, "2", Field.Store.NO)); doc.add(new TextField(typeField, "product", Field.Store.NO)); doc.add(new TextField("description", "random text", Field.Store.NO)); doc.add(new TextField("name", "name2", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("2"))); w.addDocument(doc); // 2 doc = new Document(); doc.add(new TextField(productIdField, "1", Field.Store.NO)); doc.add(new TextField(typeField, "price", Field.Store.NO)); doc.add(new TextField("price", "10.0", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("1"))); w.addDocument(doc); // 3 doc = new Document(); doc.add(new TextField(productIdField, "2", Field.Store.NO)); doc.add(new TextField(typeField, "price", Field.Store.NO)); doc.add(new TextField("price", "20.0", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("1"))); w.addDocument(doc); if (random().nextBoolean()) { w.flush(); } // 4 doc = new Document(); doc.add(new TextField(productIdField, "3", Field.Store.NO)); doc.add(new TextField(typeField, "price", Field.Store.NO)); doc.add(new TextField("price", "5.0", Field.Store.NO)); doc.add(new SortedDocValuesField(joinField, new BytesRef("2"))); w.addDocument(doc); // 5 doc = new Document(); doc.add(new TextField("field", "value", Field.Store.NO)); w.addDocument(doc); IndexReader r = DirectoryReader.open(w); IndexSearcher indexSearcher = new IndexSearcher(r); SortedDocValues[] values = new SortedDocValues[r.leaves().size()]; for (int i = 0; i < values.length; i++) { LeafReader leafReader = r.leaves().get(i).reader(); values[i] = DocValues.getSorted(leafReader, joinField); } MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(r.getCoreCacheKey(), values, PackedInts.DEFAULT); Query toQuery = new TermQuery(new Term("price", "5.0")); Query fromQuery = new TermQuery(new Term("name", "name2")); for (ScoreMode scoreMode : ScoreMode.values()) { Query joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, scoreMode, ordinalMap); TopDocs result = indexSearcher.search(joinQuery, 10); assertEquals(1, result.totalHits); // doc with price: 5.0 assertEquals(4, result.scoreDocs[0].doc); Explanation explanation = indexSearcher.explain(joinQuery, 4); assertTrue(explanation.isMatch()); assertEquals(explanation.getDescription(), "A match, join value 2"); explanation = indexSearcher.explain(joinQuery, 3); assertFalse(explanation.isMatch()); assertEquals(explanation.getDescription(), "Not a match, join value 1"); explanation = indexSearcher.explain(joinQuery, 5); assertFalse(explanation.isMatch()); assertEquals(explanation.getDescription(), "Not a match"); } w.close(); indexSearcher.getIndexReader().close(); dir.close(); }
19. PhraseFilterTest#createReaderWithSampleDocuments()
Project: greplin-lucene-utils
File: PhraseFilterTest.java
File: PhraseFilterTest.java
private IndexReader createReaderWithSampleDocuments() throws IOException { IndexWriter w = createWriter(); Document doc1 = new Document(); doc1.add(new Field("f", "i love to say hello world to everyone", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("name", "one", Field.Store.NO, Field.Index.ANALYZED)); w.addDocument(doc1); Document doc2 = new Document(); doc2.add(new Field("f", "i hate to say world hello to anyone", Field.Store.YES, Field.Index.ANALYZED)); doc2.add(new Field("name", "two", Field.Store.NO, Field.Index.ANALYZED)); w.addDocument(doc2); Document doc3 = new Document(); doc3.add(new Field("f", "hello hello world world", Field.Store.YES, Field.Index.ANALYZED)); doc3.add(new Field("name", "three", Field.Store.NO, Field.Index.ANALYZED)); w.addDocument(doc3); return createReader(w); }
20. GeoFieldDataTests#fillSingleValueAllSet()
Project: elasticsearch
File: GeoFieldDataTests.java
File: GeoFieldDataTests.java
@Override protected void fillSingleValueAllSet() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "2", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "3", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); }
21. GeoFieldDataTests#fillMultiValueWithMissing()
Project: elasticsearch
File: GeoFieldDataTests.java
File: GeoFieldDataTests.java
@Override protected void fillMultiValueWithMissing() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); // missing d = new Document(); d.add(new StringField("_id", "2", Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "3", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); }
22. ExpressionAggregationFacetsExample#index()
Project: lucene-solr
File: ExpressionAggregationFacetsExample.java
File: ExpressionAggregationFacetsExample.java
/** Build the example index. */ private void index() throws IOException { IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE)); // Writes facet ords to a separate directory from the main index DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); Document doc = new Document(); doc.add(new TextField("c", "foo bar", Store.NO)); doc.add(new NumericDocValuesField("popularity", 5L)); doc.add(new FacetField("A", "B")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new TextField("c", "foo foo bar", Store.NO)); doc.add(new NumericDocValuesField("popularity", 3L)); doc.add(new FacetField("A", "C")); indexWriter.addDocument(config.build(taxoWriter, doc)); indexWriter.close(); taxoWriter.close(); }
23. GeoFieldDataTests#fillSingleValueWithMissing()
Project: elasticsearch
File: GeoFieldDataTests.java
File: GeoFieldDataTests.java
@Override protected void fillSingleValueWithMissing() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "2", Field.Store.NO)); //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING.... writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "3", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); }
24. FacetNotValuesTest#createDataTwo()
Project: bobo
File: FacetNotValuesTest.java
File: FacetNotValuesTest.java
public Document[] createDataTwo() { ArrayList<Document> dataList = new ArrayList<Document>(); String color = "red"; String ID = Integer.toString(10); Document d = new Document(); d.add(new StringField("id", ID, Field.Store.YES)); d.add(new StringField("color", color, Field.Store.YES)); d.add(new IntField("NUM", 10, Field.Store.YES)); dataList.add(d); color = "green"; ID = Integer.toString(11); d = new Document(); d.add(new StringField("id", ID, Field.Store.YES)); d.add(new StringField("color", color, Field.Store.YES)); d.add(new IntField("NUM", 11, Field.Store.YES)); dataList.add(d); return dataList.toArray(new Document[dataList.size()]); }
25. TermsForFieldTest#setUp()
Project: greplin-lucene-utils
File: TermsForFieldTest.java
File: TermsForFieldTest.java
@Before public void setUp() throws Exception { Directory d = new RAMDirectory(); IndexWriter w = new IndexWriter(d, new IndexWriterConfig(Version.LUCENE_32, new WhitespaceAnalyzer(Version.LUCENE_32))); Document doc1 = new Document(); doc1.add(new Field("stored", "1", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("stored", "2", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("notStored", "a", Field.Store.NO, Field.Index.ANALYZED)); w.addDocument(doc1); Document doc2 = new Document(); doc2.add(new Field("stored", "3", Field.Store.YES, Field.Index.ANALYZED)); doc2.add(new Field("notStored", "b", Field.Store.NO, Field.Index.ANALYZED)); doc2.add(new Field("noIndex", "?", Field.Store.YES, Field.Index.NO)); w.addDocument(doc2); w.close(); this.reader = IndexReader.open(d); }
26. FiltersTest#setUp()
Project: greplin-lucene-utils
File: FiltersTest.java
File: FiltersTest.java
@Before public void setUp() throws Exception { IndexWriter w = createWriter(); Document doc1 = new Document(); doc1.add(new Field("f", "dog", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("f", "cat", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("f", "rat", Field.Store.YES, Field.Index.ANALYZED)); w.addDocument(doc1); Document doc2 = new Document(); doc2.add(new Field("f", "dog", Field.Store.YES, Field.Index.ANALYZED)); doc2.add(new Field("f", "monkey", Field.Store.YES, Field.Index.ANALYZED)); doc2.add(new Field("f", "goose", Field.Store.YES, Field.Index.ANALYZED)); w.addDocument(doc2); this.reader = createReader(w); }
27. LuceneService#createDocument()
Project: gitblit
File: LuceneService.java
File: LuceneService.java
/** * Creates a Lucene document for a commit * * @param commit * @param tags * @return a Lucene document */ private Document createDocument(RevCommit commit, List<String> tags) { Document doc = new Document(); doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.commit.name(), StringField.TYPE_STORED)); doc.add(new Field(FIELD_COMMIT, commit.getName(), TextField.TYPE_STORED)); doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L, Resolution.MINUTE), StringField.TYPE_STORED)); doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), TextField.TYPE_STORED)); doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), TextField.TYPE_STORED)); doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), TextField.TYPE_STORED)); doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), TextField.TYPE_STORED)); if (!ArrayUtils.isEmpty(tags)) { doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), TextField.TYPE_STORED)); } return doc; }
28. ShingleAnalyzerWrapperTest#setUp()
Project: lucene-solr
File: ShingleAnalyzerWrapperTest.java
File: ShingleAnalyzerWrapperTest.java
/** * Set up a new index in RAM with three test phrases and the supplied Analyzer. * * @throws Exception if an error occurs with index writer or searcher */ @Override public void setUp() throws Exception { super.setUp(); analyzer = new ShingleAnalyzerWrapper(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), 2); directory = newDirectory(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(analyzer)); Document doc; doc = new Document(); doc.add(new TextField("content", "please divide this sentence into shingles", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); doc.add(new TextField("content", "just another test sentence", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); doc.add(new TextField("content", "a sentence which contains no test", Field.Store.YES)); writer.addDocument(doc); writer.close(); reader = DirectoryReader.open(directory); searcher = newSearcher(reader); }
29. AnalyzingInfixSuggester#buildDocument()
Project: lucene-solr
File: AnalyzingInfixSuggester.java
File: AnalyzingInfixSuggester.java
private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException { String textString = text.utf8ToString(); Document doc = new Document(); FieldType ft = getTextFieldType(); doc.add(new Field(TEXT_FIELD_NAME, textString, ft)); doc.add(new Field("textgrams", textString, ft)); doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO)); doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text)); doc.add(new NumericDocValuesField("weight", weight)); if (payload != null) { doc.add(new BinaryDocValuesField("payloads", payload)); } if (contexts != null) { for (BytesRef context : contexts) { doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO)); doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context)); } } return doc; }
30. LuceneQueryTest#createDocument()
Project: querydsl
File: LuceneQueryTest.java
File: LuceneQueryTest.java
private Document createDocument(final String docTitle, final String docAuthor, final String docText, final int docYear, final double docGross) { final Document doc = new Document(); doc.add(new Field("title", docTitle, Store.YES, Index.ANALYZED)); doc.add(new Field("author", docAuthor, Store.YES, Index.ANALYZED)); doc.add(new Field("text", docText, Store.YES, Index.ANALYZED)); doc.add(new NumericField("year", Store.YES, true).setIntValue(docYear)); doc.add(new NumericField("gross", Store.YES, true).setDoubleValue(docGross)); return doc; }
31. HasFieldFilterTest#setUp()
Project: greplin-lucene-utils
File: HasFieldFilterTest.java
File: HasFieldFilterTest.java
@Before public void setUp() throws Exception { IndexWriter w = createWriter(); Document doc1 = new Document(); doc1.add(new Field("field1", "1", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("field2", "2", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("field2", "2again", Field.Store.YES, Field.Index.ANALYZED)); w.addDocument(doc1); Document doc2 = new Document(); doc2.add(new Field("field2", "2", Field.Store.YES, Field.Index.ANALYZED)); w.addDocument(doc2); this.reader = createReader(w); }
32. GeoFieldDataTests#add2SingleValuedDocumentsAndDeleteOneOfThem()
Project: elasticsearch
File: GeoFieldDataTests.java
File: GeoFieldDataTests.java
@Override protected void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.YES)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "2", Field.Store.NO)); d.add(randomGeoPointField(FIELD_NAME, Field.Store.NO)); writer.addDocument(d); writer.commit(); writer.deleteDocuments(new Term("_id", "1")); }
33. AbstractGeoFieldDataTestCase#fillAllMissing()
Project: elasticsearch
File: AbstractGeoFieldDataTestCase.java
File: AbstractGeoFieldDataTestCase.java
@Override protected void fillAllMissing() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "2", Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "3", Field.Store.NO)); writer.addDocument(d); }
34. TestParallelLeafReader#getDir2()
Project: lucene-solr
File: TestParallelLeafReader.java
File: TestParallelLeafReader.java
private Directory getDir2(Random random) throws IOException { Directory dir2 = newDirectory(); IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random))); Document d3 = new Document(); d3.add(newTextField("f3", "v1", Field.Store.YES)); d3.add(newTextField("f4", "v1", Field.Store.YES)); w2.addDocument(d3); Document d4 = new Document(); d4.add(newTextField("f3", "v2", Field.Store.YES)); d4.add(newTextField("f4", "v2", Field.Store.YES)); w2.addDocument(d4); w2.forceMerge(1); w2.close(); return dir2; }
35. TestParallelLeafReader#getDir1()
Project: lucene-solr
File: TestParallelLeafReader.java
File: TestParallelLeafReader.java
private Directory getDir1(Random random) throws IOException { Directory dir1 = newDirectory(); IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig(new MockAnalyzer(random))); Document d1 = new Document(); d1.add(newTextField("f1", "v1", Field.Store.YES)); d1.add(newTextField("f2", "v1", Field.Store.YES)); w1.addDocument(d1); Document d2 = new Document(); d2.add(newTextField("f1", "v2", Field.Store.YES)); d2.add(newTextField("f2", "v2", Field.Store.YES)); w1.addDocument(d2); w1.forceMerge(1); w1.close(); return dir1; }
36. TestDirectoryReader#addDocumentWithTermVectorFields()
Project: lucene-solr
File: TestDirectoryReader.java
File: TestDirectoryReader.java
static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException { Document doc = new Document(); FieldType customType5 = new FieldType(TextField.TYPE_STORED); customType5.setStoreTermVectors(true); FieldType customType6 = new FieldType(TextField.TYPE_STORED); customType6.setStoreTermVectors(true); customType6.setStoreTermVectorOffsets(true); FieldType customType7 = new FieldType(TextField.TYPE_STORED); customType7.setStoreTermVectors(true); customType7.setStoreTermVectorPositions(true); FieldType customType8 = new FieldType(TextField.TYPE_STORED); customType8.setStoreTermVectors(true); customType8.setStoreTermVectorOffsets(true); customType8.setStoreTermVectorPositions(true); doc.add(newTextField("tvnot", "tvnot", Field.Store.YES)); doc.add(newField("termvector", "termvector", customType5)); doc.add(newField("tvoffset", "tvoffset", customType6)); doc.add(newField("tvposition", "tvposition", customType7)); doc.add(newField("tvpositionoffset", "tvpositionoffset", customType8)); writer.addDocument(doc); }
37. DocumentClassificationTestBase#getBatmanDocument()
Project: lucene-solr
File: DocumentClassificationTestBase.java
File: DocumentClassificationTestBase.java
protected Document getBatmanDocument() { Document doc = new Document(); String title = "Batman: Arkham Knight new adventures for the super hero across Gotham, the Dark Knight has returned!"; String title2 = "I am a second title !"; String text = "This game is the electronic version of the famous super hero adventures.It involves the interaction with the open world" + " of the city of Gotham. Finally the player will be able to have fun on its personal device." + " The three-dimensional images of the game are stunning, because it uses the Unreal Engine 3." + " The systems available are PS4, X-Box and personal computer." + " Will the simulate missile that is going to be fired, success ?\" +\n" + " Will this video game make the history" + " Help you favourite super hero to defeat all his enemies. The Dark Knight has returned !"; String author = "Rocksteady Studios"; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(titleFieldName, title, ft)); doc.add(new Field(titleFieldName, title2, ft)); doc.add(new Field(authorFieldName, author, ft)); doc.add(new Field(booleanFieldName, "false", ft)); return doc; }
38. TestFieldCacheSort#testEmptyStringVsNullStringSort()
Project: lucene-solr
File: TestFieldCacheSort.java
File: TestFieldCacheSort.java
public void testEmptyStringVsNullStringSort() throws Exception { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); Document doc = new Document(); doc.add(newStringField("f", "", Field.Store.NO)); doc.add(newStringField("t", "1", Field.Store.NO)); w.addDocument(doc); w.commit(); doc = new Document(); doc.add(newStringField("t", "1", Field.Store.NO)); w.addDocument(doc); IndexReader r = UninvertingReader.wrap(DirectoryReader.open(w), Collections.singletonMap("f", Type.SORTED)); w.close(); IndexSearcher s = newSearcher(r); TopDocs hits = s.search(new TermQuery(new Term("t", "1")), 10, new Sort(new SortField("f", SortField.Type.STRING))); assertEquals(2, hits.totalHits); // null sorts first assertEquals(1, hits.scoreDocs[0].doc); assertEquals(0, hits.scoreDocs[1].doc); TestUtil.checkReader(r); r.close(); dir.close(); }
39. TestPrefixCompletionQuery#testAnalyzerWithPreservePosAndNoSep()
Project: lucene-solr
File: TestPrefixCompletionQuery.java
File: TestPrefixCompletionQuery.java
@Test public void testAnalyzerWithPreservePosAndNoSep() throws Exception { Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, true); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(completionAnalyzer, "suggest_field_no_p_sep")); Document document = new Document(); document.add(new SuggestField("suggest_field_no_p_sep", "foobar", 7)); document.add(new SuggestField("suggest_field_no_p_sep", "foo bar", 8)); document.add(new SuggestField("suggest_field_no_p_sep", "the fo", 9)); document.add(new SuggestField("suggest_field_no_p_sep", "the foo bar", 10)); iw.addDocument(document); DirectoryReader reader = iw.getReader(); SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader); CompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field_no_p_sep", "fo")); // matches all 4 TopSuggestDocs suggest = indexSearcher.suggest(query, 4); assertSuggestions(suggest, new Entry("the foo bar", 10), new Entry("the fo", 9), new Entry("foo bar", 8), new Entry("foobar", 7)); query = new PrefixCompletionQuery(analyzer, new Term("suggest_field_no_p_sep", "foob")); // except the fo suggest = indexSearcher.suggest(query, 4); assertSuggestions(suggest, new Entry("the foo bar", 10), new Entry("foo bar", 8), new Entry("foobar", 7)); reader.close(); iw.close(); }
40. TestPrefixCompletionQuery#testAnalyzerWithSepAndNoPreservePos()
Project: lucene-solr
File: TestPrefixCompletionQuery.java
File: TestPrefixCompletionQuery.java
@Test public void testAnalyzerWithSepAndNoPreservePos() throws Exception { Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, true, false); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(completionAnalyzer, "suggest_field_no_p_pos_inc")); Document document = new Document(); document.add(new SuggestField("suggest_field_no_p_pos_inc", "foobar", 7)); document.add(new SuggestField("suggest_field_no_p_pos_inc", "foo bar", 8)); document.add(new SuggestField("suggest_field_no_p_pos_inc", "the fo", 9)); document.add(new SuggestField("suggest_field_no_p_pos_inc", "the foo bar", 10)); iw.addDocument(document); DirectoryReader reader = iw.getReader(); SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader); CompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field_no_p_pos_inc", "fo")); //matches all 4 TopSuggestDocs suggest = indexSearcher.suggest(query, 4); assertSuggestions(suggest, new Entry("the foo bar", 10), new Entry("the fo", 9), new Entry("foo bar", 8), new Entry("foobar", 7)); query = new PrefixCompletionQuery(analyzer, new Term("suggest_field_no_p_pos_inc", "foob")); // only foobar suggest = indexSearcher.suggest(query, 4); assertSuggestions(suggest, new Entry("foobar", 7)); reader.close(); iw.close(); }
41. TestPrefixCompletionQuery#testAnalyzerWithoutPreservePosAndSep()
Project: lucene-solr
File: TestPrefixCompletionQuery.java
File: TestPrefixCompletionQuery.java
@Test public void testAnalyzerWithoutPreservePosAndSep() throws Exception { Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET); CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer, false, false); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwcWithSuggestField(completionAnalyzer, "suggest_field_no_p_sep_or_pos_inc")); Document document = new Document(); document.add(new SuggestField("suggest_field_no_p_sep_or_pos_inc", "foobar", 7)); document.add(new SuggestField("suggest_field_no_p_sep_or_pos_inc", "foo bar", 8)); document.add(new SuggestField("suggest_field_no_p_sep_or_pos_inc", "the fo", 9)); document.add(new SuggestField("suggest_field_no_p_sep_or_pos_inc", "the foo bar", 10)); iw.addDocument(document); DirectoryReader reader = iw.getReader(); SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader); CompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field_no_p_sep_or_pos_inc", "fo")); // all 4 TopSuggestDocs suggest = indexSearcher.suggest(query, 4); assertSuggestions(suggest, new Entry("the foo bar", 10), new Entry("the fo", 9), new Entry("foo bar", 8), new Entry("foobar", 7)); query = new PrefixCompletionQuery(analyzer, new Term("suggest_field_no_p_sep_or_pos_inc", "foob")); // not the fo suggest = indexSearcher.suggest(query, 4); assertSuggestions(suggest, new Entry("the foo bar", 10), new Entry("foo bar", 8), new Entry("foobar", 7)); reader.close(); iw.close(); }
42. TestIndexWriter#testEmptyFieldNameWithEmptyTerm()
Project: lucene-solr
File: TestIndexWriter.java
File: TestIndexWriter.java
public void testEmptyFieldNameWithEmptyTerm() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); Document doc = new Document(); doc.add(newStringField("", "", Field.Store.NO)); doc.add(newStringField("", "a", Field.Store.NO)); doc.add(newStringField("", "b", Field.Store.NO)); doc.add(newStringField("", "c", Field.Store.NO)); writer.addDocument(doc); writer.close(); DirectoryReader reader = DirectoryReader.open(dir); LeafReader subreader = getOnlyLeafReader(reader); TermsEnum te = subreader.fields().terms("").iterator(); assertEquals(new BytesRef(""), te.next()); assertEquals(new BytesRef("a"), te.next()); assertEquals(new BytesRef("b"), te.next()); assertEquals(new BytesRef("c"), te.next()); assertNull(te.next()); reader.close(); dir.close(); }
43. TestPointValues#testPointsFieldMissingFromOneSegment()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testPointsFieldMissingFromOneSegment() throws Exception { Directory dir = FSDirectory.open(createTempDir()); IndexWriterConfig iwc = new IndexWriterConfig(null); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new StringField("id", "0", Field.Store.NO)); doc.add(new IntPoint("int0", 0)); w.addDocument(doc); w.commit(); doc = new Document(); doc.add(new IntPoint("int1", 17)); w.addDocument(doc); w.forceMerge(1); w.close(); dir.close(); }
44. ADMLuceneIndexerImpl#addFtsStatusDoc()
Project: community-edition
File: ADMLuceneIndexerImpl.java
File: ADMLuceneIndexerImpl.java
private void addFtsStatusDoc(List<Document> docs, FTSStatus ftsStatus, NodeRef nodeRef, NodeRef.Status nodeStatus) { // If we are being called during FTS failover, then don't bother generating a new doc if (ftsStatus == FTSStatus.Clean) { return; } Document doc = new Document(); doc.add(new Field("ID", GUID.generate(), Field.Store.YES, Field.Index.NO_NORMS, Field.TermVector.NO)); doc.add(new Field("FTSREF", nodeRef.toString(), Field.Store.YES, Field.Index.NO_NORMS, Field.TermVector.NO)); doc.add(new Field("TX", nodeStatus == null ? AlfrescoTransactionSupport.getTransactionId() : nodeStatus.getChangeTxnId(), Field.Store.YES, Field.Index.NO_NORMS, Field.TermVector.NO)); doc.add(new Field("FTSSTATUS", ftsStatus.name(), Field.Store.NO, Field.Index.NO_NORMS, Field.TermVector.NO)); docs.add(doc); }
45. GroupingSearchTest#testBasic()
Project: lucene-solr
File: GroupingSearchTest.java
File: GroupingSearchTest.java
// Tests some very basic usages... public void testBasic() throws Exception { final String groupField = "author"; FieldType customType = new FieldType(); customType.setStored(true); Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); boolean canUseIDV = true; List<Document> documents = new ArrayList<>(); // 0 Document doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); doc.add(new TextField("content", "random text", Field.Store.YES)); doc.add(new Field("id", "1", customType)); documents.add(doc); // 1 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); doc.add(new TextField("content", "some more random text", Field.Store.YES)); doc.add(new Field("id", "2", customType)); documents.add(doc); // 2 doc = new Document(); addGroupField(doc, groupField, "author1", canUseIDV); doc.add(new TextField("content", "some more random textual data", Field.Store.YES)); doc.add(new Field("id", "3", customType)); doc.add(new StringField("groupend", "x", Field.Store.NO)); documents.add(doc); w.addDocuments(documents); documents.clear(); // 3 doc = new Document(); addGroupField(doc, groupField, "author2", canUseIDV); doc.add(new TextField("content", "some random text", Field.Store.YES)); doc.add(new Field("id", "4", customType)); doc.add(new StringField("groupend", "x", Field.Store.NO)); w.addDocument(doc); // 4 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); doc.add(new TextField("content", "some more random text", Field.Store.YES)); doc.add(new Field("id", "5", customType)); documents.add(doc); // 5 doc = new Document(); addGroupField(doc, groupField, "author3", canUseIDV); doc.add(new TextField("content", "random", Field.Store.YES)); doc.add(new Field("id", "6", customType)); doc.add(new StringField("groupend", "x", Field.Store.NO)); documents.add(doc); w.addDocuments(documents); documents.clear(); // 6 -- no author field doc = new Document(); doc.add(new TextField("content", "random word stuck in alot of other text", Field.Store.YES)); doc.add(new Field("id", "6", customType)); doc.add(new StringField("groupend", "x", Field.Store.NO)); w.addDocument(doc); IndexSearcher indexSearcher = newSearcher(w.getReader()); w.close(); Sort groupSort = Sort.RELEVANCE; GroupingSearch groupingSearch = createRandomGroupingSearch(groupField, groupSort, 5, canUseIDV); TopGroups<?> groups = groupingSearch.search(indexSearcher, new TermQuery(new Term("content", "random")), 0, 10); assertEquals(7, groups.totalHitCount); assertEquals(7, groups.totalGroupedHitCount); assertEquals(4, groups.groups.length); // relevance order: 5, 0, 3, 4, 1, 2, 6 // the later a document is added the higher this docId // value GroupDocs<?> group = groups.groups[0]; compareGroupValue("author3", group); assertEquals(2, group.scoreDocs.length); assertEquals(5, group.scoreDocs[0].doc); assertEquals(4, group.scoreDocs[1].doc); assertTrue(group.scoreDocs[0].score >= group.scoreDocs[1].score); group = groups.groups[1]; compareGroupValue("author1", group); assertEquals(3, group.scoreDocs.length); assertEquals(0, group.scoreDocs[0].doc); assertEquals(1, group.scoreDocs[1].doc); assertEquals(2, group.scoreDocs[2].doc); assertTrue(group.scoreDocs[0].score >= group.scoreDocs[1].score); assertTrue(group.scoreDocs[1].score >= group.scoreDocs[2].score); group = groups.groups[2]; compareGroupValue("author2", group); assertEquals(1, group.scoreDocs.length); assertEquals(3, group.scoreDocs[0].doc); group = groups.groups[3]; compareGroupValue(null, group); assertEquals(1, group.scoreDocs.length); assertEquals(6, group.scoreDocs[0].doc); Query lastDocInBlock = new TermQuery(new Term("groupend", "x")); groupingSearch = new GroupingSearch(lastDocInBlock); groups = groupingSearch.search(indexSearcher, new TermQuery(new Term("content", "random")), 0, 10); assertEquals(7, groups.totalHitCount); assertEquals(7, groups.totalGroupedHitCount); assertEquals(4, groups.totalGroupCount.longValue()); assertEquals(4, groups.groups.length); indexSearcher.getIndexReader().close(); dir.close(); }
46. TestDocValuesIndexing#testSameFieldNameForPostingAndDocValue()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testSameFieldNameForPostingAndDocValue() throws Exception { // LUCENE-5192: FieldInfos.Builder neglected to update // globalFieldNumbers.docValuesType map if the field existed, resulting in // potentially adding the same field with different DV types. Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new StringField("f", "mock-value", Store.NO)); doc.add(new NumericDocValuesField("f", 5)); writer.addDocument(doc); writer.commit(); Document doc2 = new Document(); doc2.add(new BinaryDocValuesField("f", new BytesRef("mock"))); expectThrows(IllegalArgumentException.class, () -> { writer.addDocument(doc2); }); writer.rollback(); dir.close(); }
47. TestDocValuesIndexing#testDocsWithField()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testDocsWithField() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); doc = new Document(); doc.add(new TextField("dv", "some text", Field.Store.NO)); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); DirectoryReader r = writer.getReader(); writer.close(); LeafReader subR = r.leaves().get(0).reader(); assertEquals(2, subR.numDocs()); Bits bits = DocValues.getDocsWithField(subR, "dv"); assertTrue(bits.get(0)); assertTrue(bits.get(1)); r.close(); dir.close(); }
48. TestDocValuesIndexing#testMixedTypesAfterReopenAppend3()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testMixedTypesAfterReopenAppend3() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); Document doc = new Document(); doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo"))); w.addDocument(doc); w.close(); Document doc2 = new Document(); IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); doc2.add(new StringField("foo", "bar", Field.Store.NO)); doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo"))); // NOTE: this case follows a different code path inside // DefaultIndexingChain/FieldInfos, because the field (foo) // is first added without DocValues: expectThrows(IllegalArgumentException.class, () -> { w2.addDocument(doc2); }); // Also add another document so there is a segment to write here: w2.addDocument(new Document()); w2.forceMerge(1); w2.close(); dir.close(); }
49. TestDocValuesIndexing#testMixedTypesAfterReopenAppend2()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testMixedTypesAfterReopenAppend2() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); Document doc = new Document(); doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo"))); w.addDocument(doc); w.close(); Document doc2 = new Document(); IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); doc2.add(new StringField("foo", "bar", Field.Store.NO)); doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo"))); // NOTE: this case follows a different code path inside // DefaultIndexingChain/FieldInfos, because the field (foo) // is first added without DocValues: expectThrows(IllegalArgumentException.class, () -> { w2.addDocument(doc2); }); w2.forceMerge(1); w2.close(); dir.close(); }
50. TestDirectoryReader#addDocumentWithDifferentFields()
Project: lucene-solr
File: TestDirectoryReader.java
File: TestDirectoryReader.java
static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException { Document doc = new Document(); FieldType customType3 = new FieldType(); customType3.setStored(true); doc.add(newStringField("keyword2", "test1", Field.Store.YES)); doc.add(newTextField("text2", "test1", Field.Store.YES)); doc.add(newField("unindexed2", "test1", customType3)); doc.add(new TextField("unstored2", "test1", Field.Store.NO)); writer.addDocument(doc); }
51. TestDirectoryReader#addDocumentWithFields()
Project: lucene-solr
File: TestDirectoryReader.java
File: TestDirectoryReader.java
static void addDocumentWithFields(IndexWriter writer) throws IOException { Document doc = new Document(); FieldType customType3 = new FieldType(); customType3.setStored(true); doc.add(newStringField("keyword", "test1", Field.Store.YES)); doc.add(newTextField("text", "test1", Field.Store.YES)); doc.add(newField("unindexed", "test1", customType3)); doc.add(new TextField("unstored", "test1", Field.Store.NO)); writer.addDocument(doc); }
52. TestBinaryDocValuesUpdates#testUpdatesOrder()
Project: lucene-solr
File: TestBinaryDocValuesUpdates.java
File: TestBinaryDocValuesUpdates.java
public void testUpdatesOrder() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new StringField("upd", "t1", Store.NO)); doc.add(new StringField("upd", "t2", Store.NO)); doc.add(new BinaryDocValuesField("f1", toBytes(1L))); doc.add(new BinaryDocValuesField("f2", toBytes(1L))); writer.addDocument(doc); // update f1 to 2 writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(2L)); // update f2 to 2 writer.updateBinaryDocValue(new Term("upd", "t1"), "f2", toBytes(2L)); // update f1 to 3 writer.updateBinaryDocValue(new Term("upd", "t2"), "f1", toBytes(3L)); // update f2 to 3 writer.updateBinaryDocValue(new Term("upd", "t2"), "f2", toBytes(3L)); // update f1 to 4 (but not f2) writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(4L)); writer.close(); DirectoryReader reader = DirectoryReader.open(dir); assertEquals(4, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0)); assertEquals(3, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f2"), 0)); reader.close(); dir.close(); }
53. DocumentClassificationTestBase#getBatmanAmbiguosDocument()
Project: lucene-solr
File: DocumentClassificationTestBase.java
File: DocumentClassificationTestBase.java
protected Document getBatmanAmbiguosDocument() { Document doc = new Document(); String title = "Batman: Arkham Knight new adventures for the super hero across Gotham, the Dark Knight has returned! Batman will win !"; String text = "Early games used interactive electronic devices with various display formats. The earliest example is" + " from 1947—a device was filed for a patent on 25 January 1947, by Thomas T. Goldsmith Jr. and Estle Ray Mann," + " and issued on 14 December 1948, as U.S. Patent 2455992.[2]" + "Inspired by radar display tech, it consisted of an analog device that allowed a user to control a vector-drawn" + " dot on the screen to simulate a missile being fired at targets, which were drawings fixed to the screen.[3]"; String author = "Ign"; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(titleFieldName, title, ft)); doc.add(new Field(authorFieldName, author, ft)); doc.add(new Field(booleanFieldName, "false", ft)); return doc; }
54. DocumentClassificationTestBase#getVideoGameDocument()
Project: lucene-solr
File: DocumentClassificationTestBase.java
File: DocumentClassificationTestBase.java
protected Document getVideoGameDocument() { Document doc = new Document(); String title = "The new generation of PC and Console Video games"; String text = "Recently a lot of games have been released for the latest generations of consoles and personal computers." + "One of them is Batman: Arkham Knight released recently on PS4, X-box and personal computer." + "Another important video game that will be released in November is Assassin's Creed, a classic series that sees its new installement on Halloween." + "Recently a lot of problems affected the Assassin's creed series but this time it should ran smoothly on affordable GPUs." + "Players are waiting for the versions of their favourite video games and so do we."; String author = "Ign"; doc.add(new Field(textFieldName, text, ft)); doc.add(new Field(titleFieldName, title, ft)); doc.add(new Field(authorFieldName, author, ft)); doc.add(new Field(booleanFieldName, "false", ft)); return doc; }
55. PrefixFilterTest#testBasics()
Project: greplin-lucene-utils
File: PrefixFilterTest.java
File: PrefixFilterTest.java
@Test public void testBasics() throws Exception { IndexWriter w = createWriter(); Document doc1 = new Document(); doc1.add(new Field("f", "party", Field.Store.YES, Field.Index.ANALYZED)); doc1.add(new Field("f", "partner", Field.Store.YES, Field.Index.ANALYZED)); w.addDocument(doc1); Document doc2 = new Document(); doc2.add(new Field("f", "partridge", Field.Store.YES, Field.Index.ANALYZED)); w.addDocument(doc2); IndexReader reader = createReader(w); assertFilterBitsEqual(reader, new PrefixFilter("f", "o"), false, false); assertFilterBitsEqual(reader, new PrefixFilter("f", "par"), true, true); assertFilterBitsEqual(reader, new PrefixFilter("f", "partner"), true, false); assertFilterBitsEqual(reader, new PrefixFilter("f", "partridge"), false, true); assertFilterBitsEqual(reader, new PrefixFilter("f", "partridges"), false, false); assertFilterBitsEqual(reader, new PrefixFilter("f", "parts"), false, false); assertFilterBitsEqual(reader, new PrefixFilter("f", "partr"), false, true); assertFilterBitsEqual(reader, new PrefixFilter("f", "party"), true, false); assertFilterBitsEqual(reader, new PrefixFilter("f", "q"), false, false); }
56. TestIDVersionPostingsFormat#testMoreThanOneDocPerIDWithDeletesAcrossSegments()
Project: lucene-solr
File: TestIDVersionPostingsFormat.java
File: TestIDVersionPostingsFormat.java
public void testMoreThanOneDocPerIDWithDeletesAcrossSegments() throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); w.commit(); doc = new Document(); doc.add(makeIDField("id", 17)); // Replaces the doc we just indexed: w.updateDocument(new Term("id", "id"), doc); w.forceMerge(1); w.close(); dir.close(); }
57. TestIDVersionPostingsFormat#testMoreThanOneDocPerIDWithDeletes()
Project: lucene-solr
File: TestIDVersionPostingsFormat.java
File: TestIDVersionPostingsFormat.java
public void testMoreThanOneDocPerIDWithDeletes() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); w.deleteDocuments(new Term("id", "id")); doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); w.commit(); w.close(); dir.close(); }
58. TestIDVersionPostingsFormat#testMoreThanOneDocPerIDWithUpdates()
Project: lucene-solr
File: TestIDVersionPostingsFormat.java
File: TestIDVersionPostingsFormat.java
public void testMoreThanOneDocPerIDWithUpdates() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); doc = new Document(); doc.add(makeIDField("id", 17)); // Replaces the doc we just indexed: w.updateDocument(new Term("id", "id"), doc); w.commit(); w.close(); dir.close(); }
59. TestIDVersionPostingsFormat#testMoreThanOneDocPerIDOneSegment()
Project: lucene-solr
File: TestIDVersionPostingsFormat.java
File: TestIDVersionPostingsFormat.java
public void testMoreThanOneDocPerIDOneSegment() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(makeIDField("id", 17)); w.addDocument(doc); Document duplicate = new Document(); duplicate.add(makeIDField("id", 17)); expectThrows(IllegalArgumentException.class, () -> { w.addDocument(duplicate); w.commit(); }); w.close(); dir.close(); }
60. TestIDVersionPostingsFormat#testBasic()
Project: lucene-solr
File: TestIDVersionPostingsFormat.java
File: TestIDVersionPostingsFormat.java
public void testBasic() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat())); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(makeIDField("id0", 100)); w.addDocument(doc); doc = new Document(); doc.add(makeIDField("id1", 110)); w.addDocument(doc); IndexReader r = w.getReader(); IDVersionSegmentTermsEnum termsEnum = (IDVersionSegmentTermsEnum) r.leaves().get(0).reader().fields().terms("id").iterator(); assertTrue(termsEnum.seekExact(new BytesRef("id0"), 50)); assertTrue(termsEnum.seekExact(new BytesRef("id0"), 100)); assertFalse(termsEnum.seekExact(new BytesRef("id0"), 101)); assertTrue(termsEnum.seekExact(new BytesRef("id1"), 50)); assertTrue(termsEnum.seekExact(new BytesRef("id1"), 110)); assertFalse(termsEnum.seekExact(new BytesRef("id1"), 111)); r.close(); w.close(); dir.close(); }
61. TestFieldCacheSort#testFieldDocReverse()
Project: lucene-solr
File: TestFieldCacheSort.java
File: TestFieldCacheSort.java
/** Tests sorting on reverse internal docid order */ public void testFieldDocReverse() throws Exception { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("value", "foo", Field.Store.NO)); writer.addDocument(doc); doc = new Document(); doc.add(newStringField("value", "bar", Field.Store.NO)); writer.addDocument(doc); IndexReader ir = writer.getReader(); writer.close(); IndexSearcher searcher = newSearcher(ir); Sort sort = new Sort(new SortField(null, SortField.Type.DOC, true)); TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); assertEquals(2, td.totalHits); // docid 1, then docid 0 assertEquals(1, td.scoreDocs[0].doc); assertEquals(0, td.scoreDocs[1].doc); TestUtil.checkReader(ir); ir.close(); dir.close(); }
62. TestFieldCacheSort#testFieldDoc()
Project: lucene-solr
File: TestFieldCacheSort.java
File: TestFieldCacheSort.java
/** Tests sorting on internal docid order */ public void testFieldDoc() throws Exception { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("value", "foo", Field.Store.NO)); writer.addDocument(doc); doc = new Document(); doc.add(newStringField("value", "bar", Field.Store.NO)); writer.addDocument(doc); IndexReader ir = writer.getReader(); writer.close(); IndexSearcher searcher = newSearcher(ir); Sort sort = new Sort(SortField.FIELD_DOC); TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); assertEquals(2, td.totalHits); // docid 0, then docid 1 assertEquals(0, td.scoreDocs[0].doc); assertEquals(1, td.scoreDocs[1].doc); TestUtil.checkReader(ir); ir.close(); dir.close(); }
63. DocHelper#createDocument()
Project: lucene-solr
File: DocHelper.java
File: DocHelper.java
public static Document createDocument(int n, String indexName, int numFields) { StringBuilder sb = new StringBuilder(); FieldType customType = new FieldType(TextField.TYPE_STORED); customType.setStoreTermVectors(true); customType.setStoreTermVectorPositions(true); customType.setStoreTermVectorOffsets(true); FieldType customType1 = new FieldType(StringField.TYPE_STORED); customType1.setStoreTermVectors(true); customType1.setStoreTermVectorPositions(true); customType1.setStoreTermVectorOffsets(true); final Document doc = new Document(); doc.add(new Field("id", Integer.toString(n), customType1)); doc.add(new Field("indexname", indexName, customType1)); sb.append("a"); sb.append(n); doc.add(new Field("field1", sb.toString(), customType)); sb.append(" b"); sb.append(n); for (int i = 1; i < numFields; i++) { doc.add(new Field("field" + (i + 1), sb.toString(), customType)); } return doc; }
64. TestLongNormValueSource#beforeClass()
Project: lucene-solr
File: TestLongNormValueSource.java
File: TestLongNormValueSource.java
@BeforeClass public static void beforeClass() throws Exception { dir = newDirectory(); analyzer = new MockAnalyzer(random()); IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer); iwConfig.setMergePolicy(newLogMergePolicy()); iwConfig.setSimilarity(sim); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig); Document doc = new Document(); doc.add(new TextField("text", "this is a test test test", Field.Store.NO)); iw.addDocument(doc); doc = new Document(); doc.add(new TextField("text", "second test", Field.Store.NO)); iw.addDocument(doc); reader = iw.getReader(); searcher = newSearcher(reader); iw.close(); }
65. TestPKIndexSplitter#createDocument()
Project: lucene-solr
File: TestPKIndexSplitter.java
File: TestPKIndexSplitter.java
private Document createDocument(int n, String indexName, int numFields, NumberFormat format) { StringBuilder sb = new StringBuilder(); Document doc = new Document(); String id = format.format(n); doc.add(newStringField("id", id, Field.Store.YES)); doc.add(newStringField("indexname", indexName, Field.Store.YES)); sb.append("a"); sb.append(n); doc.add(newTextField("field1", sb.toString(), Field.Store.YES)); sb.append(" b"); sb.append(n); for (int i = 1; i < numFields; i++) { doc.add(newTextField("field" + (i + 1), sb.toString(), Field.Store.YES)); } return doc; }
66. TestMemoryIndex#testBuildFromDocument()
Project: lucene-solr
File: TestMemoryIndex.java
File: TestMemoryIndex.java
@Test public void testBuildFromDocument() { Document doc = new Document(); doc.add(new TextField("field1", "some text", Field.Store.NO)); doc.add(new TextField("field1", "some more text", Field.Store.NO)); doc.add(new StringField("field2", "untokenized text", Field.Store.NO)); analyzer.setPositionIncrementGap(100); MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer); assertThat(mi.search(new TermQuery(new Term("field1", "text"))), not(0.0f)); assertThat(mi.search(new TermQuery(new Term("field2", "text"))), is(0.0f)); assertThat(mi.search(new TermQuery(new Term("field2", "untokenized text"))), not(0.0f)); assertThat(mi.search(new PhraseQuery("field1", "some", "more", "text")), not(0.0f)); assertThat(mi.search(new PhraseQuery("field1", "some", "text")), not(0.0f)); assertThat(mi.search(new PhraseQuery("field1", "text", "some")), is(0.0f)); }
67. TestJoinUtil#testRewrite()
Project: lucene-solr
File: TestJoinUtil.java
File: TestJoinUtil.java
public void testRewrite() throws IOException { Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(new SortedDocValuesField("join_field", new BytesRef("abc"))); w.addDocument(doc); doc = new Document(); doc.add(new SortedDocValuesField("join_field", new BytesRef("abd"))); w.addDocument(doc); IndexReader reader = w.getReader(); IndexSearcher searcher = newSearcher(reader); OrdinalMap ordMap = OrdinalMap.build(null, new SortedDocValues[0], 0f); Query joinQuery = JoinUtil.createJoinQuery("join_field", new MatchNoDocsQuery(), new MatchNoDocsQuery(), searcher, RandomPicks.randomFrom(random(), ScoreMode.values()), ordMap, 0, Integer.MAX_VALUE); // no exception due to missing rewrites searcher.search(joinQuery, 1); reader.close(); w.close(); dir.close(); }
68. TestBlockJoin#testAdvanceSingleParentSingleChild()
Project: lucene-solr
File: TestBlockJoin.java
File: TestBlockJoin.java
public void testAdvanceSingleParentSingleChild() throws Exception { Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document childDoc = new Document(); childDoc.add(newStringField("child", "1", Field.Store.NO)); Document parentDoc = new Document(); parentDoc.add(newStringField("parent", "1", Field.Store.NO)); w.addDocuments(Arrays.asList(childDoc, parentDoc)); IndexReader r = w.getReader(); w.close(); IndexSearcher s = newSearcher(r); Query tq = new TermQuery(new Term("child", "1")); BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term("parent", "1"))); CheckJoinIndex.check(s.getIndexReader(), parentFilter); ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(tq, parentFilter, ScoreMode.Avg); Weight weight = s.createNormalizedWeight(q, true); Scorer sc = weight.scorer(s.getIndexReader().leaves().get(0)); assertEquals(1, sc.iterator().advance(1)); r.close(); dir.close(); }
69. SimpleLuceneTests#testSimpleNumericOps()
Project: elasticsearch
File: SimpleLuceneTests.java
File: SimpleLuceneTests.java
public void testSimpleNumericOps() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); Document document = new Document(); document.add(new TextField("_id", "1", Field.Store.YES)); document.add(new LegacyIntField("test", 2, LegacyIntField.TYPE_STORED)); indexWriter.addDocument(document); IndexReader reader = DirectoryReader.open(indexWriter); IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); Document doc = searcher.doc(topDocs.scoreDocs[0].doc); IndexableField f = doc.getField("test"); assertThat(f.stringValue(), equalTo("2")); BytesRefBuilder bytes = new BytesRefBuilder(); LegacyNumericUtils.intToPrefixCoded(2, 0, bytes); topDocs = searcher.search(new TermQuery(new Term("test", bytes.get())), 1); doc = searcher.doc(topDocs.scoreDocs[0].doc); f = doc.getField("test"); assertThat(f.stringValue(), equalTo("2")); indexWriter.close(); }
70. TestTaxonomyFacetCounts#testDetectMultiValuedField()
Project: lucene-solr
File: TestTaxonomyFacetCounts.java
File: TestTaxonomyFacetCounts.java
// Make sure we catch when app didn't declare field as // multi-valued but it was: public void testDetectMultiValuedField() throws Exception { Directory dir = newDirectory(); Directory taxoDir = newDirectory(); TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); FacetsConfig config = new FacetsConfig(); Document doc = new Document(); doc.add(newTextField("field", "text", Field.Store.NO)); doc.add(new FacetField("a", "path")); doc.add(new FacetField("a", "path2")); expectThrows(IllegalArgumentException.class, () -> { config.build(taxoWriter, doc); }); writer.close(); IOUtils.close(taxoWriter, dir, taxoDir); }
71. TestPointQueries#testNumericNoValuesMatch()
Project: lucene-solr
File: TestPointQueries.java
File: TestPointQueries.java
public void testNumericNoValuesMatch() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setCodec(getCodec()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(new SortedNumericDocValuesField("value", 17)); w.addDocument(doc); doc = new Document(); doc.add(new SortedNumericDocValuesField("value", 22)); w.addDocument(doc); IndexReader r = w.getReader(); IndexSearcher s = new IndexSearcher(r); assertEquals(0, s.count(LongPoint.newRangeQuery("value", 17L, 13L))); IOUtils.close(r, w, dir); }
72. TestPointQueries#testSortedSetNoOrdsMatch()
Project: lucene-solr
File: TestPointQueries.java
File: TestPointQueries.java
public void testSortedSetNoOrdsMatch() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setCodec(getCodec()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("value", toUTF8("a"))); w.addDocument(doc); doc = new Document(); doc.add(new BinaryPoint("value", toUTF8("z"))); w.addDocument(doc); IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); assertEquals(0, s.count(BinaryPoint.newRangeQuery("value", toUTF8("m"), toUTF8("m")))); IOUtils.close(r, w, dir); }
73. TestPointQueries#testLongMinMaxSortedSet()
Project: lucene-solr
File: TestPointQueries.java
File: TestPointQueries.java
public void testLongMinMaxSortedSet() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setCodec(getCodec()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(new LongPoint("value", Long.MIN_VALUE)); w.addDocument(doc); doc = new Document(); doc.add(new LongPoint("value", Long.MAX_VALUE)); w.addDocument(doc); IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE))); assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE - 1))); assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE + 1, Long.MAX_VALUE))); assertEquals(0, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE + 1, Long.MAX_VALUE - 1))); IOUtils.close(r, w, dir); }
74. TestPointQueries#testLongMinMaxNumeric()
Project: lucene-solr
File: TestPointQueries.java
File: TestPointQueries.java
public void testLongMinMaxNumeric() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setCodec(getCodec()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(new LongPoint("value", Long.MIN_VALUE)); w.addDocument(doc); doc = new Document(); doc.add(new LongPoint("value", Long.MAX_VALUE)); w.addDocument(doc); IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE))); assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE - 1))); assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE + 1, Long.MAX_VALUE))); assertEquals(0, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE + 1, Long.MAX_VALUE - 1))); IOUtils.close(r, w, dir); }
75. TestPointQueries#testMinMaxLong()
Project: lucene-solr
File: TestPointQueries.java
File: TestPointQueries.java
public void testMinMaxLong() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setCodec(getCodec()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(new LongPoint("value", Long.MIN_VALUE)); w.addDocument(doc); doc = new Document(); doc.add(new LongPoint("value", Long.MAX_VALUE)); w.addDocument(doc); IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); assertEquals(1, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, 0L))); assertEquals(1, s.count(LongPoint.newRangeQuery("value", 0L, Long.MAX_VALUE))); assertEquals(2, s.count(LongPoint.newRangeQuery("value", Long.MIN_VALUE, Long.MAX_VALUE))); IOUtils.close(r, w, dir); }
76. TestDateSort#createDocument()
Project: lucene-solr
File: TestDateSort.java
File: TestDateSort.java
private Document createDocument(String text, long time) { Document document = new Document(); // Add the text field. Field textField = newTextField(TEXT_FIELD, text, Field.Store.YES); document.add(textField); // Add the date/time field. String dateTimeString = DateTools.timeToString(time, DateTools.Resolution.SECOND); Field dateTimeField = newStringField(DATE_TIME_FIELD, dateTimeString, Field.Store.YES); document.add(dateTimeField); document.add(new SortedDocValuesField(DATE_TIME_FIELD, new BytesRef(dateTimeString))); return document; }
77. TestAutomatonQuery#setUp()
Project: lucene-solr
File: TestAutomatonQuery.java
File: TestAutomatonQuery.java
@Override public void setUp() throws Exception { super.setUp(); directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory); Document doc = new Document(); Field titleField = newTextField("title", "some title", Field.Store.NO); Field field = newTextField(FN, "this is document one 2345", Field.Store.NO); Field footerField = newTextField("footer", "a footer", Field.Store.NO); doc.add(titleField); doc.add(field); doc.add(footerField); writer.addDocument(doc); field.setStringValue("some text from doc two a short piece 5678.91"); writer.addDocument(doc); field.setStringValue("doc three has some different stuff" + " with numbers 1234 5678.9 and letter b"); writer.addDocument(doc); reader = writer.getReader(); searcher = newSearcher(reader); writer.close(); }
78. TestTermsEnum#testZeroTerms()
Project: lucene-solr
File: TestTermsEnum.java
File: TestTermsEnum.java
public void testZeroTerms() throws Exception { d = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), d); Document doc = new Document(); doc.add(newTextField("field", "one two three", Field.Store.NO)); doc = new Document(); doc.add(newTextField("field2", "one two three", Field.Store.NO)); w.addDocument(doc); w.commit(); w.deleteDocuments(new Term("field", "one")); w.forceMerge(1); IndexReader r = w.getReader(); w.close(); assertEquals(1, r.numDocs()); assertEquals(1, r.maxDoc()); Terms terms = MultiFields.getTerms(r, "field"); if (terms != null) { assertNull(terms.iterator().next()); } r.close(); d.close(); }
79. TestPointValues#testDifferentCodecs2()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
// Write point values, one segment with Lucene62, another with SimpleText, then forceMerge with Lucene60 public void testDifferentCodecs2() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(Codec.forName("SimpleText")); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new IntPoint("int", 1)); w.addDocument(doc); w.close(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(Codec.forName("Lucene62")); w = new IndexWriter(dir, iwc); doc = new Document(); doc.add(new IntPoint("int", 1)); w.addDocument(doc); w.forceMerge(1); w.close(); dir.close(); }
80. TestPointValues#testDifferentCodecs1()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
// Write point values, one segment with Lucene62, another with SimpleText, then forceMerge with SimpleText public void testDifferentCodecs1() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(Codec.forName("Lucene62")); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new IntPoint("int", 1)); w.addDocument(doc); w.close(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); iwc.setCodec(Codec.forName("SimpleText")); w = new IndexWriter(dir, iwc); doc = new Document(); doc.add(new IntPoint("int", 1)); w.addDocument(doc); w.forceMerge(1); w.close(); dir.close(); }
81. TestPointValues#testIllegalTooManyBytes()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testIllegalTooManyBytes() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("dim", new byte[PointValues.MAX_NUM_BYTES + 1])); expectThrows(IllegalArgumentException.class, () -> { w.addDocument(doc); }); Document doc2 = new Document(); doc2.add(new IntPoint("dim", 17)); w.addDocument(doc2); w.close(); dir.close(); }
82. TestPointValues#testIllegalNumBytesChangeViaAddIndexesSlowCodecReader()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testIllegalNumBytesChangeViaAddIndexesSlowCodecReader() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("dim", new byte[4])); w.addDocument(doc); w.close(); Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w2 = new IndexWriter(dir2, iwc); doc = new Document(); doc.add(new BinaryPoint("dim", new byte[6])); w2.addDocument(doc); DirectoryReader r = DirectoryReader.open(dir); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { TestUtil.addIndexesSlowly(w2, r); }); assertEquals("cannot change point numBytes from 6 to 4 for field=\"dim\"", expected.getMessage()); IOUtils.close(r, w2, dir, dir2); }
83. TestPointValues#testIllegalNumBytesChangeViaAddIndexesCodecReader()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testIllegalNumBytesChangeViaAddIndexesCodecReader() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("dim", new byte[4])); w.addDocument(doc); w.close(); Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w2 = new IndexWriter(dir2, iwc); doc = new Document(); doc.add(new BinaryPoint("dim", new byte[6])); w2.addDocument(doc); DirectoryReader r = DirectoryReader.open(dir); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { w2.addIndexes(new CodecReader[] { (CodecReader) getOnlyLeafReader(r) }); }); assertEquals("cannot change point numBytes from 6 to 4 for field=\"dim\"", expected.getMessage()); IOUtils.close(r, w2, dir, dir2); }
84. TestPointValues#testIllegalNumBytesChangeViaAddIndexesDirectory()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testIllegalNumBytesChangeViaAddIndexesDirectory() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("dim", new byte[4])); w.addDocument(doc); w.close(); Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w2 = new IndexWriter(dir2, iwc); doc = new Document(); doc.add(new BinaryPoint("dim", new byte[6])); w2.addDocument(doc); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { w2.addIndexes(new Directory[] { dir }); }); assertEquals("cannot change point numBytes from 6 to 4 for field=\"dim\"", expected.getMessage()); IOUtils.close(w2, dir, dir2); }
85. TestPointValues#testIllegalNumBytesChangeTwoWriters()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testIllegalNumBytesChangeTwoWriters() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("dim", new byte[4])); w.addDocument(doc); w.close(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w2 = new IndexWriter(dir, iwc); Document doc2 = new Document(); doc2.add(new BinaryPoint("dim", new byte[6])); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { w2.addDocument(doc2); }); assertEquals("cannot change point numBytes from 4 to 6 for field=\"dim\"", expected.getMessage()); w2.close(); dir.close(); }
86. TestPointValues#testIllegalNumBytesChangeTwoSegments()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testIllegalNumBytesChangeTwoSegments() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("dim", new byte[4])); w.addDocument(doc); w.commit(); Document doc2 = new Document(); doc2.add(new BinaryPoint("dim", new byte[6])); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { w.addDocument(doc2); }); assertEquals("cannot change point numBytes from 4 to 6 for field=\"dim\"", expected.getMessage()); w.close(); dir.close(); }
87. TestPointValues#testIllegalNumBytesChangeTwoDocs()
Project: lucene-solr
File: TestPointValues.java
File: TestPointValues.java
public void testIllegalNumBytesChangeTwoDocs() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(new BinaryPoint("dim", new byte[4])); w.addDocument(doc); Document doc2 = new Document(); doc2.add(new BinaryPoint("dim", new byte[6])); IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { w.addDocument(doc2); }); assertEquals("cannot change point numBytes from 4 to 6 for field=\"dim\"", expected.getMessage()); w.close(); dir.close(); }
88. LatentRelationalAnalysis#indexFile()
Project: S-Space
File: LatentRelationalAnalysis.java
File: LatentRelationalAnalysis.java
/** * method to actually index a file using Lucene, adds a document * onto the index writer **/ private static void indexFile(IndexWriter writer, File f) throws IOException { if (f.isHidden() || !f.exists() || !f.canRead()) { System.err.println("Could not write " + f.getName()); return; } System.err.println("Indexing " + f.getCanonicalPath()); Document doc = new Document(); doc.add(new Field("path", f.getCanonicalPath(), Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field("modified", DateTools.timeToString(f.lastModified(), DateTools.Resolution.MINUTE), Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field("contents", new FileReader(f))); writer.addDocument(doc); }
89. FileDocument#getDocument()
Project: railo
File: FileDocument.java
File: FileDocument.java
/** Makes a document for a File. <p> The document has three fields: <ul> <li><code>path</code>--containing the pathname of the file, as a stored, tokenized field; <li><code>modified</code>--containing the last modified date of the file as a keyword field as encoded by <a href="lucene.document.DateField.html">DateField</a>; and <li><code>contents</code>--containing the full contents of the file, as a Reader field; * @param res * @return matching document * @throws IOException */ public static Document getDocument(Resource res, String charset) throws IOException { // make a new, empty document Document doc = new Document(); doc.add(FieldUtil.UnIndexed("mime-type", "text/plain")); String content = IOUtil.toString(res, charset); FieldUtil.setRaw(doc, content); //doc.add(FieldUtil.UnIndexed("raw", content)); doc.add(FieldUtil.Text("contents", content.toLowerCase())); doc.add(FieldUtil.UnIndexed("summary", StringUtil.max(content, SUMMERY_SIZE))); return doc; }
90. DefaultDocumentCreator#getDocument()
Project: lenya
File: DefaultDocumentCreator.java
File: DefaultDocumentCreator.java
/** * DOCUMENT ME! * * @param file DOCUMENT ME! * @param htdocsDumpDir DOCUMENT ME! * * @return DOCUMENT ME! * * @throws Exception DOCUMENT ME! */ public Document getDocument(File file, File htdocsDumpDir) throws Exception { Document document = super.getDocument(file, htdocsDumpDir); HTMLParser parser = HTMLParserFactory.newInstance(file); parser.parse(file); document.add(Field.Text("title", parser.getTitle())); document.add(Field.Text("keywords", parser.getKeywords())); document.add(Field.Text("contents", parser.getReader())); return document; }
91. FileDocument#Document()
Project: lenya
File: FileDocument.java
File: FileDocument.java
/** * Makes a document for a File. * * <p> * The document has three fields: * * <ul> * <li> * <code>path</code>--containing the pathname of the file, as a stored, tokenized field; * </li> * <li> * <code>modified</code>--containing the last modified date of the file as a keyword field as * encoded by <a href="lucene.document.DateField.html">DateField</a>; and * </li> * <li> * <code>contents</code>--containing the full contents of the file, as a Reader field. * </li> * </ul> * </p> * * @param f DOCUMENT ME! * * @return DOCUMENT ME! * * @throws java.io.FileNotFoundException DOCUMENT ME! */ public static Document Document(File f) throws java.io.FileNotFoundException { // make a new, empty document Document doc = new Document(); // Add the path of the file as a field named "path". Use a Text field, so // that the index stores the path, and so that the path is searchable doc.add(Field.Text("path", f.getPath())); // Add the last modified date of the file a field named "modified". Use a // Keyword field, so that it's searchable, but so that no attempt is made // to tokenize the field into words. doc.add(Field.Keyword("modified", DateField.timeToString(f.lastModified()))); // Add the contents of the file a field named "contents". Use a Text // field, specifying a Reader, so that the text of the file is tokenized. // ?? why doesn't FileReader work here ?? FileInputStream is = new FileInputStream(f); Reader reader = new BufferedReader(new InputStreamReader(is)); doc.add(Field.Text("contents", reader)); // return the document return doc; }
92. IndexMigrationTest#createDocument()
Project: jackrabbit
File: IndexMigrationTest.java
File: IndexMigrationTest.java
protected static Document createDocument(String name, String value) { Document doc = new Document(); doc.add(new Field(FieldNames.UUID, false, UUID.randomUUID().toString(), Field.Store.YES, Field.Index.NO, Field.TermVector.NO)); doc.add(new Field(FieldNames.PROPERTIES, false, createNamedValue14(name, value), Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); doc.add(new Field(FieldNames.FULLTEXT_PREFIX + ":" + name, true, value, Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); return doc; }
93. NestedAggregatorTests#testResetRootDocId()
Project: elasticsearch
File: NestedAggregatorTests.java
File: NestedAggregatorTests.java
public void testResetRootDocId() throws Exception { Directory directory = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(null); iwc.setMergePolicy(NoMergePolicy.INSTANCE); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, iwc); List<Document> documents = new ArrayList<>(); // 1 segment with, 1 root document, with 3 nested sub docs Document document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); indexWriter.addDocuments(documents); indexWriter.commit(); documents.clear(); // 1 segment with: // 1 document, with 1 nested subdoc document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); indexWriter.addDocuments(documents); documents.clear(); // and 1 document, with 1 nested subdoc document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); indexWriter.addDocuments(documents); indexWriter.commit(); indexWriter.close(); IndexService indexService = createIndex("test"); DirectoryReader directoryReader = DirectoryReader.open(directory); directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(directoryReader); indexService.mapperService().merge("test", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("test", "nested_field", "type=nested").string()), MapperService.MergeReason.MAPPING_UPDATE, false); SearchContext searchContext = createSearchContext(indexService); AggregationContext context = new AggregationContext(searchContext); AggregatorFactories.Builder builder = AggregatorFactories.builder(); NestedAggregationBuilder factory = new NestedAggregationBuilder("test", "nested_field"); builder.addAggregator(factory); AggregatorFactories factories = builder.build(context, null); searchContext.aggregations(new SearchContextAggregations(factories)); Aggregator[] aggs = factories.createTopLevelAggregators(); BucketCollector collector = BucketCollector.wrap(Arrays.asList(aggs)); collector.preCollection(); // A regular search always exclude nested docs, so we use NonNestedDocsFilter.INSTANCE here (otherwise MatchAllDocsQuery would be sufficient) // We exclude root doc with uid type#2, this will trigger the bug if we don't reset the root doc when we process a new segment, because // root doc type#3 and root doc type#1 have the same segment docid BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(Queries.newNonNestedFilter(), Occur.MUST); bq.add(new TermQuery(new Term(UidFieldMapper.NAME, "type#2")), Occur.MUST_NOT); searcher.search(new ConstantScoreQuery(bq.build()), collector); collector.postCollection(); Nested nested = (Nested) aggs[0].buildAggregation(0); // The bug manifests if 6 docs are returned, because currentRootDoc isn't reset the previous child docs from the first segment are emitted as hits. assertThat(nested.getDocCount(), equalTo(4L)); directoryReader.close(); directory.close(); }
94. TestDocValuesIndexing#testTypeChangeViaAddIndexesIR2()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testTypeChangeViaAddIndexesIR2() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); writer.close(); Directory dir2 = newDirectory(); conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer2 = new IndexWriter(dir2, conf); DirectoryReader reader = DirectoryReader.open(dir); TestUtil.addIndexesSlowly(writer2, reader); reader.close(); Document doc2 = new Document(); doc2.add(new SortedDocValuesField("dv", new BytesRef("foo"))); expectThrows(IllegalArgumentException.class, () -> { writer2.addDocument(doc2); }); writer2.close(); dir2.close(); dir.close(); }
95. TestDocValuesIndexing#testTypeChangeViaAddIndexes2()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testTypeChangeViaAddIndexes2() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); writer.close(); Directory dir2 = newDirectory(); conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer2 = new IndexWriter(dir2, conf); writer2.addIndexes(dir); Document doc2 = new Document(); doc2.add(new SortedDocValuesField("dv", new BytesRef("foo"))); expectThrows(IllegalArgumentException.class, () -> { writer2.addDocument(doc2); }); writer2.close(); dir2.close(); dir.close(); }
96. TestDocValuesIndexing#testTypeChangeViaAddIndexes()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testTypeChangeViaAddIndexes() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); writer.close(); Directory dir2 = newDirectory(); conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer2 = new IndexWriter(dir2, conf); doc = new Document(); doc.add(new SortedDocValuesField("dv", new BytesRef("foo"))); writer2.addDocument(doc); expectThrows(IllegalArgumentException.class, () -> { writer2.addIndexes(dir); }); writer2.close(); dir.close(); dir2.close(); }
97. TestDocValuesIndexing#testTypeChangeAfterOpenCreate()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testTypeChangeAfterOpenCreate() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); writer.close(); conf = newIndexWriterConfig(new MockAnalyzer(random())); conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE); writer = new IndexWriter(dir, conf); doc = new Document(); doc.add(new SortedDocValuesField("dv", new BytesRef("foo"))); writer.addDocument(doc); writer.close(); dir.close(); }
98. TestDocValuesIndexing#testTypeChangeAfterCommitAndDeleteAll()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testTypeChangeAfterCommitAndDeleteAll() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); writer.commit(); writer.deleteAll(); doc = new Document(); doc.add(new SortedDocValuesField("dv", new BytesRef("foo"))); writer.addDocument(doc); writer.close(); dir.close(); }
99. TestDocValuesIndexing#testTypeChangeAfterDeleteAll()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testTypeChangeAfterDeleteAll() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); writer.deleteAll(); doc = new Document(); doc.add(new SortedDocValuesField("dv", new BytesRef("foo"))); writer.addDocument(doc); writer.close(); dir.close(); }
100. TestDocValuesIndexing#testTypeChangeAfterCloseAndDeleteAll()
Project: lucene-solr
File: TestDocValuesIndexing.java
File: TestDocValuesIndexing.java
public void testTypeChangeAfterCloseAndDeleteAll() throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); doc.add(new NumericDocValuesField("dv", 0L)); writer.addDocument(doc); writer.close(); conf = newIndexWriterConfig(new MockAnalyzer(random())); writer = new IndexWriter(dir, conf); writer.deleteAll(); doc = new Document(); doc.add(new SortedDocValuesField("dv", new BytesRef("foo"))); writer.addDocument(doc); writer.close(); dir.close(); }