Here are the examples of the java api class org.apache.accumulo.core.security.Authorizations taken from open source projects.
1. TestAccumuloStorageOptions#testAuths()
Project: pig
File: TestAccumuloStorageOptions.java
File: TestAccumuloStorageOptions.java
@Test public void testAuths() throws ParseException, IOException { AbstractAccumuloStorage storage = TestAbstractAccumuloStorage.getAbstractAccumuloStorage("", "--authorizations auth1,auth2"); Authorizations auths = new Authorizations("auth1,auth2"); Assert.assertEquals(auths, storage.authorizations); storage = TestAbstractAccumuloStorage.getAbstractAccumuloStorage("", "-auths auth1,auth2"); Assert.assertEquals(auths, storage.authorizations); }
2. TestAbstractAccumuloStorage#getDefaultExpectedLoadJob()
Project: pig
File: TestAbstractAccumuloStorage.java
File: TestAbstractAccumuloStorage.java
public Job getDefaultExpectedLoadJob() throws IOException { String inst = "myinstance"; String zookeepers = "127.0.0.1:2181"; String user = "root"; String password = "secret"; String table = "table1"; String start = "abc"; String end = "z"; Authorizations authorizations = new Authorizations("PRIVATE,PUBLIC".split(",")); List<Pair<Text, Text>> columnFamilyColumnQualifierPairs = new LinkedList<Pair<Text, Text>>(); columnFamilyColumnQualifierPairs.add(new Pair<Text, Text>(new Text("col1"), new Text("cq1"))); columnFamilyColumnQualifierPairs.add(new Pair<Text, Text>(new Text("col2"), new Text("cq2"))); columnFamilyColumnQualifierPairs.add(new Pair<Text, Text>(new Text("col3"), null)); Job expected = getExpectedLoadJob(inst, zookeepers, user, password, table, start, end, authorizations, columnFamilyColumnQualifierPairs); return expected; }
3. AccumuloGeoTableTest#testGetGeoTables()
Project: mrgeo
File: AccumuloGeoTableTest.java
File: AccumuloGeoTableTest.java
//@Ignore @Test @Category(UnitTest.class) public void testGetGeoTables() throws Exception { ZooKeeperInstance zkinst = new ZooKeeperInstance(inst, zoo); PasswordToken pwTok = new PasswordToken(pw.getBytes()); Connector conn = zkinst.getConnector(u, pwTok); Assert.assertNotNull(conn); PasswordToken token = new PasswordToken(pw.getBytes()); //Authorizations auths = new Authorizations(authsStr.split(",")); Authorizations auths = new Authorizations("A,B,C,D,ROLE_USER,U".split(",")); System.out.println(auths.toString()); Hashtable<String, String> ht = AccumuloUtils.getGeoTables(null, token, auths, conn); for (String k : ht.keySet()) { System.out.println(k + " => " + ht.get(k)); } }
4. AccumuloRdfConfigurationTest#testAuths()
Project: incubator-rya
File: AccumuloRdfConfigurationTest.java
File: AccumuloRdfConfigurationTest.java
@Test public void testAuths() { String[] arr = { "U", "FOUO" }; String str = "U,FOUO"; Authorizations auths = new Authorizations(arr); AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setAuths(arr); assertTrue(Arrays.equals(arr, conf.getAuths())); assertEquals(str, conf.getAuth()); assertEquals(auths, conf.getAuthorizations()); conf.setAuth(str); assertTrue(Arrays.equals(arr, conf.getAuths())); assertEquals(str, conf.getAuth()); assertEquals(auths, conf.getAuthorizations()); }
5. BasicAccumuloOperations#insureAuthorization()
Project: geowave
File: BasicAccumuloOperations.java
File: BasicAccumuloOperations.java
@Override public void insureAuthorization(final String clientUser, final String... authorizations) throws AccumuloException, AccumuloSecurityException { Authorizations auths = connector.securityOperations().getUserAuthorizations(clientUser); final List<byte[]> newSet = new ArrayList<byte[]>(); for (final String auth : authorizations) { if (!auths.contains(auth)) { newSet.add(auth.getBytes(StringUtils.GEOWAVE_CHAR_SET)); } } if (newSet.size() > 0) { newSet.addAll(auths.getAuthorizations()); connector.securityOperations().changeUserAuthorizations(clientUser, new Authorizations(newSet)); auths = connector.securityOperations().getUserAuthorizations(clientUser); LOGGER.trace(clientUser + " has authorizations " + ArrayUtils.toString(auths.getAuthorizations())); } }
6. AccumuloTestHelper#getAccumuloTuple()
Project: apex-malhar
File: AccumuloTestHelper.java
File: AccumuloTestHelper.java
public static AccumuloTuple getAccumuloTuple(String row, String colFam, String colName) { Authorizations auths = new Authorizations(); Scanner scan = null; try { scan = con.createScanner("tab1", auths); } catch (TableNotFoundException e) { logger.error("error in test helper"); DTThrowable.rethrow(e); } scan.setRange(new Range(new Text(row))); scan.fetchColumn(new Text(colFam), new Text(colName)); // assuming only one row for (Entry<Key, Value> entry : scan) { AccumuloTuple tuple = new AccumuloTuple(); tuple.setRow(entry.getKey().getRow().toString()); tuple.setColFamily(entry.getKey().getColumnFamily().toString()); tuple.setColName(entry.getKey().getColumnQualifier().toString()); tuple.setColValue(entry.getValue().toString()); return tuple; } return null; }
7. AccumuloWindowStore#getCommittedWindowId()
Project: apex-malhar
File: AccumuloWindowStore.java
File: AccumuloWindowStore.java
@Override public long getCommittedWindowId(String appId, int operatorId) { byte[] value = null; Authorizations auths = new Authorizations(); Scanner scan = null; String columnKey = appId + "_" + operatorId + "_" + lastWindowColumnName; lastWindowColumnBytes = columnKey.getBytes(); try { scan = connector.createScanner(tableName, auths); } catch (TableNotFoundException e) { logger.error("error getting committed window id", e); DTThrowable.rethrow(e); } scan.setRange(new Range(new Text(rowBytes))); scan.fetchColumn(new Text(columnFamilyBytes), new Text(lastWindowColumnBytes)); for (Entry<Key, Value> entry : scan) { value = entry.getValue().get(); } if (value != null) { long longval = toLong(value); return longval; } return -1; }
8. AccumuloGeoTableTest#testGetTile()
Project: mrgeo
File: AccumuloGeoTableTest.java
File: AccumuloGeoTableTest.java
// end testConnectorAll @Test @Category(UnitTest.class) public void testGetTile() throws Exception { ZooKeeperInstance zkinst = new ZooKeeperInstance(inst, zoo); PasswordToken pwTok = new PasswordToken(pw.getBytes()); Connector conn = zkinst.getConnector(u, pwTok); Assert.assertNotNull(conn); PasswordToken token = new PasswordToken(pw.getBytes()); Authorizations auths = new Authorizations(authsStr.split(",")); long start = 0; long end = Long.MAX_VALUE; Key sKey = AccumuloUtils.toKey(start); Key eKey = AccumuloUtils.toKey(end); Range r = new Range(sKey, eKey); Scanner s = conn.createScanner("paris4", auths); s.fetchColumnFamily(new Text(Integer.toString(10))); s.setRange(r); Iterator<Entry<Key, Value>> it = s.iterator(); while (it.hasNext()) { Entry<Key, Value> ent = it.next(); if (ent == null) { return; } System.out.println("current key = " + AccumuloUtils.toLong(ent.getKey().getRow())); System.out.println("current value = " + ent.getValue().getSize()); } }
9. AccumuloUtils#createAuthorizationsFromDelimitedString()
Project: mrgeo
File: AccumuloUtils.java
File: AccumuloUtils.java
public static Authorizations createAuthorizationsFromDelimitedString(String auths) { Authorizations authorizations = null; if (auths == null || auths.isEmpty()) { authorizations = new Authorizations(); } else { authorizations = new Authorizations(auths.split(",")); } return authorizations; }
10. AccumuloUtils#getGeoTables()
Project: mrgeo
File: AccumuloUtils.java
File: AccumuloUtils.java
// end public static Hashtable<String, String> getGeoTables(Properties providerProperties) throws DataProviderException { Properties props = AccumuloConnector.getAccumuloProperties(); if (providerProperties != null) { props.putAll(providerProperties); } String strAuths = props.getProperty(DataProviderFactory.PROVIDER_PROPERTY_USER_ROLES); if (strAuths == null) { strAuths = props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_AUTHS); } Authorizations auths = createAuthorizationsFromDelimitedString(strAuths); Connector conn; try { //conn = AccumuloConnector.getConnector(); Instance inst = new ZooKeeperInstance(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_INSTANCE), props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_ZOOKEEPERS)); AuthenticationToken token = new PasswordToken(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD).getBytes()); conn = inst.getConnector(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_USER), token); } catch (AccumuloSecurityException ase) { ase.printStackTrace(); return null; } catch (AccumuloException ae) { ae.printStackTrace(); return null; } //catch (DataProviderException dpe) { //dpe.printStackTrace(); //return null; //} // TCredentials tcr = new TCredentials(); // tcr.setInstanceId(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_INSTANCE)); // tcr.setPrincipal(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_USER)); // tcr.setToken(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD).getBytes()); AuthenticationToken at = new PasswordToken(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD).getBytes()); return getGeoTables(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_EXCLUDETABLES), at, auths, conn); }
11. AccumuloUtils#getRaster()
Project: mrgeo
File: AccumuloUtils.java
File: AccumuloUtils.java
// end putMetadataIntoTable /* * End Metadata utilities */ /** * Get a single raster * * @param table this is the table containing the raster * @param tid the tile id to get * @param zl the zoom level of the raster * @param conn the Accumulo connector to use * @param auths the authorizations to use for access * @return */ @Deprecated public static RasterWritable getRaster(String table, long tid, int zl, Connector conn, String auths) { RasterWritable retRaster = null; Authorizations authorizations = createAuthorizationsFromDelimitedString(auths); Scanner scanner = null; try { scanner = conn.createScanner(table, authorizations); Range r = new Range(toRowId(tid), toRowId(tid + 1)); scanner.setRange(r); scanner.fetchColumnFamily(new Text(Integer.toString(zl).getBytes())); for (Entry<Key, Value> entry : scanner) { System.out.println("Key: " + entry.getKey().toString()); retRaster = new RasterWritable(entry.getValue().get()); break; } } catch (TableNotFoundException tnfe) { } return retRaster; }
12. AccumuloUtils#buildMetadataFromTable()
Project: mrgeo
File: AccumuloUtils.java
File: AccumuloUtils.java
// end getMetadataFromTable /** * This method will rebuild the Metadata object from the items stored in an Accumulo table. * * @param table the table to pull metadata from * @param conn is the Accumulo connector * @param auths is the authorizations to use when pulling metadata * @return the metadata object */ @Deprecated public static MrsPyramidMetadata buildMetadataFromTable(String table, Connector conn, String auths) { MrsPyramidMetadata retMeta = new MrsPyramidMetadata(); Authorizations authorizations = createAuthorizationsFromDelimitedString(auths); Scanner scanner = null; try { scanner = conn.createScanner(table, authorizations); Range r = new Range(new Text(MrGeoAccumuloConstants.MRGEO_ACC_METADATA)); scanner.setRange(r); for (Entry<Key, Value> entry : scanner) { System.out.println("Key: " + entry.getKey().toString()); String cf = entry.getKey().getColumnFamily().toString(); // parse the values if (cf.equals("zoomLevel")) { // looking at the maximum zoom level retMeta.setMaxZoomLevel(Integer.parseInt(entry.getValue().toString())); } else if (cf.equals("bounds")) { // looking at the geospatial bounds of the data Bounds bounds = Bounds.fromCommaString(entry.getValue().toString()); retMeta.setBounds(bounds); } else if (cf.equals("tileBounds")) { // looking at the tile bounds for a zoom level int curZL = Integer.parseInt(entry.getKey().getColumnQualifier().toString()); String[] vals = entry.getValue().toString().split(","); LongRectangle lr = new LongRectangle(Long.parseLong(vals[0]), Long.parseLong(vals[1]), Long.parseLong(vals[2]), Long.parseLong(vals[3])); retMeta.setTileBounds(curZL, lr); } else if (cf.equals("pixelBounds")) { // looking at the pixel bounds int curZL = Integer.parseInt(entry.getKey().getColumnQualifier().toString()); String[] vals = entry.getValue().toString().split(","); LongRectangle lr = new LongRectangle(Long.parseLong(vals[0]), Long.parseLong(vals[1]), Long.parseLong(vals[2]), Long.parseLong(vals[3])); retMeta.setPixelBounds(curZL, lr); } else if (cf.equals("bands")) { // get the bands of the original image int b = Integer.parseInt(entry.getValue().toString()); retMeta.setBands(b); } else if (cf.equals("classification")) { // get the classification Classification c = Classification.valueOf(entry.getValue().toString()); retMeta.setClassification(c); } else if (cf.equals("defaultValues")) { // parse the double values String[] dVals = entry.getValue().toString().split(","); double[] dv = new double[dVals.length]; for (int x = 0; x < dVals.length; x++) { dv[x] = Double.parseDouble(dVals[x]); } retMeta.setDefaultValues(dv); } else if (cf.equals("imageString")) { int curZL = Integer.parseInt(entry.getKey().getColumnQualifier().toString()); retMeta.setName(curZL, entry.getValue().toString()); } else if (cf.equals("tileSize")) { retMeta.setTilesize(Integer.parseInt(entry.getValue().toString())); } else if (cf.equals("name")) { retMeta.setPyramid(entry.getValue().toString()); } } } catch (TableNotFoundException tnfe) { } return retMeta; }
13. AccumuloUtils#getMetadataFromTable()
Project: mrgeo
File: AccumuloUtils.java
File: AccumuloUtils.java
// end importDirectory /* * Start Metadata utilities */ /** * This method will decode the MrsImagePyramidMetadata object stored in Accumulo. * * @param table the table to pull metadata from * @param conn is the Accumulo connector * @param auths is the authorizations to use when pulling metadata * @return the metadata object */ public static MrsPyramidMetadata getMetadataFromTable(String table, Connector conn, String auths) { MrsPyramidMetadata retMeta = null; Authorizations authorizations = createAuthorizationsFromDelimitedString(auths); Scanner scanner = null; try { scanner = conn.createScanner(table, authorizations); Range r = new Range(new Text(MrGeoAccumuloConstants.MRGEO_ACC_METADATA)); scanner.setRange(r); scanner.fetchColumnFamily(new Text("all")); for (Entry<Key, Value> entry : scanner) { System.out.println("Key: " + entry.getKey().toString()); if (entry.getKey().getColumnFamily().toString().equals("all")) { retMeta = (MrsPyramidMetadata) Base64Utils.decodeToObject(entry.getValue().toString()); break; } } if (retMeta == null) { return buildMetadataFromTable(table, conn, auths); } } catch (TableNotFoundException tnfe) { } catch (ClassNotFoundException cnfe) { } catch (IOException ioe) { } return retMeta; }
14. AccumuloSelectivityEvalDAO#getAuths()
Project: incubator-rya
File: AccumuloSelectivityEvalDAO.java
File: AccumuloSelectivityEvalDAO.java
private Authorizations getAuths(RdfCloudTripleStoreConfiguration conf) { String[] auths = conf.getAuths(); Authorizations authorizations = null; if (auths == null || auths.length == 0) { authorizations = new Authorizations(); } else { authorizations = new Authorizations(auths); } return authorizations; }
15. AccumuloSelectivityEvalDAO#getTableSize()
Project: incubator-rya
File: AccumuloSelectivityEvalDAO.java
File: AccumuloSelectivityEvalDAO.java
public int getTableSize(RdfCloudTripleStoreConfiguration conf) throws TableNotFoundException { Authorizations authorizations = getAuths(conf); if (joinMap.containsKey("subjectpredicateobject" + DELIM + "FullTableCardinality")) { FullTableCardinality = joinMap.get("subjectpredicateobject" + DELIM + "FullTableCardinality").intValue(); return FullTableCardinality; } if (FullTableCardinality == 0) { Scanner joinScanner = connector.createScanner(tableLayoutStrategy.getSelectivity(), authorizations); joinScanner.setRange(Range.prefix(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"))); Iterator<Map.Entry<Key, Value>> iterator = joinScanner.iterator(); if (iterator.hasNext()) { Map.Entry<Key, Value> entry = iterator.next(); if (entry.getKey().getColumnFamily().toString().equals("FullTableCardinality")) { String Count = entry.getKey().getColumnQualifier().toString(); FullTableCardinality = Integer.parseInt(Count); } } if (FullTableCardinality == 0) { throw new RuntimeException("Table does not contain full cardinality"); } } return FullTableCardinality; }
16. ConfigUtils#createBatchScanner()
Project: incubator-rya
File: ConfigUtils.java
File: ConfigUtils.java
public static BatchScanner createBatchScanner(String tablename, Configuration conf) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { Connector connector = ConfigUtils.getConnector(conf); Authorizations auths = ConfigUtils.getAuthorizations(conf); Integer numThreads = null; if (conf instanceof RdfCloudTripleStoreConfiguration) numThreads = ((RdfCloudTripleStoreConfiguration) conf).getNumThreads(); else numThreads = conf.getInt(RdfCloudTripleStoreConfiguration.CONF_NUM_THREADS, 2); return connector.createBatchScanner(tablename, auths, numThreads); }
17. ConfigUtils#createScanner()
Project: incubator-rya
File: ConfigUtils.java
File: ConfigUtils.java
public static Scanner createScanner(String tablename, Configuration conf) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { Connector connector = ConfigUtils.getConnector(conf); Authorizations auths = ConfigUtils.getAuthorizations(conf); return connector.createScanner(tablename, auths); }
18. AccumuloRyaQueryEngine#query()
Project: incubator-rya
File: AccumuloRyaQueryEngine.java
File: AccumuloRyaQueryEngine.java
@Override public CloseableIterable<RyaStatement> query(BatchRyaQuery ryaQuery) throws RyaDAOException { Preconditions.checkNotNull(ryaQuery); Iterable<RyaStatement> stmts = ryaQuery.getQueries(); Preconditions.checkNotNull(stmts); //query configuration String[] auths = ryaQuery.getAuths(); final Authorizations authorizations = auths != null ? new Authorizations(auths) : configuration.getAuthorizations(); final Long ttl = ryaQuery.getTtl(); Long currentTime = ryaQuery.getCurrentTime(); Long maxResults = ryaQuery.getMaxResults(); Integer batchSize = ryaQuery.getBatchSize(); Integer numQueryThreads = ryaQuery.getNumQueryThreads(); String regexSubject = ryaQuery.getRegexSubject(); String regexPredicate = ryaQuery.getRegexPredicate(); String regexObject = ryaQuery.getRegexObject(); TableLayoutStrategy tableLayoutStrategy = configuration.getTableLayoutStrategy(); int maxRanges = ryaQuery.getMaxRanges(); //TODO: cannot span multiple tables here try { Collection<Range> ranges = new HashSet<Range>(); TABLE_LAYOUT layout = null; RyaURI context = null; TriplePatternStrategy strategy = null; for (RyaStatement stmt : stmts) { //TODO: This will be overwritten context = stmt.getContext(); strategy = ryaContext.retrieveStrategy(stmt); if (strategy == null) { throw new IllegalArgumentException("TriplePattern[" + stmt + "] not supported"); } Map.Entry<RdfCloudTripleStoreConstants.TABLE_LAYOUT, ByteRange> entry = strategy.defineRange(stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), stmt.getContext(), null); //use range to set scanner //populate scanner based on authorizations, ttl layout = entry.getKey(); ByteRange byteRange = entry.getValue(); Range range = new Range(new Text(byteRange.getStart()), new Text(byteRange.getEnd())); ranges.add(range); } //no ranges if (layout == null) throw new IllegalArgumentException("No table layout specified"); final TripleRowRegex tripleRowRegex = strategy.buildRegex(regexSubject, regexPredicate, regexObject, null, null); final String table = layoutToTable(layout, tableLayoutStrategy); boolean useBatchScanner = ranges.size() > maxRanges; FluentCloseableIterable<RyaStatement> results = null; if (useBatchScanner) { BatchScanner scanner = connector.createBatchScanner(table, authorizations, numQueryThreads); scanner.setRanges(ranges); fillScanner(scanner, context, null, ttl, null, tripleRowRegex, ryaQuery.getConf()); results = FluentCloseableIterable.from(new ScannerBaseCloseableIterable(scanner)).transform(keyValueToRyaStatementFunctionMap.get(layout)); } else { final RyaURI fcontext = context; final RdfCloudTripleStoreConfiguration fconf = ryaQuery.getConf(); FluentIterable<RyaStatement> fluent = FluentIterable.from(ranges).transformAndConcat(new Function<Range, Iterable<Map.Entry<Key, Value>>>() { @Override public Iterable<Map.Entry<Key, Value>> apply(Range range) { try { Scanner scanner = connector.createScanner(table, authorizations); scanner.setRange(range); fillScanner(scanner, fcontext, null, ttl, null, tripleRowRegex, fconf); return scanner; } catch (Exception e) { throw new RuntimeException(e); } } }).transform(keyValueToRyaStatementFunctionMap.get(layout)); results = FluentCloseableIterable.from(CloseableIterables.wrap(fluent)); } if (maxResults != null) { results = results.limit(maxResults.intValue()); } return results; } catch (Exception e) { throw new RyaDAOException(e); } }
19. AccumuloRyaQueryEngine#query()
Project: incubator-rya
File: AccumuloRyaQueryEngine.java
File: AccumuloRyaQueryEngine.java
@Override public CloseableIterable<RyaStatement> query(RyaQuery ryaQuery) throws RyaDAOException { Preconditions.checkNotNull(ryaQuery); RyaStatement stmt = ryaQuery.getQuery(); Preconditions.checkNotNull(stmt); //query configuration String[] auths = ryaQuery.getAuths(); Authorizations authorizations = auths != null ? new Authorizations(auths) : configuration.getAuthorizations(); Long ttl = ryaQuery.getTtl(); Long currentTime = ryaQuery.getCurrentTime(); Long maxResults = ryaQuery.getMaxResults(); Integer batchSize = ryaQuery.getBatchSize(); String regexSubject = ryaQuery.getRegexSubject(); String regexPredicate = ryaQuery.getRegexPredicate(); String regexObject = ryaQuery.getRegexObject(); TableLayoutStrategy tableLayoutStrategy = configuration.getTableLayoutStrategy(); try { //find triple pattern range TriplePatternStrategy strategy = ryaContext.retrieveStrategy(stmt); TABLE_LAYOUT layout; Range range; RyaURI subject = stmt.getSubject(); RyaURI predicate = stmt.getPredicate(); RyaType object = stmt.getObject(); RyaURI context = stmt.getContext(); String qualifier = stmt.getQualifer(); TripleRowRegex tripleRowRegex = null; if (strategy != null) { //otherwise, full table scan is supported Map.Entry<RdfCloudTripleStoreConstants.TABLE_LAYOUT, ByteRange> entry = strategy.defineRange(subject, predicate, object, context, null); layout = entry.getKey(); ByteRange byteRange = entry.getValue(); range = new Range(new Text(byteRange.getStart()), new Text(byteRange.getEnd())); byte[] objectTypeInfo = null; if (object != null) { //TODO: Not good to serialize this twice if (object instanceof RyaRange) { objectTypeInfo = RyaContext.getInstance().serializeType(((RyaRange) object).getStart())[1]; } else { objectTypeInfo = RyaContext.getInstance().serializeType(object)[1]; } } tripleRowRegex = strategy.buildRegex(regexSubject, regexPredicate, regexObject, null, objectTypeInfo); } else { range = new Range(); layout = TABLE_LAYOUT.SPO; } //use range to set scanner //populate scanner based on authorizations, ttl String table = layoutToTable(layout, tableLayoutStrategy); Scanner scanner = connector.createScanner(table, authorizations); scanner.setRange(range); if (batchSize != null) { scanner.setBatchSize(batchSize); } fillScanner(scanner, context, qualifier, ttl, currentTime, tripleRowRegex, ryaQuery.getConf()); FluentCloseableIterable<RyaStatement> results = FluentCloseableIterable.from(new ScannerBaseCloseableIterable(scanner)).transform(keyValueToRyaStatementFunctionMap.get(layout)); if (maxResults != null) { results = results.limit(maxResults.intValue()); } return results; } catch (Exception e) { throw new RyaDAOException(e); } }
20. AccumuloRyaQueryEngine#queryWithBindingSet()
Project: incubator-rya
File: AccumuloRyaQueryEngine.java
File: AccumuloRyaQueryEngine.java
@Override public CloseableIteration<? extends Map.Entry<RyaStatement, BindingSet>, RyaDAOException> queryWithBindingSet(Collection<Map.Entry<RyaStatement, BindingSet>> stmts, AccumuloRdfConfiguration conf) throws RyaDAOException { if (conf == null) { conf = configuration; } //query configuration Authorizations authorizations = conf.getAuthorizations(); Long ttl = conf.getTtl(); Long maxResults = conf.getLimit(); Integer maxRanges = conf.getMaxRangesForScanner(); Integer numThreads = conf.getNumThreads(); //TODO: cannot span multiple tables here try { Collection<Range> ranges = new HashSet<Range>(); RangeBindingSetEntries rangeMap = new RangeBindingSetEntries(); TABLE_LAYOUT layout = null; RyaURI context = null; TriplePatternStrategy strategy = null; for (Map.Entry<RyaStatement, BindingSet> stmtbs : stmts) { RyaStatement stmt = stmtbs.getKey(); //TODO: This will be overwritten context = stmt.getContext(); BindingSet bs = stmtbs.getValue(); strategy = ryaContext.retrieveStrategy(stmt); if (strategy == null) { throw new IllegalArgumentException("TriplePattern[" + stmt + "] not supported"); } Map.Entry<RdfCloudTripleStoreConstants.TABLE_LAYOUT, ByteRange> entry = strategy.defineRange(stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), stmt.getContext(), conf); //use range to set scanner //populate scanner based on authorizations, ttl layout = entry.getKey(); ByteRange byteRange = entry.getValue(); Range range = new Range(new Text(byteRange.getStart()), new Text(byteRange.getEnd())); ranges.add(range); rangeMap.ranges.add(new RdfCloudTripleStoreUtils.CustomEntry<Range, BindingSet>(range, bs)); } //no ranges if (layout == null) return null; String regexSubject = conf.getRegexSubject(); String regexPredicate = conf.getRegexPredicate(); String regexObject = conf.getRegexObject(); TripleRowRegex tripleRowRegex = strategy.buildRegex(regexSubject, regexPredicate, regexObject, null, null); String table = layoutToTable(layout, conf); boolean useBatchScanner = ranges.size() > maxRanges; RyaStatementBindingSetKeyValueIterator iterator = null; if (useBatchScanner) { ScannerBase scanner = connector.createBatchScanner(table, authorizations, numThreads); ((BatchScanner) scanner).setRanges(ranges); fillScanner(scanner, context, null, ttl, null, tripleRowRegex, conf); iterator = new RyaStatementBindingSetKeyValueIterator(layout, ryaContext, scanner, rangeMap); } else { Scanner scannerBase = null; Iterator<Map.Entry<Key, Value>>[] iters = new Iterator[ranges.size()]; int i = 0; for (Range range : ranges) { scannerBase = connector.createScanner(table, authorizations); scannerBase.setRange(range); fillScanner(scannerBase, context, null, ttl, null, tripleRowRegex, conf); iters[i] = scannerBase.iterator(); i++; } iterator = new RyaStatementBindingSetKeyValueIterator(layout, Iterators.concat(iters), rangeMap, ryaContext); } if (maxResults != null) { iterator.setMaxResults(maxResults); } return iterator; } catch (Exception e) { throw new RyaDAOException(e); } }
21. MapReduceIT#testIngestOSMPBF()
Project: geowave
File: MapReduceIT.java
File: MapReduceIT.java
@Test public void testIngestOSMPBF() throws Exception { TestUtils.deleteAll(dataStoreOptions); // NOTE: This will probably fail unless you bump up the memory for the // tablet // servers, for whatever reason, using the // miniAccumuloConfig.setMemory() function. MapReduceTestEnvironment mrEnv = MapReduceTestEnvironment.getInstance(); // TODO: for now this only works with accumulo, generalize the data // store usage AccumuloStoreTestEnvironment accumuloEnv = AccumuloStoreTestEnvironment.getInstance(); String hdfsPath = mrEnv.getHdfsBaseDirectory() + "/osm_stage/"; StageOSMToHDFSCommand stage = new StageOSMToHDFSCommand(); stage.setParameters(TEST_DATA_BASE_DIR, mrEnv.getHdfs(), hdfsPath); stage.execute(new ManualOperationParams()); Connector conn = new ZooKeeperInstance(accumuloEnv.getAccumuloInstance(), accumuloEnv.getZookeeper()).getConnector(accumuloEnv.getAccumuloUser(), new PasswordToken(accumuloEnv.getAccumuloPassword())); Authorizations auth = new Authorizations(new String[] { "public" }); conn.securityOperations().changeUserAuthorizations(accumuloEnv.getAccumuloUser(), auth); IngestOSMToGeoWaveCommand ingest = new IngestOSMToGeoWaveCommand(); ingest.setParameters(mrEnv.getHdfs(), hdfsPath, null); ingest.setInputStoreOptions(dataStoreOptions); ingest.getIngestOptions().setJobName("ConversionTest"); // Execute for node's ways, and relations. ingest.getIngestOptions().setMapperType("NODE"); ingest.execute(new ManualOperationParams()); System.out.println("finished accumulo ingest Node"); ingest.getIngestOptions().setMapperType("WAY"); ingest.execute(new ManualOperationParams()); System.out.println("finished accumulo ingest Way"); ingest.getIngestOptions().setMapperType("RELATION"); ingest.execute(new ManualOperationParams()); System.out.println("finished accumulo ingest Relation"); }