Here are the examples of the java api class com.datatorrent.lib.logs.ApacheLogParseMapOutputOperator taken from open source projects.
1. ApplicationLocalLog#populateDAG()
Project: apex-malhar
File: ApplicationLocalLog.java
File: ApplicationLocalLog.java
@Override public void populateDAG(DAG dag, Configuration conf) { TailFsInputOperator log = dag.addOperator("log", new TailFsInputOperator()); log.setDelimiter('\n'); log.setFilePath("/var/log/apache2/access.log"); ApacheLogParseMapOutputOperator parse = dag.addOperator("parse", new ApacheLogParseMapOutputOperator()); GeoIPExtractor geoIPExtractor = new GeoIPExtractor(); // Can't put this file in resources until licensing issue is straightened out geoIPExtractor.setDatabasePath("/home/david/GeoLiteCity.dat"); parse.registerInformationExtractor("ip", geoIPExtractor); parse.registerInformationExtractor("agent", new UserAgentExtractor()); TimestampExtractor timestampExtractor = new TimestampExtractor(); timestampExtractor.setDateFormatString("dd/MMM/yyyy:HH:mm:ss Z"); parse.registerInformationExtractor("time", timestampExtractor); ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator()); dag.addStream("log-parse", log.output, parse.data); dag.addStream("parse-console", parse.output, console.input).setLocality(Locality.CONTAINER_LOCAL); }
2. ApplicationLogGenerator#populateDAG()
Project: apex-malhar
File: ApplicationLogGenerator.java
File: ApplicationLogGenerator.java
// private void setLibraryJars(DAG dag) // { // List<Class<?>> containingJars = new ArrayList<Class<?>>(); // containingJars.add(com.maxmind.geoip.LookupService.class); // containingJars.add(net.sf.uadetector.UserAgentStringParser.class); // containingJars.add(net.sf.uadetector.service.UADetectorServiceFactory.class); // containingJars.add(net.sf.qualitycheck.Check.class); // // String oldlibjar = dag.getValue(Context.DAGContext.LIBRARY_JARS); // if (oldlibjar == null) { // oldlibjar = ""; // } // // StringBuilder libjars = new StringBuilder(oldlibjar); // for (Class<?> clazz : containingJars) { // if (libjars.length() != 0) { // libjars.append(","); // } // libjars.append(clazz.getProtectionDomain().getCodeSource().getLocation().toString()); // } // dag.setAttribute(Context.DAGContext.LIBRARY_JARS, libjars.toString()); // } @Override public void populateDAG(DAG dag, Configuration conf) { // The code left here so that when a package gets built out of this, the builder scambles // a tiny bit less to identify the jars // setLibraryJars(dag); ApacheLogInputGenerator log = dag.addOperator("log", new ApacheLogInputGenerator()); log.setIpAddressFile("/com/datatorrent/contrib/apachelog/ipaddress.txt"); log.setUrlFile("/com/datatorrent/contrib/apachelog/urls.txt"); log.setAgentFile("/com/datatorrent/contrib/apachelog/agents.txt"); log.setRefererFile("/com/datatorrent/contrib/apachelog/referers.txt"); ApacheLogParseMapOutputOperator parse = dag.addOperator("parse", new ApacheLogParseMapOutputOperator()); GeoIPExtractor geoIPExtractor = new GeoIPExtractor(); // Can't put this file in resources until licensing issue is straightened out geoIPExtractor.setDatabasePath("/home/david/GeoLiteCity.dat"); parse.registerInformationExtractor("ip", geoIPExtractor); parse.registerInformationExtractor("agent", new UserAgentExtractor()); TimestampExtractor timestampExtractor = new TimestampExtractor(); timestampExtractor.setDateFormatString("dd/MMM/yyyy:HH:mm:ss Z"); parse.registerInformationExtractor("time", timestampExtractor); ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator()); dag.addStream("log-parse", log.output, parse.data); dag.addStream("parse-console", parse.output, console.input).setLocality(Locality.CONTAINER_LOCAL); }