Here are the examples of the java api org.apache.flink.api.common.Plan taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
313 Examples
19
Source : TestEnvironment.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
private OptimizedPlan compileProgram(String jobName) {
Plan p = createProgramPlan(jobName);
Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
return pc.compile(p);
}
19
Source : WorksetIterationsRecordApiCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testRecordApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
Plan plan = getTestPlan(false, false);
OptimizedPlan oPlan;
try {
oPlan = compileNoStats(plan);
} catch (CompilerException ce) {
ce.printStackTrace();
fail("The pact compiler is unable to compile this plan correctly.");
// silence the compiler
return;
}
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
// iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
// the in-loop parreplacedioning is before the final reducer
// verify joinWithInvariant
replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
replacedertEquals(list0, joinWithInvariantNode.getKeysForInput1());
replacedertEquals(list0, joinWithInvariantNode.getKeysForInput2());
// verify joinWithSolutionSet
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
// verify reducer
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
replacedertEquals(list0, worksetReducer.getKeys(0));
// verify solution delta
replacedertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
new JobGraphGenerator().compileJobGraph(oPlan);
}
19
Source : WorksetIterationsRecordApiCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testRecordApiWithDirectSoltionSetUpdate() {
Plan plan = getTestPlan(true, false);
OptimizedPlan oPlan;
try {
oPlan = compileNoStats(plan);
} catch (CompilerException ce) {
ce.printStackTrace();
fail("The pact compiler is unable to compile this plan correctly.");
// silence the compiler
return;
}
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
// iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
// the in-loop parreplacedioning is before the final reducer
// verify joinWithInvariant
replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
replacedertEquals(list0, joinWithInvariantNode.getKeysForInput1());
replacedertEquals(list0, joinWithInvariantNode.getKeysForInput2());
// verify joinWithSolutionSet
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
// verify reducer
replacedertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
replacedertEquals(list0, worksetReducer.getKeys(0));
// verify solution delta
replacedertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
new JobGraphGenerator().compileJobGraph(oPlan);
}
19
Source : CompilerTestBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
public static OperatorResolver getContractResolver(Plan plan) {
return new OperatorResolver(plan);
}
19
Source : CompilerTestBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// ------------------------------------------------------------------------
public OptimizedPlan compileWithStats(Plan p) {
return this.withStatsCompiler.compile(p);
}
19
Source : CompilerTestBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
public OptimizedPlan compileNoStats(Plan p) {
return this.noStatsCompiler.compile(p);
}
19
Source : WorksetIterationsJavaApiCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testJavaApiWithDirectSoltionSetUpdate() {
try {
Plan plan = getJavaTestPlan(true, false);
OptimizedPlan oPlan = compileNoStats(plan);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
// iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
// the in-loop parreplacedioning is before the final reducer
// verify joinWithInvariant
replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
// verify joinWithSolutionSet
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
replacedertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
// verify reducer
replacedertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
replacedertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
// verify solution delta
replacedertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test errored: " + e.getMessage());
}
}
19
Source : WorksetIterationsJavaApiCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testJavaApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
try {
Plan plan = getJavaTestPlan(false, false);
OptimizedPlan oPlan = compileNoStats(plan);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
// iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
// the in-loop parreplacedioning is before the final reducer
// verify joinWithInvariant
replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
// verify joinWithSolutionSet
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
replacedertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
// verify reducer
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
replacedertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
// verify solution delta
replacedertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test errored: " + e.getMessage());
}
}
19
Source : Optimizer.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// ------------------------------------------------------------------------
// Compilation
// ------------------------------------------------------------------------
/**
* Translates the given program to an OptimizedPlan, where all nodes have their local strategy replacedigned
* and all channels have a shipping strategy replacedigned.
*
* For more details on the optimization phase, see the comments for
* {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpreplaced.OptimizerPostPreplaced)}.
*
* @param program The program to be translated.
* @return The optimized plan.
*
* @throws CompilerException
* Thrown, if the plan is invalid or the optimizer encountered an inconsistent
* situation during the compilation process.
*/
public OptimizedPlan compile(Plan program) throws CompilerException {
final OptimizerPostPreplaced postPreplaceder = getPostPreplacedFromPlan(program);
return compile(program, postPreplaceder);
}
19
Source : Optimizer.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// ------------------------------------------------------------------------
// Miscellaneous
// ------------------------------------------------------------------------
private OptimizerPostPreplaced getPostPreplacedFromPlan(Plan program) {
final String clreplacedName = program.getPostPreplacedClreplacedName();
if (clreplacedName == null) {
throw new CompilerException("Optimizer Post Preplaced clreplaced description is null");
}
try {
Clreplaced<? extends OptimizerPostPreplaced> clazz = Clreplaced.forName(clreplacedName).replacedubclreplaced(OptimizerPostPreplaced.clreplaced);
try {
return InstantiationUtil.instantiate(clazz, OptimizerPostPreplaced.clreplaced);
} catch (RuntimeException rtex) {
// unwrap the source exception
if (rtex.getCause() != null) {
throw new CompilerException("Cannot instantiate optimizer post preplaced: " + rtex.getMessage(), rtex.getCause());
} else {
throw rtex;
}
}
} catch (ClreplacedNotFoundException cnfex) {
throw new CompilerException("Cannot load Optimizer post-preplaced clreplaced '" + clreplacedName + "'.", cnfex);
} catch (ClreplacedCastException ccex) {
throw new CompilerException("Clreplaced '" + clreplacedName + "' is not an optimizer post-preplaced.", ccex);
}
}
19
Source : OperatorTranslation.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
public Plan translateToPlan(List<DataSink<?>> sinks, String jobName) {
List<GenericDataSinkBase<?>> planSinks = new ArrayList<>();
for (DataSink<?> sink : sinks) {
planSinks.add(translate(sink));
}
Plan p = new Plan(planSinks);
p.setJobName(jobName);
return p;
}
19
Source : ClusterClient.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException {
Logger log = LoggerFactory.getLogger(ClusterClient.clreplaced);
if (parallelism > 0 && p.getDefaultParallelism() <= 0) {
log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism);
p.setDefaultParallelism(parallelism);
}
log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism());
return compiler.compile(p);
}
19
Source : LocalExecutor.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Executes the given dataflow plan.
*
* @param plan The dataflow plan.
* @return The execution result.
*
* @throws Exception Thrown, if either the startup of the local execution context, or the execution
* caused an exception.
*/
public static JobExecutionResult execute(Plan plan) throws Exception {
return new LocalExecutor().executePlan(plan);
}
19
Source : LocalExecutor.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Creates a JSON representation of the given dataflow's execution plan.
*
* @param plan The dataflow plan.
* @return The dataflow's execution plan, as a JSON string.
* @throws Exception Thrown, if the optimization process that creates the execution plan failed.
*/
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
pc.setDefaultParallelism(parallelism);
OptimizedPlan op = pc.compile(plan);
return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
19
Source : JobGraphGeneratorTest.java
with Apache License 2.0
from flink-tpc-ds
with Apache License 2.0
from flink-tpc-ds
private static JobGraph compileJob(ExecutionEnvironment env) {
Plan plan = env.createProgramPlan();
Optimizer pc = new Optimizer(new Configuration());
OptimizedPlan op = pc.compile(plan);
JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
18
Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// ------------------------------------------------------------------------
/**
* Verifies that a robust reparreplacedioning plan with a hash join is created in the absence of statistics.
*/
@Test
public void testQueryNoStatistics() {
try {
Plan p = getTPCH3Plan();
p.setExecutionConfig(defaultExecutionConfig);
// compile
final OptimizedPlan plan = compileNoStats(p);
final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
// get the nodes from the final plan
final SinkPlanNode sink = or.getNode(SINK);
final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ? (SingleInputPlanNode) reducer.getPredecessor() : null;
final DualInputPlanNode join = or.getNode(JOIN_NAME);
final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);
// verify the optimizer choices
checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
replacedert.replacedertTrue(checkReparreplacedionShipStrategies(join, reducer, combiner));
replacedert.replacedertTrue(checkHashJoinStrategies(join, reducer, true) || checkHashJoinStrategies(join, reducer, false));
} catch (Exception e) {
e.printStackTrace();
replacedert.fail(e.getMessage());
}
}
18
Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Statistics that push towards a reparreplacedion merge join. If the join blows the data volume up significantly,
* re-exploiting the sorted order is cheaper.
*/
@Test
public void testQueryWithStatsForReparreplacedionMerge() {
Plan p = getTPCH3Plan();
p.setExecutionConfig(defaultExecutionConfig);
// set compiler hints
OperatorResolver cr = getContractResolver(p);
DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
match.getCompilerHints().setFilterFactor(100f);
testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}
18
Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
private void testQueryGeneric(Plan p, long orderSize, long lineitemSize, float orderSelectivity, float joinSelectivity, boolean broadcastOkay, boolean parreplacedionedOkay, boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
try {
// set statistics
OperatorResolver cr = getContractResolver(p);
GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
setSourceStatistics(ordersSource, orderSize, 100f);
setSourceStatistics(lineItemSource, lineitemSize, 140f);
mapper.getCompilerHints().setAvgOutputRecordSize(16f);
mapper.getCompilerHints().setFilterFactor(orderSelectivity);
joiner.getCompilerHints().setFilterFactor(joinSelectivity);
// compile
final OptimizedPlan plan = compileWithStats(p);
final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
// get the nodes from the final plan
final SinkPlanNode sink = or.getNode(SINK);
final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ? (SingleInputPlanNode) reducer.getPredecessor() : null;
final DualInputPlanNode join = or.getNode(JOIN_NAME);
final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);
checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
// check the possible variants and that the variant ia allowed in this specific setting
if (checkBroadcastShipStrategies(join, reducer, combiner)) {
replacedert.replacedertTrue("Broadcast join incorrectly chosen.", broadcastOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
replacedert.replacedertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
replacedert.replacedertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkBroadcastMergeJoin(join, reducer)) {
replacedert.replacedertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
replacedert.fail("Plan has no correct hash join or merge join strategies.");
}
} else if (checkReparreplacedionShipStrategies(join, reducer, combiner)) {
replacedert.replacedertTrue("Parreplacedioned join incorrectly chosen.", parreplacedionedOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
replacedert.replacedertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
replacedert.replacedertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkReparreplacedionMergeJoin(join, reducer)) {
replacedert.replacedertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
replacedert.fail("Plan has no correct hash join or merge join strategies.");
}
} else {
replacedert.fail("Plan has neither correct BC join or parreplacedioned join configuration.");
}
} catch (Exception e) {
e.printStackTrace();
replacedert.fail(e.getMessage());
}
}
18
Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// ------------------------------------------------------------------------
private void testQueryGeneric(long orderSize, long lineItemSize, float ordersFilterFactor, float joinFilterFactor, boolean broadcastOkay, boolean parreplacedionedOkay, boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
Plan p = getTPCH3Plan();
p.setExecutionConfig(defaultExecutionConfig);
testQueryGeneric(p, orderSize, lineItemSize, ordersFilterFactor, joinFilterFactor, broadcastOkay, parreplacedionedOkay, hashJoinFirstOkay, hashJoinSecondOkay, mergeJoinOkay);
}
18
Source : KMeansSingleStepTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testCompileKMeansSingleStepWithOutStats() {
Plan p = getKMeansPlan();
p.setExecutionConfig(new ExecutionConfig());
OptimizedPlan plan = compileNoStats(p);
checkPlan(plan);
}
18
Source : CancelingTestBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
private JobGraph getJobGraph(final Plan plan) {
final Optimizer pc = new Optimizer(new DataStatistics(), getConfiguration());
final OptimizedPlan op = pc.compile(plan);
final JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
18
Source : AccumulatorLiveITCase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Helpers to generate the JobGraph.
*/
private static JobGraph getJobGraph(Plan plan) {
Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
JobGraphGenerator jgg = new JobGraphGenerator();
OptimizedPlan op = pc.compile(plan);
return jgg.compileJobGraph(op);
}
18
Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
*/
@Test
public void testRightSideCountercheck() {
try {
Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
OptimizedPlan oPlan = compileNoStats(plan);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
// verify correct join strategy
replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy());
replacedertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
replacedertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test errored: " + e.getMessage());
}
}
18
Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* This tests whether a HYBRIDHASH_BUILD_FIRST is correctly transformed to a HYBRIDHASH_BUILD_FIRST_CACHED
* when inside of an iteration an on the static path
*/
@Test
public void testLeftSide() {
try {
Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
OptimizedPlan oPlan = compileNoStats(plan);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
// verify correct join strategy
replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED, innerJoin.getDriverStrategy());
replacedertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
replacedertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test errored: " + e.getMessage());
}
}
18
Source : OptimizedPlan.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* The execution plan generated by the Optimizer. It contains {@link PlanNode}s
* and {@link Channel}s that describe exactly how the program should be executed.
*
* <p>The optimized plan defines all ship strategies (local pipe, shuffle, broadcast, rebalance),
* all operator strategies (sorting-merge join, hash join, sorted grouping, ...),
* and the data exchange modes (batched, pipelined).</p>
*/
public clreplaced OptimizedPlan implements FlinkPlan, Visitable<PlanNode> {
/**
* The data sources in the plan.
*/
private final Collection<SourcePlanNode> dataSources;
/**
* The data sinks in the plan.
*/
private final Collection<SinkPlanNode> dataSinks;
/**
* All nodes in the optimizer plan.
*/
private final Collection<PlanNode> allNodes;
/**
* The original program (as a dataflow plan).
*/
private final Plan originalProgram;
/**
* Name of the job
*/
private final String jobName;
/**
* Creates a new instance of this optimizer plan container. The plan is given and fully
* described by the data sources, sinks and the collection of all nodes.
*
* @param sources The data sources.
* @param sinks The data sinks.
* @param allNodes A collection containing all nodes in the plan.
* @param jobName The name of the program
*/
public OptimizedPlan(Collection<SourcePlanNode> sources, Collection<SinkPlanNode> sinks, Collection<PlanNode> allNodes, String jobName, Plan programPlan) {
this.dataSources = sources;
this.dataSinks = sinks;
this.allNodes = allNodes;
this.jobName = jobName;
this.originalProgram = programPlan;
}
/**
* Gets the data sources from this OptimizedPlan.
*
* @return The data sources.
*/
public Collection<SourcePlanNode> getDataSources() {
return dataSources;
}
/**
* Gets the data sinks from this OptimizedPlan.
*
* @return The data sinks.
*/
public Collection<SinkPlanNode> getDataSinks() {
return dataSinks;
}
/**
* Gets all the nodes from this OptimizedPlan.
*
* @return All nodes.
*/
public Collection<PlanNode> getAllNodes() {
return allNodes;
}
/**
* Returns the name of the program.
*
* @return The name of the program.
*/
public String getJobName() {
return this.jobName;
}
/**
* Gets the original program's dataflow plan from which this optimized plan was created.
*
* @return The original program's dataflow plan.
*/
public Plan getOriginalPlan() {
return this.originalProgram;
}
// ------------------------------------------------------------------------
/**
* Applies the given visitor top down to all nodes, starting at the sinks.
*
* @param visitor
* The visitor to apply to the nodes in this plan.
* @see org.apache.flink.util.Visitable#accept(org.apache.flink.util.Visitor)
*/
@Override
public void accept(Visitor<PlanNode> visitor) {
for (SinkPlanNode node : this.dataSinks) {
node.accept(visitor);
}
}
}
18
Source : Optimizer.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* This function performs only the first step to the compilation process - the creation of the optimizer
* representation of the plan. No estimations or enumerations of alternatives are done here.
*
* @param program The plan to generate the optimizer representation for.
* @return The optimizer representation of the plan, as a collection of all data sinks
* from the plan can be traversed.
*/
public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
program.accept(graphCreator);
return graphCreator.getSinks();
}
18
Source : NamesTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
private static void testForName(final String expected, Plan plan) {
plan.accept(new Visitor<Operator<?>>() {
@Override
public boolean preVisit(Operator<?> visitable) {
if (visitable instanceof PlanFilterOperator<?>) {
// cast is actually not required. Its just a check for the right element
PlanFilterOperator<?> filterOp = (PlanFilterOperator<?>) visitable;
replacedert.replacedertEquals(expected, filterOp.getName());
}
return true;
}
@Override
public void postVisit(Operator<?> visitable) {
//
}
});
}
18
Source : CollectionExecutor.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// --------------------------------------------------------------------------------------------
// General execution methods
// --------------------------------------------------------------------------------------------
public JobExecutionResult execute(Plan program) throws Exception {
long startTime = System.currentTimeMillis();
initCache(program.getCachedFiles());
Collection<? extends GenericDataSinkBase<?>> sinks = program.getDataSinks();
for (Operator<?> sink : sinks) {
execute(sink);
}
long endTime = System.currentTimeMillis();
Map<String, OptionalFailure<Object>> acreplacedulatorResults = AcreplacedulatorHelper.toResultMap(acreplacedulators);
return new JobExecutionResult(null, endTime - startTime, acreplacedulatorResults);
}
18
Source : RemoteExecutor.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
OptimizedPlan optPlan = opt.compile(plan);
return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan);
}
18
Source : RemoteExecutor.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// ------------------------------------------------------------------------
// Executing programs
// ------------------------------------------------------------------------
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
if (plan == null) {
throw new IllegalArgumentException("The plan may not be null.");
}
JobWithJars p = new JobWithJars(plan, this.jarFiles, this.globalClreplacedpaths);
return executePlanWithJars(p);
}
18
Source : PreviewPlanEnvironment.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Environment to extract the pre-optimized plan.
*/
public final clreplaced PreviewPlanEnvironment extends ExecutionEnvironment {
List<DataSinkNode> previewPlan;
String preview;
Plan plan;
@Override
public JobExecutionResult execute(String jobName) throws Exception {
this.plan = createProgramPlan(jobName);
this.previewPlan = Optimizer.createPreOptimizedPlan(plan);
// do not go on with anything now!
throw new OptimizerPlanEnvironment.ProgramAbortException();
}
@Override
public String getExecutionPlan() throws Exception {
Plan plan = createProgramPlan("unused");
this.previewPlan = Optimizer.createPreOptimizedPlan(plan);
// do not go on with anything now!
throw new OptimizerPlanEnvironment.ProgramAbortException();
}
@Override
public void startNewSession() {
}
public void setAsContext() {
ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {
@Override
public ExecutionEnvironment createExecutionEnvironment() {
return PreviewPlanEnvironment.this;
}
};
initializeContextEnvironment(factory);
}
public void unsetAsContext() {
resetContextEnvironment();
}
public void setPreview(String preview) {
this.preview = preview;
}
public Plan getPlan() {
return plan;
}
}
18
Source : PreviewPlanEnvironment.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Override
public String getExecutionPlan() throws Exception {
Plan plan = createProgramPlan("unused");
this.previewPlan = Optimizer.createPreOptimizedPlan(plan);
// do not go on with anything now!
throw new OptimizerPlanEnvironment.ProgramAbortException();
}
18
Source : OptimizerPlanEnvironment.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Override
public String getExecutionPlan() throws Exception {
Plan plan = createProgramPlan(null, false);
this.optimizerPlan = compiler.compile(plan);
// do not go on with anything now!
throw new ProgramAbortException();
}
18
Source : OptimizerPlanEnvironment.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
// ------------------------------------------------------------------------
// Execution Environment methods
// ------------------------------------------------------------------------
@Override
public JobExecutionResult execute(String jobName) throws Exception {
Plan plan = createProgramPlan(jobName);
this.optimizerPlan = compiler.compile(plan);
// do not go on with anything now!
throw new ProgramAbortException();
}
18
Source : JobWithJars.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* A JobWithJars is a Flink dataflow plan, together with a bunch of JAR files that contain
* the clreplacedes of the functions and libraries necessary for the execution.
*/
public clreplaced JobWithJars {
private Plan plan;
private List<URL> jarFiles;
/**
* clreplacedpaths that are needed during user code execution.
*/
private List<URL> clreplacedpaths;
private ClreplacedLoader userCodeClreplacedLoader;
public JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths) throws IOException {
this.plan = plan;
this.jarFiles = new ArrayList<URL>(jarFiles.size());
this.clreplacedpaths = new ArrayList<URL>(clreplacedpaths.size());
for (URL jarFile : jarFiles) {
checkJarFile(jarFile);
this.jarFiles.add(jarFile);
}
for (URL path : clreplacedpaths) {
this.clreplacedpaths.add(path);
}
}
public JobWithJars(Plan plan, URL jarFile) throws IOException {
this.plan = plan;
checkJarFile(jarFile);
this.jarFiles = Collections.singletonList(jarFile);
this.clreplacedpaths = Collections.<URL>emptyList();
}
JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths, ClreplacedLoader userCodeClreplacedLoader) {
this.plan = plan;
this.jarFiles = jarFiles;
this.clreplacedpaths = clreplacedpaths;
this.userCodeClreplacedLoader = userCodeClreplacedLoader;
}
/**
* Returns the plan.
*/
public Plan getPlan() {
return this.plan;
}
/**
* Returns list of jar files that need to be submitted with the plan.
*/
public List<URL> getJarFiles() {
return this.jarFiles;
}
/**
* Returns list of clreplacedpaths that need to be submitted with the plan.
*/
public List<URL> getClreplacedpaths() {
return clreplacedpaths;
}
/**
* Gets the {@link java.lang.ClreplacedLoader} that must be used to load user code clreplacedes.
*
* @return The user code ClreplacedLoader.
*/
public ClreplacedLoader getUserCodeClreplacedLoader() {
if (this.userCodeClreplacedLoader == null) {
this.userCodeClreplacedLoader = buildUserCodeClreplacedLoader(jarFiles, clreplacedpaths, getClreplaced().getClreplacedLoader());
}
return this.userCodeClreplacedLoader;
}
public static void checkJarFile(URL jar) throws IOException {
File jarFile;
try {
jarFile = new File(jar.toURI());
} catch (URISyntaxException e) {
throw new IOException("JAR file path is invalid '" + jar + "'");
}
if (!jarFile.exists()) {
throw new IOException("JAR file does not exist '" + jarFile.getAbsolutePath() + "'");
}
if (!jarFile.canRead()) {
throw new IOException("JAR file can't be read '" + jarFile.getAbsolutePath() + "'");
}
// TODO: Check if proper JAR file
}
public static ClreplacedLoader buildUserCodeClreplacedLoader(List<URL> jars, List<URL> clreplacedpaths, ClreplacedLoader parent) {
URL[] urls = new URL[jars.size() + clreplacedpaths.size()];
for (int i = 0; i < jars.size(); i++) {
urls[i] = jars.get(i);
}
for (int i = 0; i < clreplacedpaths.size(); i++) {
urls[i + jars.size()] = clreplacedpaths.get(i);
}
return FlinkUserCodeClreplacedLoaders.parentFirst(urls, parent);
}
}
18
Source : ContextEnvironment.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Override
public JobExecutionResult execute(String jobName) throws Exception {
Plan p = createProgramPlan(jobName);
JobWithJars toRun = new JobWithJars(p, this.jarFilesToAttach, this.clreplacedpathsToAttach, this.userCodeClreplacedLoader);
this.lastJobExecutionResult = client.run(toRun, getParallelism(), savepointSettings).getJobExecutionResult();
return this.lastJobExecutionResult;
}
18
Source : LocalExecutor.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Creates a JSON representation of the given dataflow plan.
*
* @param plan The dataflow plan.
* @return The dataflow plan (prior to optimization) as a JSON string.
*/
public static String getPlanAsJSON(Plan plan) {
List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
return new PlanJSONDumpGenerator().getPactPlanAsJSON(sinks);
}
18
Source : LocalExecutor.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Creates a JSON representation of the given dataflow's execution plan.
*
* @param plan The dataflow plan.
* @return The dataflow's execution plan, as a JSON string.
* @throws Exception Thrown, if the optimization process that creates the execution plan failed.
*/
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration);
pc.setDefaultParallelism(parallelism);
OptimizedPlan op = pc.compile(plan);
return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
18
Source : JobWithJars.java
with Apache License 2.0
from flink-tpc-ds
with Apache License 2.0
from flink-tpc-ds
/**
* A JobWithJars is a Flink dataflow plan, together with a bunch of JAR files that contain
* the clreplacedes of the functions and libraries necessary for the execution.
*/
public clreplaced JobWithJars {
private Plan plan;
private List<URL> jarFiles;
/**
* clreplacedpaths that are needed during user code execution.
*/
private List<URL> clreplacedpaths;
private ClreplacedLoader userCodeClreplacedLoader;
public JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths) throws IOException {
this.plan = plan;
this.jarFiles = new ArrayList<URL>(jarFiles.size());
this.clreplacedpaths = new ArrayList<URL>(clreplacedpaths.size());
for (URL jarFile : jarFiles) {
checkJarFile(jarFile);
this.jarFiles.add(jarFile);
}
for (URL path : clreplacedpaths) {
this.clreplacedpaths.add(path);
}
}
public JobWithJars(Plan plan, URL jarFile) throws IOException {
this.plan = plan;
checkJarFile(jarFile);
this.jarFiles = Collections.singletonList(jarFile);
this.clreplacedpaths = Collections.<URL>emptyList();
}
JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths, ClreplacedLoader userCodeClreplacedLoader) {
this.plan = plan;
this.jarFiles = jarFiles;
this.clreplacedpaths = clreplacedpaths;
this.userCodeClreplacedLoader = userCodeClreplacedLoader;
}
/**
* Returns the plan.
*/
public Plan getPlan() {
return this.plan;
}
/**
* Returns list of jar files that need to be submitted with the plan.
*/
public List<URL> getJarFiles() {
return this.jarFiles;
}
/**
* Returns list of clreplacedpaths that need to be submitted with the plan.
*/
public List<URL> getClreplacedpaths() {
return clreplacedpaths;
}
/**
* Gets the {@link java.lang.ClreplacedLoader} that must be used to load user code clreplacedes.
*
* @return The user code ClreplacedLoader.
*/
public ClreplacedLoader getUserCodeClreplacedLoader() {
if (this.userCodeClreplacedLoader == null) {
this.userCodeClreplacedLoader = buildUserCodeClreplacedLoader(jarFiles, clreplacedpaths, getClreplaced().getClreplacedLoader());
}
return this.userCodeClreplacedLoader;
}
public static void checkJarFile(URL jar) throws IOException {
File jarFile;
try {
jarFile = new File(jar.toURI());
} catch (URISyntaxException e) {
throw new IOException("JAR file path is invalid '" + jar + '\'');
}
if (!jarFile.exists()) {
throw new IOException("JAR file does not exist '" + jarFile.getAbsolutePath() + '\'');
}
if (!jarFile.canRead()) {
throw new IOException("JAR file can't be read '" + jarFile.getAbsolutePath() + '\'');
}
try (JarFile ignored = new JarFile(jarFile)) {
// verify that we can open the Jar file
} catch (IOException e) {
throw new IOException("Error while opening jar file '" + jarFile.getAbsolutePath() + '\'', e);
}
}
public static ClreplacedLoader buildUserCodeClreplacedLoader(List<URL> jars, List<URL> clreplacedpaths, ClreplacedLoader parent) {
URL[] urls = new URL[jars.size() + clreplacedpaths.size()];
for (int i = 0; i < jars.size(); i++) {
urls[i] = jars.get(i);
}
for (int i = 0; i < clreplacedpaths.size(); i++) {
urls[i + jars.size()] = clreplacedpaths.get(i);
}
return FlinkUserCodeClreplacedLoaders.parentFirst(urls, parent);
}
}
17
Source : KMeansSingleStepTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testCompileKMeansSingleStepWithStats() {
Plan p = getKMeansPlan();
p.setExecutionConfig(new ExecutionConfig());
// set the statistics
OperatorResolver cr = getContractResolver(p);
GenericDataSourceBase<?, ?> pointsSource = cr.getNode(DATAPOINTS);
GenericDataSourceBase<?, ?> centersSource = cr.getNode(CENTERS);
setSourceStatistics(pointsSource, 100L * 1024 * 1024 * 1024, 32f);
setSourceStatistics(centersSource, 1024 * 1024, 32f);
OptimizedPlan plan = compileWithStats(p);
checkPlan(plan);
}
17
Source : WorksetIterationsRecordApiCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testRecordApiWithDeferredSoltionSetUpdateWithMapper() {
Plan plan = getTestPlan(false, true);
OptimizedPlan oPlan;
try {
oPlan = compileNoStats(plan);
} catch (CompilerException ce) {
ce.printStackTrace();
fail("The pact compiler is unable to compile this plan correctly.");
// silence the compiler
return;
}
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
// iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
// the in-loop parreplacedioning is before the final reducer
// verify joinWithInvariant
replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
replacedertEquals(list0, joinWithInvariantNode.getKeysForInput1());
replacedertEquals(list0, joinWithInvariantNode.getKeysForInput2());
// verify joinWithSolutionSet
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
// verify reducer
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
replacedertEquals(list0, worksetReducer.getKeys(0));
// currently, the system may parreplacedion before or after the mapper
ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
replacedertTrue((ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARreplacedION_HASH) || (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARreplacedION_HASH));
new JobGraphGenerator().compileJobGraph(oPlan);
}
17
Source : WorksetIterationsJavaApiCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testJavaApiWithDeferredSoltionSetUpdateWithMapper() {
try {
Plan plan = getJavaTestPlan(false, true);
OptimizedPlan oPlan = compileNoStats(plan);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
// iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
// the in-loop parreplacedioning is before the final reducer
// verify joinWithInvariant
replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
// verify joinWithSolutionSet
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
replacedertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
// verify reducer
replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
replacedertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
// currently, the system may parreplacedion before or after the mapper
ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
replacedertTrue((ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARreplacedION_HASH) || (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARreplacedION_HASH));
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test errored: " + e.getMessage());
}
}
17
Source : JoinTranslationTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
private DualInputPlanNode createPlanAndGetJoinNode(JoinHint hint) {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Long> i1 = env.generateSequence(1, 1000);
DataSet<Long> i2 = env.generateSequence(1, 1000);
i1.join(i2, hint).where(new IdenreplacedyKeySelector<Long>()).equalTo(new IdenreplacedyKeySelector<Long>()).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
Plan plan = env.createProgramPlan();
// set statistics to the sources
plan.accept(new Visitor<Operator<?>>() {
@Override
public boolean preVisit(Operator<?> visitable) {
if (visitable instanceof GenericDataSourceBase) {
GenericDataSourceBase<?, ?> source = (GenericDataSourceBase<?, ?>) visitable;
setSourceStatistics(source, 10000000, 1000);
}
return true;
}
@Override
public void postVisit(Operator<?> visitable) {
}
});
OptimizedPlan op = compileWithStats(plan);
return (DualInputPlanNode) ((SinkPlanNode) op.getDataSinks().iterator().next()).getInput().getSource();
}
17
Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
*/
@Test
public void testLeftSideCountercheck() {
try {
Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
OptimizedPlan oPlan = compileNoStats(plan);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
// verify correct join strategy
replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, innerJoin.getDriverStrategy());
replacedertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
replacedertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test errored: " + e.getMessage());
}
}
17
Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* This test simulates a join of a big left side with a small right side inside of an iteration, where the small side is on a static path.
* Currently the best execution plan is a HYBRIDHASH_BUILD_SECOND_CACHED, where the small side is hashed and cached.
* This test also makes sure that all relevant plans are correctly enumerated by the optimizer.
*/
@Test
public void testCorrectChoosing() {
try {
Plan plan = getTestPlanRightStatic("");
SourceCollectorVisitor sourceCollector = new SourceCollectorVisitor();
plan.accept(sourceCollector);
for (GenericDataSourceBase<?, ?> s : sourceCollector.getSources()) {
if (s.getName().equals("bigFile")) {
this.setSourceStatistics(s, 10000000, 1000);
} else if (s.getName().equals("smallFile")) {
this.setSourceStatistics(s, 100, 100);
}
}
OptimizedPlan oPlan = compileNoStats(plan);
OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
// verify correct join strategy
replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
replacedertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
replacedertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Test errored: " + e.getMessage());
}
}
17
Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* <pre>
* SINK
* |
* COGROUP
* +---/ \----+
* / \
* / MATCH10
* / | \
* / | MATCH9
* MATCH5 | | \
* | \ | | MATCH8
* | MATCH4 | | | \
* | | \ | | | MATCH7
* | | MATCH3 | | | | \
* | | | \ | | | | MATCH6
* | | | MATCH2 | | | | | |
* | | | | \ +--+--+--+--+--+
* | | | | MATCH1 MAP
* \ | | | | | /-----------/
* (DATA SOURCE ONE)
* </pre>
*/
@Test
public void testBranchingSourceMultipleTimes() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
DataSet<Tuple2<Long, Long>> joined1 = source.join(source).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined2 = source.join(joined1).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined3 = source.join(joined2).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined4 = source.join(joined3).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined5 = source.join(joined4).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> mapped = source.map(new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {
@Override
public Tuple2<Long, Long> map(Tuple2<Long, Long> value) {
return null;
}
});
DataSet<Tuple2<Long, Long>> joined6 = mapped.join(mapped).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined7 = mapped.join(joined6).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined8 = mapped.join(joined7).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined9 = mapped.join(joined8).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
DataSet<Tuple2<Long, Long>> joined10 = mapped.join(joined9).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
joined5.coGroup(joined10).where(1).equalTo(1).with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>()).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileNoStats(plan);
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
17
Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testBranchesOnlyInBCVariables1() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(100);
DataSet<Long> input = env.generateSequence(1, 10);
DataSet<Long> bc_input = env.generateSequence(1, 10);
input.map(new IdenreplacedyMapper<Long>()).withBroadcastSet(bc_input, "name1").map(new IdenreplacedyMapper<Long>()).withBroadcastSet(bc_input, "name2").output(new DiscardingOutputFormat<Long>());
Plan plan = env.createProgramPlan();
compileNoStats(plan);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
17
Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testBranchingBroadcastVariable() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(100);
DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
DataSet<String> result1 = input1.map(new IdenreplacedyMapper<String>()).reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(input3, "bc");
DataSet<String> result2 = input2.map(new IdenreplacedyMapper<String>()).reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(input3, "bc");
result1.join(result2).where(new IdenreplacedyKeyExtractor<String>()).equalTo(new IdenreplacedyKeyExtractor<String>()).with(new RichJoinFunction<String, String, String>() {
@Override
public String join(String first, String second) {
return null;
}
}).withBroadcastSet(input3, "bc1").withBroadcastSet(input1, "bc2").withBroadcastSet(result1, "bc3").output(new DiscardingOutputFormat<String>());
Plan plan = env.createProgramPlan();
try {
compileNoStats(plan);
} catch (Exception e) {
e.printStackTrace();
replacedert.fail(e.getMessage());
}
}
17
Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* <pre>
* (SINK 3) (SINK 1) (SINK 2) (SINK 4)
* \ / \ /
* (SRC A) (SRC B)
* </pre>
*
* NOTE: this case is currently not caught by the compiler. we should enable the test once it is caught.
*/
@Test
public void testBranchingDisjointPlan() {
// construct the plan
final String out1Path = "file:///test/1";
final String out2Path = "file:///test/2";
final String out3Path = "file:///test/3";
final String out4Path = "file:///test/4";
// construct the plan
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
DataSet<Long> sourceA = env.generateSequence(0, 1);
DataSet<Long> sourceB = env.generateSequence(0, 1);
sourceA.writeAsText(out1Path);
sourceB.writeAsText(out2Path);
sourceA.writeAsText(out3Path);
sourceB.writeAsText(out4Path);
Plan plan = env.createProgramPlan();
compileNoStats(plan);
}
17
Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Test
public void testCostComputationWithMultipleDataSinks() {
final int SINKS = 5;
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
DataSet<Long> source = env.generateSequence(1, 10000);
DataSet<Long> mappedA = source.map(new IdenreplacedyMapper<Long>());
DataSet<Long> mappedC = source.map(new IdenreplacedyMapper<Long>());
for (int sink = 0; sink < SINKS; sink++) {
mappedA.output(new DiscardingOutputFormat<Long>());
mappedC.output(new DiscardingOutputFormat<Long>());
}
Plan plan = env.createProgramPlan("Plans With Multiple Data Sinks");
OptimizedPlan oPlan = compileNoStats(plan);
new JobGraphGenerator().compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
See More Examples