org.apache.flink.api.common.Plan

Here are the examples of the java api org.apache.flink.api.common.Plan taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

313 Examples 7

19 Source : TestEnvironment.java
with Apache License 2.0
from ljygz

private OptimizedPlan compileProgram(String jobName) {
    Plan p = createProgramPlan(jobName);
    Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
    return pc.compile(p);
}

19 Source : WorksetIterationsRecordApiCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testRecordApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
    Plan plan = getTestPlan(false, false);
    OptimizedPlan oPlan;
    try {
        oPlan = compileNoStats(plan);
    } catch (CompilerException ce) {
        ce.printStackTrace();
        fail("The pact compiler is unable to compile this plan correctly.");
        // silence the compiler
        return;
    }
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
    // iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
    // the in-loop parreplacedioning is before the final reducer
    // verify joinWithInvariant
    replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    replacedertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    replacedertEquals(list0, joinWithInvariantNode.getKeysForInput2());
    // verify joinWithSolutionSet
    replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
    // verify reducer
    replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
    replacedertEquals(list0, worksetReducer.getKeys(0));
    // verify solution delta
    replacedertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
    replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
    replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
    new JobGraphGenerator().compileJobGraph(oPlan);
}

19 Source : WorksetIterationsRecordApiCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testRecordApiWithDirectSoltionSetUpdate() {
    Plan plan = getTestPlan(true, false);
    OptimizedPlan oPlan;
    try {
        oPlan = compileNoStats(plan);
    } catch (CompilerException ce) {
        ce.printStackTrace();
        fail("The pact compiler is unable to compile this plan correctly.");
        // silence the compiler
        return;
    }
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
    // iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
    // the in-loop parreplacedioning is before the final reducer
    // verify joinWithInvariant
    replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    replacedertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    replacedertEquals(list0, joinWithInvariantNode.getKeysForInput2());
    // verify joinWithSolutionSet
    replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
    // verify reducer
    replacedertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
    replacedertEquals(list0, worksetReducer.getKeys(0));
    // verify solution delta
    replacedertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
    replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
    new JobGraphGenerator().compileJobGraph(oPlan);
}

19 Source : CompilerTestBase.java
with Apache License 2.0
from ljygz

public static OperatorResolver getContractResolver(Plan plan) {
    return new OperatorResolver(plan);
}

19 Source : CompilerTestBase.java
with Apache License 2.0
from ljygz

// ------------------------------------------------------------------------
public OptimizedPlan compileWithStats(Plan p) {
    return this.withStatsCompiler.compile(p);
}

19 Source : CompilerTestBase.java
with Apache License 2.0
from ljygz

public OptimizedPlan compileNoStats(Plan p) {
    return this.noStatsCompiler.compile(p);
}

19 Source : WorksetIterationsJavaApiCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testJavaApiWithDirectSoltionSetUpdate() {
    try {
        Plan plan = getJavaTestPlan(true, false);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
        DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
        SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
        // iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
        // the in-loop parreplacedioning is before the final reducer
        // verify joinWithInvariant
        replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
        replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
        replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
        // verify joinWithSolutionSet
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
        replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
        replacedertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
        // verify reducer
        replacedertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
        replacedertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
        // verify solution delta
        replacedertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
        replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

19 Source : WorksetIterationsJavaApiCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testJavaApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
    try {
        Plan plan = getJavaTestPlan(false, false);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
        DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
        SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
        // iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
        // the in-loop parreplacedioning is before the final reducer
        // verify joinWithInvariant
        replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
        replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
        replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
        // verify joinWithSolutionSet
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
        replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
        replacedertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
        // verify reducer
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
        replacedertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
        // verify solution delta
        replacedertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

19 Source : Optimizer.java
with Apache License 2.0
from ljygz

// ------------------------------------------------------------------------
// Compilation
// ------------------------------------------------------------------------
/**
 * Translates the given program to an OptimizedPlan, where all nodes have their local strategy replacedigned
 * and all channels have a shipping strategy replacedigned.
 *
 * For more details on the optimization phase, see the comments for
 * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpreplaced.OptimizerPostPreplaced)}.
 *
 * @param program The program to be translated.
 * @return The optimized plan.
 *
 * @throws CompilerException
 *         Thrown, if the plan is invalid or the optimizer encountered an inconsistent
 *         situation during the compilation process.
 */
public OptimizedPlan compile(Plan program) throws CompilerException {
    final OptimizerPostPreplaced postPreplaceder = getPostPreplacedFromPlan(program);
    return compile(program, postPreplaceder);
}

19 Source : Optimizer.java
with Apache License 2.0
from ljygz

// ------------------------------------------------------------------------
// Miscellaneous
// ------------------------------------------------------------------------
private OptimizerPostPreplaced getPostPreplacedFromPlan(Plan program) {
    final String clreplacedName = program.getPostPreplacedClreplacedName();
    if (clreplacedName == null) {
        throw new CompilerException("Optimizer Post Preplaced clreplaced description is null");
    }
    try {
        Clreplaced<? extends OptimizerPostPreplaced> clazz = Clreplaced.forName(clreplacedName).replacedubclreplaced(OptimizerPostPreplaced.clreplaced);
        try {
            return InstantiationUtil.instantiate(clazz, OptimizerPostPreplaced.clreplaced);
        } catch (RuntimeException rtex) {
            // unwrap the source exception
            if (rtex.getCause() != null) {
                throw new CompilerException("Cannot instantiate optimizer post preplaced: " + rtex.getMessage(), rtex.getCause());
            } else {
                throw rtex;
            }
        }
    } catch (ClreplacedNotFoundException cnfex) {
        throw new CompilerException("Cannot load Optimizer post-preplaced clreplaced '" + clreplacedName + "'.", cnfex);
    } catch (ClreplacedCastException ccex) {
        throw new CompilerException("Clreplaced '" + clreplacedName + "' is not an optimizer post-preplaced.", ccex);
    }
}

19 Source : OperatorTranslation.java
with Apache License 2.0
from ljygz

public Plan translateToPlan(List<DataSink<?>> sinks, String jobName) {
    List<GenericDataSinkBase<?>> planSinks = new ArrayList<>();
    for (DataSink<?> sink : sinks) {
        planSinks.add(translate(sink));
    }
    Plan p = new Plan(planSinks);
    p.setJobName(jobName);
    return p;
}

19 Source : ClusterClient.java
with Apache License 2.0
from ljygz

public static OptimizedPlan getOptimizedPlan(Optimizer compiler, Plan p, int parallelism) throws CompilerException {
    Logger log = LoggerFactory.getLogger(ClusterClient.clreplaced);
    if (parallelism > 0 && p.getDefaultParallelism() <= 0) {
        log.debug("Changing plan default parallelism from {} to {}", p.getDefaultParallelism(), parallelism);
        p.setDefaultParallelism(parallelism);
    }
    log.debug("Set parallelism {}, plan default parallelism {}", parallelism, p.getDefaultParallelism());
    return compiler.compile(p);
}

19 Source : LocalExecutor.java
with Apache License 2.0
from ljygz

/**
 * Executes the given dataflow plan.
 *
 * @param plan The dataflow plan.
 * @return The execution result.
 *
 * @throws Exception Thrown, if either the startup of the local execution context, or the execution
 *                   caused an exception.
 */
public static JobExecutionResult execute(Plan plan) throws Exception {
    return new LocalExecutor().executePlan(plan);
}

19 Source : LocalExecutor.java
with Apache License 2.0
from ljygz

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

19 Source : JobGraphGeneratorTest.java
with Apache License 2.0
from flink-tpc-ds

private static JobGraph compileJob(ExecutionEnvironment env) {
    Plan plan = env.createProgramPlan();
    Optimizer pc = new Optimizer(new Configuration());
    OptimizedPlan op = pc.compile(plan);
    JobGraphGenerator jgg = new JobGraphGenerator();
    return jgg.compileJobGraph(op);
}

18 Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz

// ------------------------------------------------------------------------
/**
 * Verifies that a robust reparreplacedioning plan with a hash join is created in the absence of statistics.
 */
@Test
public void testQueryNoStatistics() {
    try {
        Plan p = getTPCH3Plan();
        p.setExecutionConfig(defaultExecutionConfig);
        // compile
        final OptimizedPlan plan = compileNoStats(p);
        final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
        // get the nodes from the final plan
        final SinkPlanNode sink = or.getNode(SINK);
        final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
        final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ? (SingleInputPlanNode) reducer.getPredecessor() : null;
        final DualInputPlanNode join = or.getNode(JOIN_NAME);
        final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);
        // verify the optimizer choices
        checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
        replacedert.replacedertTrue(checkReparreplacedionShipStrategies(join, reducer, combiner));
        replacedert.replacedertTrue(checkHashJoinStrategies(join, reducer, true) || checkHashJoinStrategies(join, reducer, false));
    } catch (Exception e) {
        e.printStackTrace();
        replacedert.fail(e.getMessage());
    }
}

18 Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz

/**
 * Statistics that push towards a reparreplacedion merge join. If the join blows the data volume up significantly,
 * re-exploiting the sorted order is cheaper.
 */
@Test
public void testQueryWithStatsForReparreplacedionMerge() {
    Plan p = getTPCH3Plan();
    p.setExecutionConfig(defaultExecutionConfig);
    // set compiler hints
    OperatorResolver cr = getContractResolver(p);
    DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
    match.getCompilerHints().setFilterFactor(100f);
    testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}

18 Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz

private void testQueryGeneric(Plan p, long orderSize, long lineitemSize, float orderSelectivity, float joinSelectivity, boolean broadcastOkay, boolean parreplacedionedOkay, boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
    try {
        // set statistics
        OperatorResolver cr = getContractResolver(p);
        GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
        GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
        SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
        DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
        setSourceStatistics(ordersSource, orderSize, 100f);
        setSourceStatistics(lineItemSource, lineitemSize, 140f);
        mapper.getCompilerHints().setAvgOutputRecordSize(16f);
        mapper.getCompilerHints().setFilterFactor(orderSelectivity);
        joiner.getCompilerHints().setFilterFactor(joinSelectivity);
        // compile
        final OptimizedPlan plan = compileWithStats(p);
        final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
        // get the nodes from the final plan
        final SinkPlanNode sink = or.getNode(SINK);
        final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
        final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ? (SingleInputPlanNode) reducer.getPredecessor() : null;
        final DualInputPlanNode join = or.getNode(JOIN_NAME);
        final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);
        checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
        // check the possible variants and that the variant ia allowed in this specific setting
        if (checkBroadcastShipStrategies(join, reducer, combiner)) {
            replacedert.replacedertTrue("Broadcast join incorrectly chosen.", broadcastOkay);
            if (checkHashJoinStrategies(join, reducer, true)) {
                replacedert.replacedertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
            } else if (checkHashJoinStrategies(join, reducer, false)) {
                replacedert.replacedertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
            } else if (checkBroadcastMergeJoin(join, reducer)) {
                replacedert.replacedertTrue("Merge join incorrectly chosen", mergeJoinOkay);
            } else {
                replacedert.fail("Plan has no correct hash join or merge join strategies.");
            }
        } else if (checkReparreplacedionShipStrategies(join, reducer, combiner)) {
            replacedert.replacedertTrue("Parreplacedioned join incorrectly chosen.", parreplacedionedOkay);
            if (checkHashJoinStrategies(join, reducer, true)) {
                replacedert.replacedertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
            } else if (checkHashJoinStrategies(join, reducer, false)) {
                replacedert.replacedertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
            } else if (checkReparreplacedionMergeJoin(join, reducer)) {
                replacedert.replacedertTrue("Merge join incorrectly chosen", mergeJoinOkay);
            } else {
                replacedert.fail("Plan has no correct hash join or merge join strategies.");
            }
        } else {
            replacedert.fail("Plan has neither correct BC join or parreplacedioned join configuration.");
        }
    } catch (Exception e) {
        e.printStackTrace();
        replacedert.fail(e.getMessage());
    }
}

18 Source : RelationalQueryCompilerTest.java
with Apache License 2.0
from ljygz

// ------------------------------------------------------------------------
private void testQueryGeneric(long orderSize, long lineItemSize, float ordersFilterFactor, float joinFilterFactor, boolean broadcastOkay, boolean parreplacedionedOkay, boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
    Plan p = getTPCH3Plan();
    p.setExecutionConfig(defaultExecutionConfig);
    testQueryGeneric(p, orderSize, lineItemSize, ordersFilterFactor, joinFilterFactor, broadcastOkay, parreplacedionedOkay, hashJoinFirstOkay, hashJoinSecondOkay, mergeJoinOkay);
}

18 Source : KMeansSingleStepTest.java
with Apache License 2.0
from ljygz

@Test
public void testCompileKMeansSingleStepWithOutStats() {
    Plan p = getKMeansPlan();
    p.setExecutionConfig(new ExecutionConfig());
    OptimizedPlan plan = compileNoStats(p);
    checkPlan(plan);
}

18 Source : CancelingTestBase.java
with Apache License 2.0
from ljygz

private JobGraph getJobGraph(final Plan plan) {
    final Optimizer pc = new Optimizer(new DataStatistics(), getConfiguration());
    final OptimizedPlan op = pc.compile(plan);
    final JobGraphGenerator jgg = new JobGraphGenerator();
    return jgg.compileJobGraph(op);
}

18 Source : AccumulatorLiveITCase.java
with Apache License 2.0
from ljygz

/**
 * Helpers to generate the JobGraph.
 */
private static JobGraph getJobGraph(Plan plan) {
    Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
    JobGraphGenerator jgg = new JobGraphGenerator();
    OptimizedPlan op = pc.compile(plan);
    return jgg.compileJobGraph(op);
}

18 Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz

/**
 * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
 */
@Test
public void testRightSideCountercheck() {
    try {
        Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy());
        replacedertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
        replacedertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

18 Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz

/**
 * This tests whether a HYBRIDHASH_BUILD_FIRST is correctly transformed to a HYBRIDHASH_BUILD_FIRST_CACHED
 * when inside of an iteration an on the static path
 */
@Test
public void testLeftSide() {
    try {
        Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED, innerJoin.getDriverStrategy());
        replacedertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
        replacedertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

18 Source : OptimizedPlan.java
with Apache License 2.0
from ljygz

/**
 * The execution plan generated by the Optimizer. It contains {@link PlanNode}s
 * and {@link Channel}s that describe exactly how the program should be executed.
 *
 * <p>The optimized plan defines all ship strategies (local pipe, shuffle, broadcast, rebalance),
 * all operator strategies (sorting-merge join, hash join, sorted grouping, ...),
 * and the data exchange modes (batched, pipelined).</p>
 */
public clreplaced OptimizedPlan implements FlinkPlan, Visitable<PlanNode> {

    /**
     * The data sources in the plan.
     */
    private final Collection<SourcePlanNode> dataSources;

    /**
     * The data sinks in the plan.
     */
    private final Collection<SinkPlanNode> dataSinks;

    /**
     * All nodes in the optimizer plan.
     */
    private final Collection<PlanNode> allNodes;

    /**
     * The original program (as a dataflow plan).
     */
    private final Plan originalProgram;

    /**
     * Name of the job
     */
    private final String jobName;

    /**
     * Creates a new instance of this optimizer plan container. The plan is given and fully
     * described by the data sources, sinks and the collection of all nodes.
     *
     * @param sources The data sources.
     * @param sinks The data sinks.
     * @param allNodes A collection containing all nodes in the plan.
     * @param jobName The name of the program
     */
    public OptimizedPlan(Collection<SourcePlanNode> sources, Collection<SinkPlanNode> sinks, Collection<PlanNode> allNodes, String jobName, Plan programPlan) {
        this.dataSources = sources;
        this.dataSinks = sinks;
        this.allNodes = allNodes;
        this.jobName = jobName;
        this.originalProgram = programPlan;
    }

    /**
     * Gets the data sources from this OptimizedPlan.
     *
     * @return The data sources.
     */
    public Collection<SourcePlanNode> getDataSources() {
        return dataSources;
    }

    /**
     * Gets the data sinks from this OptimizedPlan.
     *
     * @return The data sinks.
     */
    public Collection<SinkPlanNode> getDataSinks() {
        return dataSinks;
    }

    /**
     * Gets all the nodes from this OptimizedPlan.
     *
     * @return All nodes.
     */
    public Collection<PlanNode> getAllNodes() {
        return allNodes;
    }

    /**
     * Returns the name of the program.
     *
     * @return The name of the program.
     */
    public String getJobName() {
        return this.jobName;
    }

    /**
     * Gets the original program's dataflow plan from which this optimized plan was created.
     *
     * @return The original program's dataflow plan.
     */
    public Plan getOriginalPlan() {
        return this.originalProgram;
    }

    // ------------------------------------------------------------------------
    /**
     * Applies the given visitor top down to all nodes, starting at the sinks.
     *
     * @param visitor
     *        The visitor to apply to the nodes in this plan.
     * @see org.apache.flink.util.Visitable#accept(org.apache.flink.util.Visitor)
     */
    @Override
    public void accept(Visitor<PlanNode> visitor) {
        for (SinkPlanNode node : this.dataSinks) {
            node.accept(visitor);
        }
    }
}

18 Source : Optimizer.java
with Apache License 2.0
from ljygz

/**
 * This function performs only the first step to the compilation process - the creation of the optimizer
 * representation of the plan. No estimations or enumerations of alternatives are done here.
 *
 * @param program The plan to generate the optimizer representation for.
 * @return The optimizer representation of the plan, as a collection of all data sinks
 *         from the plan can be traversed.
 */
public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
    GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
    program.accept(graphCreator);
    return graphCreator.getSinks();
}

18 Source : NamesTest.java
with Apache License 2.0
from ljygz

private static void testForName(final String expected, Plan plan) {
    plan.accept(new Visitor<Operator<?>>() {

        @Override
        public boolean preVisit(Operator<?> visitable) {
            if (visitable instanceof PlanFilterOperator<?>) {
                // cast is actually not required. Its just a check for the right element
                PlanFilterOperator<?> filterOp = (PlanFilterOperator<?>) visitable;
                replacedert.replacedertEquals(expected, filterOp.getName());
            }
            return true;
        }

        @Override
        public void postVisit(Operator<?> visitable) {
        // 
        }
    });
}

18 Source : CollectionExecutor.java
with Apache License 2.0
from ljygz

// --------------------------------------------------------------------------------------------
// General execution methods
// --------------------------------------------------------------------------------------------
public JobExecutionResult execute(Plan program) throws Exception {
    long startTime = System.currentTimeMillis();
    initCache(program.getCachedFiles());
    Collection<? extends GenericDataSinkBase<?>> sinks = program.getDataSinks();
    for (Operator<?> sink : sinks) {
        execute(sink);
    }
    long endTime = System.currentTimeMillis();
    Map<String, OptionalFailure<Object>> acreplacedulatorResults = AcreplacedulatorHelper.toResultMap(acreplacedulators);
    return new JobExecutionResult(null, endTime - startTime, acreplacedulatorResults);
}

18 Source : RemoteExecutor.java
with Apache License 2.0
from ljygz

@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
    OptimizedPlan optPlan = opt.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan);
}

18 Source : RemoteExecutor.java
with Apache License 2.0
from ljygz

// ------------------------------------------------------------------------
// Executing programs
// ------------------------------------------------------------------------
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
    if (plan == null) {
        throw new IllegalArgumentException("The plan may not be null.");
    }
    JobWithJars p = new JobWithJars(plan, this.jarFiles, this.globalClreplacedpaths);
    return executePlanWithJars(p);
}

18 Source : PreviewPlanEnvironment.java
with Apache License 2.0
from ljygz

/**
 * Environment to extract the pre-optimized plan.
 */
public final clreplaced PreviewPlanEnvironment extends ExecutionEnvironment {

    List<DataSinkNode> previewPlan;

    String preview;

    Plan plan;

    @Override
    public JobExecutionResult execute(String jobName) throws Exception {
        this.plan = createProgramPlan(jobName);
        this.previewPlan = Optimizer.createPreOptimizedPlan(plan);
        // do not go on with anything now!
        throw new OptimizerPlanEnvironment.ProgramAbortException();
    }

    @Override
    public String getExecutionPlan() throws Exception {
        Plan plan = createProgramPlan("unused");
        this.previewPlan = Optimizer.createPreOptimizedPlan(plan);
        // do not go on with anything now!
        throw new OptimizerPlanEnvironment.ProgramAbortException();
    }

    @Override
    public void startNewSession() {
    }

    public void setAsContext() {
        ExecutionEnvironmentFactory factory = new ExecutionEnvironmentFactory() {

            @Override
            public ExecutionEnvironment createExecutionEnvironment() {
                return PreviewPlanEnvironment.this;
            }
        };
        initializeContextEnvironment(factory);
    }

    public void unsetAsContext() {
        resetContextEnvironment();
    }

    public void setPreview(String preview) {
        this.preview = preview;
    }

    public Plan getPlan() {
        return plan;
    }
}

18 Source : PreviewPlanEnvironment.java
with Apache License 2.0
from ljygz

@Override
public String getExecutionPlan() throws Exception {
    Plan plan = createProgramPlan("unused");
    this.previewPlan = Optimizer.createPreOptimizedPlan(plan);
    // do not go on with anything now!
    throw new OptimizerPlanEnvironment.ProgramAbortException();
}

18 Source : OptimizerPlanEnvironment.java
with Apache License 2.0
from ljygz

@Override
public String getExecutionPlan() throws Exception {
    Plan plan = createProgramPlan(null, false);
    this.optimizerPlan = compiler.compile(plan);
    // do not go on with anything now!
    throw new ProgramAbortException();
}

18 Source : OptimizerPlanEnvironment.java
with Apache License 2.0
from ljygz

// ------------------------------------------------------------------------
// Execution Environment methods
// ------------------------------------------------------------------------
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    Plan plan = createProgramPlan(jobName);
    this.optimizerPlan = compiler.compile(plan);
    // do not go on with anything now!
    throw new ProgramAbortException();
}

18 Source : JobWithJars.java
with Apache License 2.0
from ljygz

/**
 * A JobWithJars is a Flink dataflow plan, together with a bunch of JAR files that contain
 * the clreplacedes of the functions and libraries necessary for the execution.
 */
public clreplaced JobWithJars {

    private Plan plan;

    private List<URL> jarFiles;

    /**
     * clreplacedpaths that are needed during user code execution.
     */
    private List<URL> clreplacedpaths;

    private ClreplacedLoader userCodeClreplacedLoader;

    public JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths) throws IOException {
        this.plan = plan;
        this.jarFiles = new ArrayList<URL>(jarFiles.size());
        this.clreplacedpaths = new ArrayList<URL>(clreplacedpaths.size());
        for (URL jarFile : jarFiles) {
            checkJarFile(jarFile);
            this.jarFiles.add(jarFile);
        }
        for (URL path : clreplacedpaths) {
            this.clreplacedpaths.add(path);
        }
    }

    public JobWithJars(Plan plan, URL jarFile) throws IOException {
        this.plan = plan;
        checkJarFile(jarFile);
        this.jarFiles = Collections.singletonList(jarFile);
        this.clreplacedpaths = Collections.<URL>emptyList();
    }

    JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths, ClreplacedLoader userCodeClreplacedLoader) {
        this.plan = plan;
        this.jarFiles = jarFiles;
        this.clreplacedpaths = clreplacedpaths;
        this.userCodeClreplacedLoader = userCodeClreplacedLoader;
    }

    /**
     * Returns the plan.
     */
    public Plan getPlan() {
        return this.plan;
    }

    /**
     * Returns list of jar files that need to be submitted with the plan.
     */
    public List<URL> getJarFiles() {
        return this.jarFiles;
    }

    /**
     * Returns list of clreplacedpaths that need to be submitted with the plan.
     */
    public List<URL> getClreplacedpaths() {
        return clreplacedpaths;
    }

    /**
     * Gets the {@link java.lang.ClreplacedLoader} that must be used to load user code clreplacedes.
     *
     * @return The user code ClreplacedLoader.
     */
    public ClreplacedLoader getUserCodeClreplacedLoader() {
        if (this.userCodeClreplacedLoader == null) {
            this.userCodeClreplacedLoader = buildUserCodeClreplacedLoader(jarFiles, clreplacedpaths, getClreplaced().getClreplacedLoader());
        }
        return this.userCodeClreplacedLoader;
    }

    public static void checkJarFile(URL jar) throws IOException {
        File jarFile;
        try {
            jarFile = new File(jar.toURI());
        } catch (URISyntaxException e) {
            throw new IOException("JAR file path is invalid '" + jar + "'");
        }
        if (!jarFile.exists()) {
            throw new IOException("JAR file does not exist '" + jarFile.getAbsolutePath() + "'");
        }
        if (!jarFile.canRead()) {
            throw new IOException("JAR file can't be read '" + jarFile.getAbsolutePath() + "'");
        }
    // TODO: Check if proper JAR file
    }

    public static ClreplacedLoader buildUserCodeClreplacedLoader(List<URL> jars, List<URL> clreplacedpaths, ClreplacedLoader parent) {
        URL[] urls = new URL[jars.size() + clreplacedpaths.size()];
        for (int i = 0; i < jars.size(); i++) {
            urls[i] = jars.get(i);
        }
        for (int i = 0; i < clreplacedpaths.size(); i++) {
            urls[i + jars.size()] = clreplacedpaths.get(i);
        }
        return FlinkUserCodeClreplacedLoaders.parentFirst(urls, parent);
    }
}

18 Source : ContextEnvironment.java
with Apache License 2.0
from ljygz

@Override
public JobExecutionResult execute(String jobName) throws Exception {
    Plan p = createProgramPlan(jobName);
    JobWithJars toRun = new JobWithJars(p, this.jarFilesToAttach, this.clreplacedpathsToAttach, this.userCodeClreplacedLoader);
    this.lastJobExecutionResult = client.run(toRun, getParallelism(), savepointSettings).getJobExecutionResult();
    return this.lastJobExecutionResult;
}

18 Source : LocalExecutor.java
with Apache License 2.0
from ljygz

/**
 * Creates a JSON representation of the given dataflow plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow plan (prior to optimization) as a JSON string.
 */
public static String getPlanAsJSON(Plan plan) {
    List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
    return new PlanJSONDumpGenerator().getPactPlanAsJSON(sinks);
}

18 Source : LocalExecutor.java
with Apache License 2.0
from ljygz

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();
    Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration);
    pc.setDefaultParallelism(parallelism);
    OptimizedPlan op = pc.compile(plan);
    return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}

18 Source : JobWithJars.java
with Apache License 2.0
from flink-tpc-ds

/**
 * A JobWithJars is a Flink dataflow plan, together with a bunch of JAR files that contain
 * the clreplacedes of the functions and libraries necessary for the execution.
 */
public clreplaced JobWithJars {

    private Plan plan;

    private List<URL> jarFiles;

    /**
     * clreplacedpaths that are needed during user code execution.
     */
    private List<URL> clreplacedpaths;

    private ClreplacedLoader userCodeClreplacedLoader;

    public JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths) throws IOException {
        this.plan = plan;
        this.jarFiles = new ArrayList<URL>(jarFiles.size());
        this.clreplacedpaths = new ArrayList<URL>(clreplacedpaths.size());
        for (URL jarFile : jarFiles) {
            checkJarFile(jarFile);
            this.jarFiles.add(jarFile);
        }
        for (URL path : clreplacedpaths) {
            this.clreplacedpaths.add(path);
        }
    }

    public JobWithJars(Plan plan, URL jarFile) throws IOException {
        this.plan = plan;
        checkJarFile(jarFile);
        this.jarFiles = Collections.singletonList(jarFile);
        this.clreplacedpaths = Collections.<URL>emptyList();
    }

    JobWithJars(Plan plan, List<URL> jarFiles, List<URL> clreplacedpaths, ClreplacedLoader userCodeClreplacedLoader) {
        this.plan = plan;
        this.jarFiles = jarFiles;
        this.clreplacedpaths = clreplacedpaths;
        this.userCodeClreplacedLoader = userCodeClreplacedLoader;
    }

    /**
     * Returns the plan.
     */
    public Plan getPlan() {
        return this.plan;
    }

    /**
     * Returns list of jar files that need to be submitted with the plan.
     */
    public List<URL> getJarFiles() {
        return this.jarFiles;
    }

    /**
     * Returns list of clreplacedpaths that need to be submitted with the plan.
     */
    public List<URL> getClreplacedpaths() {
        return clreplacedpaths;
    }

    /**
     * Gets the {@link java.lang.ClreplacedLoader} that must be used to load user code clreplacedes.
     *
     * @return The user code ClreplacedLoader.
     */
    public ClreplacedLoader getUserCodeClreplacedLoader() {
        if (this.userCodeClreplacedLoader == null) {
            this.userCodeClreplacedLoader = buildUserCodeClreplacedLoader(jarFiles, clreplacedpaths, getClreplaced().getClreplacedLoader());
        }
        return this.userCodeClreplacedLoader;
    }

    public static void checkJarFile(URL jar) throws IOException {
        File jarFile;
        try {
            jarFile = new File(jar.toURI());
        } catch (URISyntaxException e) {
            throw new IOException("JAR file path is invalid '" + jar + '\'');
        }
        if (!jarFile.exists()) {
            throw new IOException("JAR file does not exist '" + jarFile.getAbsolutePath() + '\'');
        }
        if (!jarFile.canRead()) {
            throw new IOException("JAR file can't be read '" + jarFile.getAbsolutePath() + '\'');
        }
        try (JarFile ignored = new JarFile(jarFile)) {
        // verify that we can open the Jar file
        } catch (IOException e) {
            throw new IOException("Error while opening jar file '" + jarFile.getAbsolutePath() + '\'', e);
        }
    }

    public static ClreplacedLoader buildUserCodeClreplacedLoader(List<URL> jars, List<URL> clreplacedpaths, ClreplacedLoader parent) {
        URL[] urls = new URL[jars.size() + clreplacedpaths.size()];
        for (int i = 0; i < jars.size(); i++) {
            urls[i] = jars.get(i);
        }
        for (int i = 0; i < clreplacedpaths.size(); i++) {
            urls[i + jars.size()] = clreplacedpaths.get(i);
        }
        return FlinkUserCodeClreplacedLoaders.parentFirst(urls, parent);
    }
}

17 Source : KMeansSingleStepTest.java
with Apache License 2.0
from ljygz

@Test
public void testCompileKMeansSingleStepWithStats() {
    Plan p = getKMeansPlan();
    p.setExecutionConfig(new ExecutionConfig());
    // set the statistics
    OperatorResolver cr = getContractResolver(p);
    GenericDataSourceBase<?, ?> pointsSource = cr.getNode(DATAPOINTS);
    GenericDataSourceBase<?, ?> centersSource = cr.getNode(CENTERS);
    setSourceStatistics(pointsSource, 100L * 1024 * 1024 * 1024, 32f);
    setSourceStatistics(centersSource, 1024 * 1024, 32f);
    OptimizedPlan plan = compileWithStats(p);
    checkPlan(plan);
}

17 Source : WorksetIterationsRecordApiCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testRecordApiWithDeferredSoltionSetUpdateWithMapper() {
    Plan plan = getTestPlan(false, true);
    OptimizedPlan oPlan;
    try {
        oPlan = compileNoStats(plan);
    } catch (CompilerException ce) {
        ce.printStackTrace();
        fail("The pact compiler is unable to compile this plan correctly.");
        // silence the compiler
        return;
    }
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
    SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
    // iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
    // the in-loop parreplacedioning is before the final reducer
    // verify joinWithInvariant
    replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    replacedertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    replacedertEquals(list0, joinWithInvariantNode.getKeysForInput2());
    // verify joinWithSolutionSet
    replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
    // verify reducer
    replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
    replacedertEquals(list0, worksetReducer.getKeys(0));
    // currently, the system may parreplacedion before or after the mapper
    ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
    ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
    replacedertTrue((ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARreplacedION_HASH) || (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARreplacedION_HASH));
    new JobGraphGenerator().compileJobGraph(oPlan);
}

17 Source : WorksetIterationsJavaApiCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testJavaApiWithDeferredSoltionSetUpdateWithMapper() {
    try {
        Plan plan = getJavaTestPlan(false, true);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
        DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
        SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
        SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
        // iteration preserves parreplacedioning in reducer, so the first parreplacedioning is out of the loop,
        // the in-loop parreplacedioning is before the final reducer
        // verify joinWithInvariant
        replacedertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
        replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
        replacedertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
        // verify joinWithSolutionSet
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
        replacedertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
        replacedertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
        // verify reducer
        replacedertEquals(ShipStrategyType.PARreplacedION_HASH, worksetReducer.getInput().getShipStrategy());
        replacedertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
        // currently, the system may parreplacedion before or after the mapper
        ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
        ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
        replacedertTrue((ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARreplacedION_HASH) || (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARreplacedION_HASH));
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

17 Source : JoinTranslationTest.java
with Apache License 2.0
from ljygz

private DualInputPlanNode createPlanAndGetJoinNode(JoinHint hint) {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Long> i1 = env.generateSequence(1, 1000);
    DataSet<Long> i2 = env.generateSequence(1, 1000);
    i1.join(i2, hint).where(new IdenreplacedyKeySelector<Long>()).equalTo(new IdenreplacedyKeySelector<Long>()).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    Plan plan = env.createProgramPlan();
    // set statistics to the sources
    plan.accept(new Visitor<Operator<?>>() {

        @Override
        public boolean preVisit(Operator<?> visitable) {
            if (visitable instanceof GenericDataSourceBase) {
                GenericDataSourceBase<?, ?> source = (GenericDataSourceBase<?, ?>) visitable;
                setSourceStatistics(source, 10000000, 1000);
            }
            return true;
        }

        @Override
        public void postVisit(Operator<?> visitable) {
        }
    });
    OptimizedPlan op = compileWithStats(plan);
    return (DualInputPlanNode) ((SinkPlanNode) op.getDataSinks().iterator().next()).getInput().getSource();
}

17 Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz

/**
 * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
 */
@Test
public void testLeftSideCountercheck() {
    try {
        Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, innerJoin.getDriverStrategy());
        replacedertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
        replacedertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

17 Source : CachedMatchStrategyCompilerTest.java
with Apache License 2.0
from ljygz

/**
 * This test simulates a join of a big left side with a small right side inside of an iteration, where the small side is on a static path.
 * Currently the best execution plan is a HYBRIDHASH_BUILD_SECOND_CACHED, where the small side is hashed and cached.
 * This test also makes sure that all relevant plans are correctly enumerated by the optimizer.
 */
@Test
public void testCorrectChoosing() {
    try {
        Plan plan = getTestPlanRightStatic("");
        SourceCollectorVisitor sourceCollector = new SourceCollectorVisitor();
        plan.accept(sourceCollector);
        for (GenericDataSourceBase<?, ?> s : sourceCollector.getSources()) {
            if (s.getName().equals("bigFile")) {
                this.setSourceStatistics(s, 10000000, 1000);
            } else if (s.getName().equals("smallFile")) {
                this.setSourceStatistics(s, 100, 100);
            }
        }
        OptimizedPlan oPlan = compileNoStats(plan);
        OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
        DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
        // verify correct join strategy
        replacedertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
        replacedertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
        replacedertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test errored: " + e.getMessage());
    }
}

17 Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz

/**
 * <pre>
 *                              SINK
 *                               |
 *                            COGROUP
 *                        +---/    \----+
 *                       /               \
 *                      /             MATCH10
 *                     /               |    \
 *                    /                |  MATCH9
 *                MATCH5               |  |   \
 *                |   \                |  | MATCH8
 *                | MATCH4             |  |  |   \
 *                |  |   \             |  |  | MATCH7
 *                |  | MATCH3          |  |  |  |   \
 *                |  |  |   \          |  |  |  | MATCH6
 *                |  |  | MATCH2       |  |  |  |  |  |
 *                |  |  |  |   \       +--+--+--+--+--+
 *                |  |  |  | MATCH1            MAP
 *                \  |  |  |  |  | /-----------/
 *                (DATA SOURCE ONE)
 * </pre>
 */
@Test
public void testBranchingSourceMultipleTimes() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(DEFAULT_PARALLELISM);
        DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 10000000).map(new Duplicator<Long>());
        DataSet<Tuple2<Long, Long>> joined1 = source.join(source).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined2 = source.join(joined1).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined3 = source.join(joined2).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined4 = source.join(joined3).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined5 = source.join(joined4).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> mapped = source.map(new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {

            @Override
            public Tuple2<Long, Long> map(Tuple2<Long, Long> value) {
                return null;
            }
        });
        DataSet<Tuple2<Long, Long>> joined6 = mapped.join(mapped).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined7 = mapped.join(joined6).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined8 = mapped.join(joined7).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined9 = mapped.join(joined8).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        DataSet<Tuple2<Long, Long>> joined10 = mapped.join(joined9).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
        joined5.coGroup(joined10).where(1).equalTo(1).with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>()).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
        Plan plan = env.createProgramPlan();
        OptimizedPlan oPlan = compileNoStats(plan);
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

17 Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testBranchesOnlyInBCVariables1() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(100);
        DataSet<Long> input = env.generateSequence(1, 10);
        DataSet<Long> bc_input = env.generateSequence(1, 10);
        input.map(new IdenreplacedyMapper<Long>()).withBroadcastSet(bc_input, "name1").map(new IdenreplacedyMapper<Long>()).withBroadcastSet(bc_input, "name2").output(new DiscardingOutputFormat<Long>());
        Plan plan = env.createProgramPlan();
        compileNoStats(plan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

17 Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testBranchingBroadcastVariable() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(100);
    DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
    DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
    DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
    DataSet<String> result1 = input1.map(new IdenreplacedyMapper<String>()).reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(input3, "bc");
    DataSet<String> result2 = input2.map(new IdenreplacedyMapper<String>()).reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(input3, "bc");
    result1.join(result2).where(new IdenreplacedyKeyExtractor<String>()).equalTo(new IdenreplacedyKeyExtractor<String>()).with(new RichJoinFunction<String, String, String>() {

        @Override
        public String join(String first, String second) {
            return null;
        }
    }).withBroadcastSet(input3, "bc1").withBroadcastSet(input1, "bc2").withBroadcastSet(result1, "bc3").output(new DiscardingOutputFormat<String>());
    Plan plan = env.createProgramPlan();
    try {
        compileNoStats(plan);
    } catch (Exception e) {
        e.printStackTrace();
        replacedert.fail(e.getMessage());
    }
}

17 Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz

/**
 * <pre>
 *     (SINK 3) (SINK 1)   (SINK 2) (SINK 4)
 *         \     /             \     /
 *         (SRC A)             (SRC B)
 * </pre>
 *
 * NOTE: this case is currently not caught by the compiler. we should enable the test once it is caught.
 */
@Test
public void testBranchingDisjointPlan() {
    // construct the plan
    final String out1Path = "file:///test/1";
    final String out2Path = "file:///test/2";
    final String out3Path = "file:///test/3";
    final String out4Path = "file:///test/4";
    // construct the plan
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSet<Long> sourceA = env.generateSequence(0, 1);
    DataSet<Long> sourceB = env.generateSequence(0, 1);
    sourceA.writeAsText(out1Path);
    sourceB.writeAsText(out2Path);
    sourceA.writeAsText(out3Path);
    sourceB.writeAsText(out4Path);
    Plan plan = env.createProgramPlan();
    compileNoStats(plan);
}

17 Source : BranchingPlansCompilerTest.java
with Apache License 2.0
from ljygz

@Test
public void testCostComputationWithMultipleDataSinks() {
    final int SINKS = 5;
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(DEFAULT_PARALLELISM);
        DataSet<Long> source = env.generateSequence(1, 10000);
        DataSet<Long> mappedA = source.map(new IdenreplacedyMapper<Long>());
        DataSet<Long> mappedC = source.map(new IdenreplacedyMapper<Long>());
        for (int sink = 0; sink < SINKS; sink++) {
            mappedA.output(new DiscardingOutputFormat<Long>());
            mappedC.output(new DiscardingOutputFormat<Long>());
        }
        Plan plan = env.createProgramPlan("Plans With Multiple Data Sinks");
        OptimizedPlan oPlan = compileNoStats(plan);
        new JobGraphGenerator().compileJobGraph(oPlan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

See More Examples