com.google.common.util.concurrent.ListeningExecutorService

Here are the examples of the java api class com.google.common.util.concurrent.ListeningExecutorService taken from open source projects.

@Test
public void testConcurrentFetchTasks() throws Exception {
    // Test for regression of AURORA-1625
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(5));
    assertStoreContents();
    saveTasks(TASK_A, TASK_B, TASK_C, TASK_D);
    List<ListenableFuture<Integer>> futures = Lists.newArrayList();
    for (int i = 0; i < 100; i++) {
        futures.add(executor.submit(() -> Iterables.size(fetchTasks(Query.unscoped()))));
    }
    Future<List<Integer>> f = Futures.allAsList(futures);
    executor.shutdown();
    executor.awaitTermination(1, TimeUnit.MINUTES);
    assertEquals(Iterables.getOnlyElement(ImmutableSet.copyOf(f.get())), (Integer) 4);
}

private static void publishLogs() throws Exception {
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    StatusPrinter.setPrintStream(new PrintStream(bos));
    StatusPrinter.print((LoggerContext) LoggerFactory.getILoggerFactory());
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
    List<ListenableFuture<?>> futures = Lists.newArrayList();
    futures.add(executor.submit(new LogPublisher(0, new FlowletLoggingContext("NS_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN1", "INSTANCE1"))));
    futures.add(executor.submit(new LogPublisher(0, new FlowletLoggingContext("NS_2", "APP_2", "FLOW_2", "FLOWLET_2", "RUN1", "INSTANCE1"))));
    futures.add(executor.submit(new LogPublisher(0, new ServiceLoggingContext("system", "services", "metrics"))));
    // Make sure the final segments of logs are added at end to simplify checking for done in waitTillLogSaverDone
    futures.add(executor.submit(new LogPublisher(60, new FlowletLoggingContext("NS_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN2", "INSTANCE2"))));
    futures.add(executor.submit(new LogPublisher(60, new FlowletLoggingContext("NS_2", "APP_2", "FLOW_2", "FLOWLET_2", "RUN2", "INSTANCE2"))));
    Futures.allAsList(futures).get();
    System.out.println(bos.toString());
    executor.shutdownNow();
}

private void publishLogs() throws Exception {
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    StatusPrinter.setPrintStream(new PrintStream(bos));
    StatusPrinter.print((LoggerContext) LoggerFactory.getILoggerFactory());
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    List<ListenableFuture<?>> futures = Lists.newArrayList();
    futures.add(executor.submit(new LogPublisher(0, new FlowletLoggingContext("NS_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN1", "INSTANCE1"))));
    Futures.allAsList(futures).get();
    executor.shutdownNow();
}

@Test(timeout = 5000)
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        @SuppressWarnings("deprecation") ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        ContainerContext containerContext = new ContainerContext(containerId.toString());
        ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
        getTaskFuture.get();
        assertEquals(1, umbilical.getTaskInvocations);
    } finally {
        executor.shutdownNow();
    }
}

private List<SingularityS3Log> getS3Logs(S3Configuration s3Configuration, Optional<String> group, Collection<String> prefixes) throws InterruptedException, ExecutionException, TimeoutException {
    if (prefixes.isEmpty()) {
        return Collections.emptyList();
    }
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Math.min(prefixes.size(), s3Configuration.getMaxS3Threads()), new ThreadFactoryBuilder().setNameFormat("S3LogFetcher-%d").build()));
    try {
        List<SingularityS3Log> logs = Lists.newArrayList(getS3LogsWithExecutorService(s3Configuration, group, executorService, prefixes));
        Collections.sort(logs, LOG_COMPARATOR);
        return logs;
    } finally {
        executorService.shutdownNow();
    }
}

private void onFinish(SingularityExecutorTask task, Protos.TaskState taskState) {
    processKiller.cancelDestroyFuture(task.getTaskId());
    tasks.remove(task.getTaskId());
    processRunningTasks.remove(task.getTaskId());
    processBuildingTasks.remove(task.getTaskId());
    task.cleanup(taskState);
    ListeningExecutorService executorService = taskToShellCommandPool.remove(task.getTaskId());
    if (executorService != null) {
        executorService.shutdownNow();
        try {
            executorService.awaitTermination(5, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            LOG.warn("Awaiting shutdown of shell executor service", e);
        }
    }
    logging.stopTaskLogger(task.getTaskId(), task.getLogbackLog());
    checkIdleExecutorShutdown(task.getDriver());
}

@BeforeMethod
public void setUp() throws Exception {
    mockSession = new MockCassandraSession(CONNECTOR_ID, new CassandraClientConfig());
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    schemaProvider = new CachingCassandraSchemaProvider(CONNECTOR_ID, mockSession, executor, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES));
}

private ListenableFuture<List<TraceThreadTuple>> executeTransactions(CountDownLatch awaitLatch, CountDownLatch executeLatch, int newTransactionCount, int sampledContinuationCount, int unsampledContinuationCount) {
    final int totalTransactionCount = newTransactionCount + sampledContinuationCount + unsampledContinuationCount;
    final ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(totalTransactionCount));
    final List<ListenableFuture<TraceThreadTuple>> futures = new ArrayList<ListenableFuture<TraceThreadTuple>>();
    for (int i = 0; i < newTransactionCount; ++i) {
        futures.add(executeNewTrace(executor, awaitLatch, executeLatch));
    }
    for (int i = 0; i < sampledContinuationCount; ++i) {
        futures.add(executeSampledContinuedTrace(executor, awaitLatch, executeLatch, i));
    }
    for (int i = 0; i < unsampledContinuationCount; ++i) {
        futures.add(executeUnsampledContinuedTrace(executor, awaitLatch, executeLatch));
    }
    return Futures.allAsList(futures);
}

private boolean openWorkspace(File dir) {
    ListeningExecutorService executor = creator.getExecutor();
    PackManagerFrame frame = new PackManagerFrame();
    Deferred<?> deferred = Deferreds.makeDeferred(executor.submit(() -> {
        PackManagerController controller = new PackManagerController(frame, dir, creator);
        addRecentEntry(dir);
        return controller;
    }), executor).handleAsync(PackManagerController::show,  ex -> {
    }, SwingExecutor.INSTANCE);
    ProgressDialog.showProgress(frame, deferred, new SettableProgress("Loading...", -1), "Loading workspace...", "Loading workspace...");
    SwingHelper.addErrorDialogCallback(frame, deferred);
    return true;
}

@Test
public void appendTest() throws EventDeliveryException, InterruptedException {
    BaseLogEventPack eventPack = generateLogEventPack();
    ListeningExecutorService rpcES = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    Mockito.when(flumeEventBuilder.generateEvents(Mockito.any(BaseLogEventPack.class), Mockito.any(RecordHeader.class), Mockito.anyString())).thenReturn(Collections.singletonList(Mockito.mock(Event.class)));
    Mockito.when(flumeClientManager.sendEventsToFlumeAsync(Mockito.anyList())).thenReturn(rpcES.submit(new Callable<AppendBatchAsyncResultPojo>() {

        public AppendBatchAsyncResultPojo call() throws Exception {
            return new AppendBatchAsyncResultPojo(true, Collections.singletonList(Mockito.mock(Event.class)));
        }
    }));
    TestLogDeliveryCallback callback = new TestLogDeliveryCallback();
    appender.doAppend(eventPack, callback);
    Thread.sleep(3000);
    Assert.assertTrue(callback.success);
}

@Test
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
        TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
        ContainerContext containerContext = new ContainerContext(containerId.toString());
        ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
        getTaskFuture.get();
        assertEquals(1, umbilical.getTaskInvocations);
    } finally {
        executor.shutdownNow();
    }
}

@Test
public void testSingleSuccessfulTask() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        boolean result = taskRunnerFuture.get();
        assertTrue(result);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
    } finally {
        executor.shutdownNow();
    }
}

// Get the exact total size of the given set of files.
// Invokes multiple requests for size estimation in parallel using a thread pool.
// TODO: replace this with bulk request API when it is available. Will require updates
// to IOChannelFactory interface.
private static long getExactTotalSizeOfFiles(Collection<String> files, IOChannelFactory ioChannelFactory) throws IOException {
    List<ListenableFuture<Long>> futures = new ArrayList<>();
    ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_POOL_SIZE));
    try {
        long totalSize = 0;
        for (String file : files) {
            futures.add(createFutureForSizeEstimation(file, ioChannelFactory, service));
        }
        for (Long val : Futures.allAsList(futures).get()) {
            totalSize += val;
        }
        return totalSize;
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new IOException(e);
    } catch (ExecutionException e) {
        throw new IOException(e.getCause());
    } finally {
        service.shutdown();
    }
}

@Test
public void testSValue() throws Exception {
    // Check that we never generate an S value that is larger than half the curve order. This avoids a malleability
    // issue that can allow someone to change a transaction [hash] without invalidating the signature.
    final int ITERATIONS = 10;
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(ITERATIONS));
    List<ListenableFuture<ECKey.ECDSASignature>> sigFutures = Lists.newArrayList();
    final ECKey key = new ECKey();
    for (byte i = 0; i < ITERATIONS; i++) {
        final byte[] hash = HashUtil.sha3(new byte[] { i });
        sigFutures.add(executor.submit(new Callable<ECDSASignature>() {

            @Override
            public ECKey.ECDSASignature call() throws Exception {
                return key.doSign(hash);
            }
        }));
    }
    List<ECKey.ECDSASignature> sigs = Futures.allAsList(sigFutures).get();
    for (ECKey.ECDSASignature signature : sigs) {
        assertTrue(signature.s.compareTo(ECKey.HALF_CURVE_ORDER) <= 0);
    }
    final ECKey.ECDSASignature duplicate = new ECKey.ECDSASignature(sigs.get(0).r, sigs.get(0).s);
    assertEquals(sigs.get(0), duplicate);
    assertEquals(sigs.get(0).hashCode(), duplicate.hashCode());
}

@Override
public QueryRunner<Row> mergeRunners(final ExecutorService exec, final Iterable<QueryRunner<Row>> queryRunners) {
    // mergeRunners should take ListeningExecutorService at some point
    final ListeningExecutorService queryExecutor = MoreExecutors.listeningDecorator(exec);
    return new QueryRunner<Row>() {

        @Override
        public Sequence<Row> run(Query<Row> query, Map<String, Object> responseContext) {
            return strategySelector.strategize((GroupByQuery) query).mergeRunners(queryExecutor, queryRunners).run(query, responseContext);
        }
    };
}

// Get the exact total size of the given set of files.
// Invokes multiple requests for size estimation in parallel using a thread pool.
// TODO: replace this with bulk request API when it is available. Will require updates
// to IOChannelFactory interface.
private static long getExactTotalSizeOfFiles(Collection<String> files, IOChannelFactory ioChannelFactory) throws Exception {
    List<ListenableFuture<Long>> futures = new ArrayList<>();
    ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(THREAD_POOL_SIZE));
    long totalSize = 0;
    try {
        for (String file : files) {
            futures.add(createFutureForSizeEstimation(file, ioChannelFactory, service));
        }
        for (Long val : Futures.allAsList(futures).get()) {
            totalSize += val;
        }
        return totalSize;
    } finally {
        service.shutdown();
    }
}

@Test(timeout = 90000)
public void testEmbeddedSDCTimeout() throws Exception {
    Properties props = new Properties();
    DummyEmbeddedSDCPool dummyEmbeddedSDCPool = new DummyEmbeddedSDCPool(props);
    Assert.assertEquals(1, dummyEmbeddedSDCPool.getInstances().size());
    Assert.assertEquals(1, dummyEmbeddedSDCPool.size());
    EmbeddedSDC embeddedSDC1 = dummyEmbeddedSDCPool.checkout();
    Assert.assertEquals(0, dummyEmbeddedSDCPool.size());
    // Nothing in pool, so wait() should return null
    assertNull(dummyEmbeddedSDCPool.waitForSDC(3000));
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    WaitOnSDCRunnable th = new WaitOnSDCRunnable(dummyEmbeddedSDCPool);
    ListenableFuture future = executorService.submit(th);
    Thread.sleep(100);
    // shouldn't be done yet
    assertTrue(!future.isDone());
    assertNull(th.embeddedSDC);
    // return back sdc to pool
    dummyEmbeddedSDCPool.checkin(embeddedSDC1);
    assertNull(future.get());
    assertEquals(embeddedSDC1, th.embeddedSDC);
}

@Test
public void testSingleCollectorGetsExhausted() throws Exception {
    ListeningExecutorService executor = MoreExecutors.newDirectExecutorService();
    Ordering<Row> rowOrdering = OrderingByPosition.rowOrdering(new int[] { 0 }, new boolean[] { false }, new Boolean[] { null });
    List<OrderedDocCollector> collectors = new ArrayList<>();
    collectors.add(mockedCollector(new ShardId("p1", 0), 0, Arrays.<Row>asList(new Row1(1), new Row1(1))));
    collectors.add(mockedCollector(new ShardId("p2", 0), 2, Arrays.<Row>asList(new Row1(2), new Row1(2), new Row1(2))));
    collectors.add(mockedCollector(new ShardId("p1", 1), 10, Arrays.<Row>asList(new Row1(3), new Row1(3))));
    CollectingRowReceiver rowReceiver = CollectingRowReceiver.withLimit(6);
    FlatProjectorChain projectorChain = FlatProjectorChain.withReceivers(Collections.singletonList(rowReceiver));
    MultiShardScoreDocCollector docCollector = new MultiShardScoreDocCollector(collectors, rowOrdering, projectorChain, executor);
    docCollector.doCollect();
    Bucket result = rowReceiver.result();
    assertThat(TestingHelpers.printedTable(result), is("1\n1\n2\n2\n2\n2\n"));
}

@Before
public void setupStore() {
    InMemoryDOMDataStore operStore = new InMemoryDOMDataStore("OPER", MoreExecutors.newDirectExecutorService());
    InMemoryDOMDataStore configStore = new InMemoryDOMDataStore("CFG", MoreExecutors.newDirectExecutorService());
    schemaContext = TestModel.createTestContext();
    operStore.onGlobalContextUpdated(schemaContext);
    configStore.onGlobalContextUpdated(schemaContext);
    ImmutableMap<LogicalDatastoreType, DOMStore> stores = //
    ImmutableMap.<LogicalDatastoreType, DOMStore>builder().put(CONFIGURATION, //
    configStore).put(OPERATIONAL, //
    operStore).build();
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
    domBroker = new SerializedDOMDataBroker(stores, executor);
}

@Before
public void setupStore() {
    InMemoryDOMDataStore operStore = new InMemoryDOMDataStore("OPER", MoreExecutors.newDirectExecutorService());
    InMemoryDOMDataStore configStore = new InMemoryDOMDataStore("CFG", MoreExecutors.newDirectExecutorService());
    schemaContext = TestModel.createTestContext();
    operStore.onGlobalContextUpdated(schemaContext);
    configStore.onGlobalContextUpdated(schemaContext);
    ImmutableMap<LogicalDatastoreType, DOMStore> stores = //
    ImmutableMap.<LogicalDatastoreType, DOMStore>builder().put(CONFIGURATION, //
    configStore).put(OPERATIONAL, //
    operStore).build();
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
    domBroker = new SerializedDOMDataBroker(stores, executor);
}

@Setup(Level.Trial)
@Override
public void setUp() throws Exception {
    ListeningExecutorService dsExec = MoreExecutors.sameThreadExecutor();
    executor = MoreExecutors.listeningDecorator(MoreExecutors.getExitingExecutorService((ThreadPoolExecutor) Executors.newFixedThreadPool(1), 1L, TimeUnit.SECONDS));
    InMemoryDOMDataStore operStore = new InMemoryDOMDataStore("OPER", dsExec);
    InMemoryDOMDataStore configStore = new InMemoryDOMDataStore("CFG", dsExec);
    Map<LogicalDatastoreType, DOMStore> datastores = ImmutableMap.of(LogicalDatastoreType.OPERATIONAL, (DOMStore) operStore, LogicalDatastoreType.CONFIGURATION, configStore);
    domBroker = new SerializedDOMDataBroker(datastores, executor);
    schemaContext = BenchmarkModel.createTestContext();
    configStore.onGlobalContextUpdated(schemaContext);
    operStore.onGlobalContextUpdated(schemaContext);
    initTestNode();
}

/**
   * Executes the given list of Runnable in parallel using a fixed thread pool executor. This method blocks
   * until all runnables finished.
   */
private void parallelRun(List<? extends Runnable> runnables) {
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(runnables.size()));
    try {
        List<ListenableFuture<?>> futures = new ArrayList<>(runnables.size());
        for (Runnable r : runnables) {
            futures.add(executor.submit(r));
        }
        Futures.getUnchecked(Futures.allAsList(futures));
    } finally {
        executor.shutdownNow();
        try {
            executor.awaitTermination(60, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            LOG.error("Interrupted", e);
        }
    }
}

@Test
public void testCloseAsync() throws Exception {
    ListeningExecutorService directExecutor = MoreExecutors.newDirectExecutorService();
    final AtomicBoolean didClose = new AtomicBoolean(false);
    try (AsyncCloseable asyncCloseable = new AsyncCloseable(directExecutor)) {
        asyncCloseable.closeAsync(new AutoCloseable() {

            @Override
            public void close() throws Exception {
                didClose.set(true);
            }
        });
    }
    assertThat(didClose.get(), Matchers.is(true));
}

@Test(expected = StepFailedException.class, timeout = 5000)
public void testParallelStepFailure() throws StepFailedException, InterruptedException, IOException {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();
    steps.add(new SleepingStep(0, 0));
    steps.add(new SleepingStep(10, 1));
    // Add a step that will also fail, but taking longer than the test timeout to complete.
    // This tests the fail-fast behaviour of runStepsInParallelAndWait (that is, since the 10ms
    // step will fail so quickly, the result of the 5000ms step will not be observed).
    steps.add(new SleepingStep(5000, 1));
    ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
    DefaultStepRunner runner = new DefaultStepRunner(TestExecutionContext.newInstance());
    runner.runStepsInParallelAndWait(steps.build(), Optional.<BuildTarget>absent(), service, StepRunner.NOOP_CALLBACK);
// Success if the test timeout is not reached.
}

@Before
public void setUp() {
    ListeningExecutorService executor = listeningDecorator(MostExecutors.newMultiThreadExecutor("UnskippedRulesTracker", 7));
    RuleDepsCache depsCache = new RuleDepsCache(executor);
    unskippedRulesTracker = new UnskippedRulesTracker(depsCache, executor);
    eventBus = new BuckEventBus(new FakeClock(1), new BuildId());
    eventBus.register(new Object() {

        @Subscribe
        public void onUnskippedRuleCountUpdated(BuckEvent event) {
            events.add(event);
        }
    });
    ruleH = createRule("//:h");
    ruleG = createRule("//:g");
    ruleF = createRule("//:f");
    ruleE = createRule("//:e", ImmutableSet.of(ruleG, ruleH));
    ruleD = createRule("//:d", ImmutableSet.of(ruleG), ImmutableSet.of(ruleF));
    ruleC = createRule("//:c", ImmutableSet.of(ruleD, ruleE));
    ruleB = createRule("//:b", ImmutableSet.<BuildRule>of(), ImmutableSet.of(ruleD));
    ruleA = createRule("//:a", ImmutableSet.of(ruleD));
}

TargetNode<?> getNode(QueryTarget target) throws QueryException, InterruptedException {
    Preconditions.checkState(target instanceof QueryBuildTarget);
    ListeningExecutorService executor = null;
    try {
        executor = com.google.common.util.concurrent.MoreExecutors.listeningDecorator(MostExecutors.newSingleThreadExecutor("buck query.getNode"));
        return params.getParser().getTargetNode(params.getBuckEventBus(), params.getCell(), enableProfiling, executor, ((QueryBuildTarget) target).getBuildTarget());
    } catch (BuildTargetExceptionBuildFileParseException |  e) {
        throw new QueryException("Error getting target node for %s\n%s", target, e.getMessage());
    } finally {
        if (executor != null) {
            executor.shutdown();
        }
    }
}

public static void showGraphAndPerformCallBack(final IViewContainer container, final INaviView view, final CGraphWindow window, final Window parent, final FutureCallback<Boolean> callBack) {
    CWindowManager.instance().bringViewToFront(view);
    final ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(10));
    final ListenableFuture<Boolean> loader = service.submit(generateViewLoader(view, container, window, parent));
    Futures.addCallback(loader, callBack);
}

@Ignore
@Test
@BenchmarkOptions(callgc = true, clock = Clock.REAL_TIME, warmupRounds = 10, benchmarkRounds = 20)
public void testConcurrentAddRead() throws InterruptedException, ExecutionException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
    final int taskCount = 30;
    final int concurrentThreads = 3;
    final int elementsPerThread = 1 << 15;
    final OnheapIncrementalIndex incrementalIndex = this.incrementalIndex.getConstructor(Long.TYPE, QueryGranularity.class, AggregatorFactory[].class, Integer.TYPE).newInstance(0, QueryGranularities.NONE, factories, elementsPerThread * taskCount);
    final ArrayList<AggregatorFactory> queryAggregatorFactories = new ArrayList<>(dimensionCount + 1);
    queryAggregatorFactories.add(new CountAggregatorFactory("rows"));
    for (int i = 0; i < dimensionCount; ++i) {
        queryAggregatorFactories.add(new LongSumAggregatorFactory(String.format("sumResult%s", i), String.format("sumResult%s", i)));
        queryAggregatorFactories.add(new DoubleSumAggregatorFactory(String.format("doubleSumResult%s", i), String.format("doubleSumResult%s", i)));
    }
    final ListeningExecutorService indexExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(concurrentThreads, new ThreadFactoryBuilder().setDaemon(false).setNameFormat("index-executor-%d").setPriority(Thread.MIN_PRIORITY).build()));
    final ListeningExecutorService queryExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(concurrentThreads, new ThreadFactoryBuilder().setDaemon(false).setNameFormat("query-executor-%d").build()));
    final long timestamp = System.currentTimeMillis();
    final Interval queryInterval = new Interval("1900-01-01T00:00:00Z/2900-01-01T00:00:00Z");
    final List<ListenableFuture<?>> indexFutures = new LinkedList<>();
    final List<ListenableFuture<?>> queryFutures = new LinkedList<>();
    final Segment incrementalIndexSegment = new IncrementalIndexSegment(incrementalIndex, null);
    final QueryRunnerFactory factory = new TimeseriesQueryRunnerFactory(new TimeseriesQueryQueryToolChest(QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()), new TimeseriesQueryEngine(), QueryRunnerTestHelper.NOOP_QUERYWATCHER);
    final AtomicInteger currentlyRunning = new AtomicInteger(0);
    final AtomicBoolean concurrentlyRan = new AtomicBoolean(false);
    final AtomicBoolean someoneRan = new AtomicBoolean(false);
    for (int j = 0; j < taskCount; j++) {
        indexFutures.add(indexExecutor.submit(new Runnable() {

            @Override
            public void run() {
                currentlyRunning.incrementAndGet();
                try {
                    for (int i = 0; i < elementsPerThread; i++) {
                        incrementalIndex.add(getLongRow(timestamp + i, 1, dimensionCount));
                    }
                } catch (IndexSizeExceededException e) {
                    throw Throwables.propagate(e);
                }
                currentlyRunning.decrementAndGet();
                someoneRan.set(true);
            }
        }));
        queryFutures.add(queryExecutor.submit(new Runnable() {

            @Override
            public void run() {
                QueryRunner<Result<TimeseriesResultValue>> runner = new FinalizeResultsQueryRunner<Result<TimeseriesResultValue>>(factory.createRunner(incrementalIndexSegment), factory.getToolchest());
                TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource("xxx").granularity(QueryGranularities.ALL).intervals(ImmutableList.of(queryInterval)).aggregators(queryAggregatorFactories).build();
                Map<String, Object> context = new HashMap<String, Object>();
                for (Result<TimeseriesResultValue> result : Sequences.toList(runner.run(query, context), new LinkedList<Result<TimeseriesResultValue>>())) {
                    if (someoneRan.get()) {
                        Assert.assertTrue(result.getValue().getDoubleMetric("doubleSumResult0") > 0);
                    }
                }
                if (currentlyRunning.get() > 0) {
                    concurrentlyRan.set(true);
                }
            }
        }));
    }
    List<ListenableFuture<?>> allFutures = new ArrayList<>(queryFutures.size() + indexFutures.size());
    allFutures.addAll(queryFutures);
    allFutures.addAll(indexFutures);
    Futures.allAsList(allFutures).get();
    //Assert.assertTrue("Did not hit concurrency, please try again", concurrentlyRan.get());
    queryExecutor.shutdown();
    indexExecutor.shutdown();
    QueryRunner<Result<TimeseriesResultValue>> runner = new FinalizeResultsQueryRunner<Result<TimeseriesResultValue>>(factory.createRunner(incrementalIndexSegment), factory.getToolchest());
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource("xxx").granularity(QueryGranularities.ALL).intervals(ImmutableList.of(queryInterval)).aggregators(queryAggregatorFactories).build();
    Map<String, Object> context = new HashMap<String, Object>();
    List<Result<TimeseriesResultValue>> results = Sequences.toList(runner.run(query, context), new LinkedList<Result<TimeseriesResultValue>>());
    final int expectedVal = elementsPerThread * taskCount;
    for (Result<TimeseriesResultValue> result : results) {
        Assert.assertEquals(elementsPerThread, result.getValue().getLongMetric("rows").intValue());
        for (int i = 0; i < dimensionCount; ++i) {
            Assert.assertEquals(String.format("Failed long sum on dimension %d", i), expectedVal, result.getValue().getLongMetric(String.format("sumResult%s", i)).intValue());
            Assert.assertEquals(String.format("Failed double sum on dimension %d", i), expectedVal, result.getValue().getDoubleMetric(String.format("doubleSumResult%s", i)).intValue());
        }
    }
}

@Test(timeout = 5000)
public void testConcurrentRequests() throws InterruptedException {
    int timeoutSecond = 5;
    int concurThread = 10;
    int exceptionCount = 0;
    List<Callable<Object>> tasks = new ArrayList<Callable<Object>>();
    List<ListenableFuture<Object>> pendingTasks = new ArrayList<ListenableFuture<Object>>();
    final ExecutorService callbackExecutor = Executors.newFixedThreadPool(concurThread, new ThreadFactoryBuilder().setDaemon(false).setNameFormat("CallbackExecutor").build());
    ListeningExecutorService taskExecutorService = MoreExecutors.listeningDecorator(callbackExecutor);
    while (concurThread > 0) {
        ListenableFuture<Object> runningTaskFuture = taskExecutorService.submit(new EnvironmentRequest());
        pendingTasks.add(runningTaskFuture);
        concurThread--;
    }
    //waiting for all threads submitted to thread pool
    for (ListenableFuture<Object> future : pendingTasks) {
        try {
            future.get();
        } catch (ExecutionException e) {
            exceptionCount++;
        }
    }
    //stop accepting new threads and shutdown threadpool
    taskExecutorService.shutdown();
    try {
        if (!taskExecutorService.awaitTermination(timeoutSecond, TimeUnit.SECONDS)) {
            taskExecutorService.shutdownNow();
        }
    } catch (InterruptedException ie) {
        taskExecutorService.shutdownNow();
    }
    assertEquals(0, exceptionCount);
}

/**
   BenchmarkIndexibleWrites.TestConcurrentReads[0]: [measured 100 out of 200 rounds, threads: 1 (sequential)]
   round: 0.28 [+- 0.02], round.block: 0.00 [+- 0.00], round.gc: 0.02 [+- 0.00], GC.calls: 396, GC.time: 1.84, time.total: 59.98, time.warmup: 30.51, time.bench: 29.48
   BenchmarkIndexibleWrites.TestConcurrentReads[1]: [measured 100 out of 200 rounds, threads: 1 (sequential)]
   round: 0.12 [+- 0.01], round.block: 0.00 [+- 0.00], round.gc: 0.02 [+- 0.00], GC.calls: 396, GC.time: 2.05, time.total: 29.21, time.warmup: 14.65, time.bench: 14.55

   */
@BenchmarkOptions(warmupRounds = 100, benchmarkRounds = 100, clock = Clock.REAL_TIME, callgc = true)
@Ignore
@Test
public void testConcurrentReads() throws ExecutionException, InterruptedException {
    final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(concurrentThreads, new ThreadFactoryBuilder().setDaemon(false).setNameFormat("indexible-writes-benchmark-reader-%d").build()));
    final AtomicInteger index = new AtomicInteger(0);
    final AtomicInteger queryableIndex = new AtomicInteger(0);
    List<ListenableFuture<?>> futures = new LinkedList<>();
    final Integer loops = totalIndexSize / concurrentThreads;
    final AtomicBoolean done = new AtomicBoolean(false);
    final CountDownLatch start = new CountDownLatch(1);
    for (int i = 0; i < concurrentThreads; ++i) {
        futures.add(executorService.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    start.await();
                } catch (InterruptedException e) {
                    throw Throwables.propagate(e);
                }
                final Random rndGen = new Random();
                while (!done.get()) {
                    Integer idx = rndGen.nextInt(queryableIndex.get() + 1);
                    Assert.assertEquals(idx, concurrentIndexible.get(idx));
                }
            }
        }));
    }
    {
        final Integer idx = index.getAndIncrement();
        concurrentIndexible.set(idx, idx);
        start.countDown();
    }
    for (int i = 1; i < totalIndexSize; ++i) {
        final Integer idx = index.getAndIncrement();
        concurrentIndexible.set(idx, idx);
        queryableIndex.incrementAndGet();
    }
    done.set(true);
    Futures.allAsList(futures).get();
    executorService.shutdown();
    Assert.assertTrue(String.format("Index too small %d, expected %d across %d loops", index.get(), totalIndexSize, loops), index.get() >= totalIndexSize);
    for (int i = 0; i < index.get(); ++i) {
        Assert.assertEquals(i, concurrentIndexible.get(i).intValue());
    }
    concurrentIndexible.clear();
    futures.clear();
}

@BenchmarkOptions(warmupRounds = 100, benchmarkRounds = 100, clock = Clock.REAL_TIME, callgc = true)
@Ignore
@Test
public /**
   * CALLEN - 2015-01-15 - OSX - Java 1.7.0_71-b14
   BenchmarkIndexibleWrites.testConcurrentWrites[0]: [measured 100 out of 200 rounds, threads: 1 (sequential)]
   round: 0.24 [+- 0.01], round.block: 0.00 [+- 0.00], round.gc: 0.02 [+- 0.00], GC.calls: 396, GC.time: 1.88, time.total: 50.60, time.warmup: 24.84, time.bench: 25.77
   BenchmarkIndexibleWrites.testConcurrentWrites[1]: [measured 100 out of 200 rounds, threads: 1 (sequential)]
   round: 0.15 [+- 0.01], round.block: 0.00 [+- 0.00], round.gc: 0.02 [+- 0.00], GC.calls: 396, GC.time: 2.11, time.total: 33.14, time.warmup: 16.09, time.bench: 17.05
   */
void testConcurrentWrites() throws ExecutionException, InterruptedException {
    final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(concurrentThreads, new ThreadFactoryBuilder().setDaemon(false).setNameFormat("indexible-writes-benchmark-%d").build()));
    final AtomicInteger index = new AtomicInteger(0);
    List<ListenableFuture<?>> futures = new LinkedList<>();
    final Integer loops = totalIndexSize / concurrentThreads;
    for (int i = 0; i < concurrentThreads; ++i) {
        futures.add(executorService.submit(new Runnable() {

            @Override
            public void run() {
                for (int i = 0; i < loops; ++i) {
                    final Integer idx = index.getAndIncrement();
                    concurrentIndexible.set(idx, idx);
                }
            }
        }));
    }
    Futures.allAsList(futures).get();
    Assert.assertTrue(String.format("Index too small %d, expected %d across %d loops", index.get(), totalIndexSize, loops), index.get() >= totalIndexSize);
    for (int i = 0; i < index.get(); ++i) {
        Assert.assertEquals(i, concurrentIndexible.get(i).intValue());
    }
    concurrentIndexible.clear();
    futures.clear();
    executorService.shutdown();
}

@Test
public void testConcurrentUpload() throws IOException, InterruptedException, ExecutionException, TimeoutException {
    final int concurrency = 10;
    ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(concurrency));
    // barrier ensures that all jobs try to add files to classpath at same time.
    final CyclicBarrier barrier = new CyclicBarrier(concurrency);
    final DistributedFileSystem fs = miniCluster.getFileSystem();
    final Path expectedJarPath = new Path(finalClasspath, dummyJarFile.getName());
    List<ListenableFuture<Boolean>> futures = new ArrayList<>();
    for (int i = 0; i < concurrency; i++) {
        futures.add(pool.submit(new Callable() {

            @Override
            public Boolean call() throws Exception {
                int id = barrier.await();
                Job job = Job.getInstance(conf, "test-job-" + id);
                Path intermediatePathForJob = new Path(intermediatePath, "job-" + id);
                JobHelper.addJarToClassPath(dummyJarFile, finalClasspath, intermediatePathForJob, fs, job);
                // check file gets uploaded to final HDFS path
                Assert.assertTrue(fs.exists(expectedJarPath));
                // check that the intermediate file is not present
                Assert.assertFalse(fs.exists(new Path(intermediatePathForJob, dummyJarFile.getName())));
                // check file gets added to the classpath
                Assert.assertEquals(expectedJarPath.toString(), job.getConfiguration().get(MRJobConfig.CLASSPATH_FILES));
                return true;
            }
        }));
    }
    Futures.allAsList(futures).get(30, TimeUnit.SECONDS);
    pool.shutdownNow();
}

@Test
public void closeWhenRunningJobs() throws Exception {
    Cell cell = EasyMock.createMock(Cell.class);
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    final CountDownLatch waitTillClosed = new CountDownLatch(1);
    final CountDownLatch firstJobRunning = new CountDownLatch(1);
    final AtomicInteger postCloseWork = new AtomicInteger(0);
    ImmutableSet<ListenableFuture<?>> futures;
    try (ProjectBuildFileParserPool parserPool = new ProjectBuildFileParserPool(/* maxParsers */
    1, createMockPaserFactory(new IAnswer<List<Map<String, Object>>>() {

        @Override
        public List<Map<String, Object>> answer() throws Throwable {
            firstJobRunning.countDown();
            waitTillClosed.await();
            return ImmutableList.of();
        }
    }))) {
        futures = scheduleWork(cell, parserPool, executorService, 5);
        for (ListenableFuture<?> future : futures) {
            Futures.transform(future, new Function<Object, Object>() {

                @Override
                public Object apply(Object input) {
                    postCloseWork.incrementAndGet();
                    return null;
                }
            });
        }
        firstJobRunning.await(1, TimeUnit.SECONDS);
    }
    waitTillClosed.countDown();
    List<Object> futureResults = Futures.successfulAsList(futures).get(1, TimeUnit.SECONDS);
    // The threadpool is of size 1, so we had 1 job in the 'running' state. That one job completed
    // normally, the rest should have been cancelled.
    int expectedCompletedJobs = 1;
    int completedJobs = FluentIterable.from(futureResults).filter(Predicates.notNull()).size();
    assertThat(completedJobs, Matchers.equalTo(expectedCompletedJobs));
    executorService.shutdown();
    assertThat(executorService.awaitTermination(1, TimeUnit.SECONDS), Matchers.is(true));
    assertThat(postCloseWork.get(), Matchers.equalTo(expectedCompletedJobs));
}

@Test
public void testSameThread() {
    local.set("whatup");
    ListeningExecutorService sameThreadExecutor = MoreExecutors.sameThreadExecutor();
    sameThreadExecutor.submit(PTExecutors.wrap(Callables.returning(null)));
    Assert.assertEquals("whatup", local.get());
}

@Test(timeout = 5000)
public void testKilledAfterComplete() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2ForTest taskRunner = createTaskRunnerForTest(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TestProcessor.awaitCompletion();
        taskRunner.awaitCallableCompletion();
        taskRunner.killTask();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);
        assertFalse(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testTaskKilled() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        taskRunner.killTask();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.KILL_REQUESTED, null, false, null);
        TestProcessor.awaitCompletion();
        assertTrue(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        // Kill events are not sent over the umbilical at the moment.
        umbilical.verifyNoCompletionEvents();
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testTaskSelfKill() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_SELF_KILL_AND_COMPLETE);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_KILL_REQUEST, createProcessorIOException(), false, null);
        TestProcessor.awaitCompletion();
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskKilledEvent(KILL_START_STRING, IOException.class.getName() + ": " + IOException.class.getSimpleName());
        assertTrue(TestProcessor.wasAborted());
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testSignalNonFatalAndThrow() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_SIGNAL_NON_FATAL_AND_THROW);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.NON_FATAL);
        TestProcessor.awaitCompletion();
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent(FAILURE_START_STRING, IOException.class.getName() + ": " + IOException.class.getSimpleName(), TaskFailureType.NON_FATAL);
        assertTrue(TestProcessor.wasAborted());
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testSignalFatalAndThrow() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_SIGNAL_FATAL_AND_THROW);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.FATAL);
        TestProcessor.awaitCompletion();
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent(FAILURE_START_STRING, IOException.class.getName() + ": " + IOException.class.getSimpleName(), TaskFailureType.FATAL);
        assertTrue(TestProcessor.wasAborted());
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testSignalDeprecatedFatalErrorAndLoop() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_SIGNAL_DEPRECATEDFATAL_AND_LOOP);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TestProcessor.awaitLoop();
        // The fatal error should have caused an interrupt.
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.NON_FATAL);
        TestProcessor.awaitCompletion();
        assertTrue(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent(FAILURE_START_STRING, IOException.class.getName() + ": " + IOException.class.getSimpleName());
        // Signal fatal error should cause the processor to fail.
        assertTrue(TestProcessor.wasAborted());
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        umbilical.signalSendShouldDie();
        umbilical.awaitRegisteredEvent();
        // Not signaling an actual start to verify task interruption
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.CONTAINER_STOP_REQUESTED, null, true, null);
        TestProcessor.awaitCompletion();
        assertTrue(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        // TODO Is this statement correct ?
        // No completion events since shouldDie was requested by the AM, which should have killed the
        // task.
        umbilical.verifyNoCompletionEvents();
        assertTrue(TestProcessor.wasAborted());
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testHeartbeatException() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        umbilical.signalThrowException();
        umbilical.awaitRegisteredEvent();
        // Not signaling an actual start to verify task interruption
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.COMMUNICATION_FAILURE, new IOException("IOException"), TaskExecutionTestHelpers.HEARTBEAT_EXCEPTION_STRING, false, TaskFailureType.NON_FATAL);
        TestProcessor.awaitCompletion();
        assertTrue(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        // No completion events since umbilical communication already failed.
        umbilical.verifyNoCompletionEvents();
        assertTrue(TestProcessor.wasAborted());
    } finally {
        executor.shutdownNow();
    }
}

// test task failed due to exception in Processor
@Test(timeout = 5000)
public void testFailedTaskIOException() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_THROW_IO_EXCEPTION);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.NON_FATAL);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent(FAILURE_START_STRING, IOException.class.getName() + ": " + IOException.class.getSimpleName());
        // Failure detected as a result of fall off from the run method. abort isn't required.
        assertFalse(TestProcessor.wasAborted());
        assertTrue(taskRunner.task.getCounters().countCounters() != 0);
    } finally {
        executor.shutdownNow();
    }
}

// Test task failed due to Processor class not found
@Test(timeout = 5000)
public void testFailedTask2() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, "NotExitedProcessor", TestProcessor.CONF_EMPTY, false, true);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_ERROR, new TezReflectionException("TezReflectionException"), false, TaskFailureType.NON_FATAL);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent(FAILURE_START_STRING, ":org.apache.tez.dag.api.TezReflectionException: " + "Unable to load class: NotExitedProcessor");
        // Failure detected as a result of fall off from the run method. abort isn't required.
        assertFalse(TestProcessor.wasAborted());
        assertTrue(taskRunner.task.getCounters().countCounters() != 0);
    } finally {
        executor.shutdownNow();
    }
}

// test task failed due to exception in Processor
@Test(timeout = 5000)
public void testFailedTaskTezException() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_THROW_TEZ_EXCEPTION);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorTezException(), false, TaskFailureType.NON_FATAL);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent(FAILURE_START_STRING, TezException.class.getName() + ": " + TezException.class.getSimpleName());
        // Failure detected as a result of fall off from the run method. abort isn't required.
        assertFalse(TestProcessor.wasAborted());
        assertTrue(taskRunner.task.getCounters().countCounters() != 0);
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testMultipleSuccessfulTasks() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY, true);
        LogicalIOProcessorRuntimeTask runtimeTask = taskRunner.task;
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
        assertFalse(TestProcessor.wasAborted());
        umbilical.resetTrackedEvents();
        TezCounters tezCounters = runtimeTask.getCounters();
        verifySysCounters(tezCounters, 5, 5);
        taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY, false);
        runtimeTask = taskRunner.task;
        // Setup the executor
        taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
        assertFalse(TestProcessor.wasAborted());
        tezCounters = runtimeTask.getCounters();
        verifySysCounters(tezCounters, -1, -1);
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeout = 5000)
public void testSingleSuccessfulTask() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        TaskRunner2Result result = taskRunnerFuture.get();
        verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
        assertFalse(TestProcessor.wasAborted());
    } finally {
        executor.shutdownNow();
    }
}

@BeforeMethod
public void setUp() throws Exception {
    mockClient = new MockHiveMetastoreClient();
    MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    metastore = new CachingHiveMetastore(mockHiveCluster, executor, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES));
    stats = ((CachingHiveMetastore) metastore).getStats();
}

@Test(timeOut = 30_000)
public void testOnlyShadowCellsUpdateIsExecuted(ITestContext context) throws Exception {
    CommitTable.Client commitTableClient = getCommitTable(context).getClient();
    PostCommitActions syncPostCommitter = spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
    ListeningExecutorService postCommitExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
    PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
    TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
    final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Do not invoke real method simulating a fail of the async clean of commit table entry
            removeCommitTableEntryCalledLatch.countDown();
            return null;
        }
    }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
    try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
        // Execute tx with async post commit actions
        Transaction tx1 = tm.begin();
        Put put1 = new Put(row1);
        put1.add(family, qualifier, Bytes.toBytes("hey!"));
        txTable.put(tx1, put1);
        Put put2 = new Put(row2);
        put2.add(family, qualifier, Bytes.toBytes("hou!"));
        txTable.put(tx1, put2);
        tm.commit(tx1);
        long tx1Id = tx1.getTransactionId();
        // We continue when the unsuccessful call of the method for cleaning commit table has been invoked
        removeCommitTableEntryCalledLatch.await();
        // We check that the shadow cells are there (because the update of the shadow cells should precede
        // the cleaning of the commit table entry) ...
        assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // ... and the commit table entry has NOT been cleaned
        Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        // Final checks
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
    }
}

@Test(timeOut = 30_000)
public void testNoAsyncPostActionsAreCalled(ITestContext context) throws Exception {
    CommitTable.Client commitTableClient = getCommitTable(context).getClient();
    PostCommitActions syncPostCommitter = spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
    ListeningExecutorService postCommitExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
    PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
    TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
    final CountDownLatch updateShadowCellsCalledLatch = new CountDownLatch(1);
    final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
    // Simulate shadow cells are not updated and commit table is not clean
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Do not invoke real method simulating a fail of the shadow cells update
            updateShadowCellsCalledLatch.countDown();
            return null;
        }
    }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Do not invoke real method simulating a fail of the async clean of commit table entry
            removeCommitTableEntryCalledLatch.countDown();
            return null;
        }
    }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
    try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
        // Execute tx with async post commit actions
        Transaction tx1 = tm.begin();
        Put put1 = new Put(row1);
        put1.add(family, qualifier, Bytes.toBytes("hey!"));
        txTable.put(tx1, put1);
        Put put2 = new Put(row2);
        put2.add(family, qualifier, Bytes.toBytes("hou!"));
        txTable.put(tx1, put2);
        tm.commit(tx1);
        long tx1Id = tx1.getTransactionId();
        // The shadow cells shouldn't be there...
        assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // ... and the should NOT have been cleaned
        Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        updateShadowCellsCalledLatch.await();
        // Not even after waiting for the method call on the shadow cells update...
        assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        removeCommitTableEntryCalledLatch.await();
        // ... and after waiting for the method call for cleaning the commit table entry
        commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        // Final checks
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
    }
}

@Test(timeOut = 30_000)
public void testPostCommitActionsAreCalledAsynchronously(ITestContext context) throws Exception {
    CommitTable.Client commitTableClient = getCommitTable(context).getClient();
    PostCommitActions syncPostCommitter = spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
    ListeningExecutorService postCommitExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
    PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
    TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
    final CountDownLatch beforeUpdatingShadowCellsLatch = new CountDownLatch(1);
    final CountDownLatch afterUpdatingShadowCellsLatch = new CountDownLatch(1);
    final CountDownLatch beforeRemovingCTEntryLatch = new CountDownLatch(1);
    final CountDownLatch afterRemovingCTEntryLatch = new CountDownLatch(1);
    doAnswer(new Answer<ListenableFuture<Void>>() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            try {
                beforeUpdatingShadowCellsLatch.await();
                invocation.callRealMethod();
                afterUpdatingShadowCellsLatch.countDown();
            } catch (Throwable throwable) {
                throwable.printStackTrace();
            }
            return SettableFuture.create();
        }
    }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
    doAnswer(new Answer<ListenableFuture<Void>>() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            try {
                beforeRemovingCTEntryLatch.await();
                LOG.info("We are here");
                invocation.callRealMethod();
                afterRemovingCTEntryLatch.countDown();
            } catch (Throwable throwable) {
                throwable.printStackTrace();
            }
            return SettableFuture.create();
        }
    }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
    try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
        // Execute tx with async post commit actions
        Transaction tx1 = tm.begin();
        Put put1 = new Put(row1);
        put1.add(family, qualifier, Bytes.toBytes("hey!"));
        txTable.put(tx1, put1);
        Put put2 = new Put(row2);
        put2.add(family, qualifier, Bytes.toBytes("hou!"));
        txTable.put(tx1, put2);
        tm.commit(tx1);
        long tx1Id = tx1.getTransactionId();
        // As we have paused the update of shadow cells, the shadow cells shouldn't be there yet
        assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // Commit Table should contain an entry for the transaction
        Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
        // Read from row1 and row2 in a different Tx and check that result is the data written by tx1 despite the
        // post commit actions have not been executed yet (the shadow cells healing process should make its work)
        Transaction tx2 = tm.begin();
        Get get1 = new Get(row1);
        Result result = txTable.get(tx2, get1);
        byte[] value = result.getValue(family, qualifier);
        assertNotNull(value);
        assertEquals("hey!", Bytes.toString(value));
        Get get2 = new Get(row2);
        result = txTable.get(tx2, get2);
        value = result.getValue(family, qualifier);
        assertNotNull(value);
        assertEquals("hou!", Bytes.toString(value));
        // Then, we continue with the update of shadow cells and we wait till completed
        beforeUpdatingShadowCellsLatch.countDown();
        afterUpdatingShadowCellsLatch.await();
        // Now we can check that the shadow cells are there...
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // ...and the transaction entry is still in the Commit Table
        commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
        // Finally, we continue till the Commit Table cleaning process is done...
        beforeRemovingCTEntryLatch.countDown();
        afterRemovingCTEntryLatch.await();
        // ...so now, the Commit Table should NOT contain the entry for the transaction anymore
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
        commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertFalse(commitTimestamp.isPresent());
        // Final checks
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
    }
}

private Observable<Boolean> parMap(Observable<List<Sample>> samples, ExecutorService executorSvc, final MetricRegistry metrics, final Func1<List<Sample>, Boolean> insert) {
    final ListeningExecutorService executor = MoreExecutors.listeningDecorator(executorSvc);
    return samples.lift(new Operator<ListenableFuture<Boolean>, List<Sample>>() {

        @Override
        public Subscriber<? super List<Sample>> call(final Subscriber<? super ListenableFuture<Boolean>> s) {
            return new Subscriber<List<Sample>>() {

                @Override
                public void onCompleted() {
                    if (!s.isUnsubscribed()) {
                        s.onCompleted();
                    }
                    executor.shutdown();
                }

                @Override
                public void onError(Throwable e) {
                    if (!s.isUnsubscribed()) {
                        s.onError(e);
                    }
                }

                @Override
                public void onNext(final List<Sample> t) {
                    if (!s.isUnsubscribed()) {
                        try {
                            ListenableFuture<Boolean> f = executor.submit(new Callable<Boolean>() {

                                @Override
                                public Boolean call() throws Exception {
                                    return insert.call(t);
                                }
                            });
                            s.onNext(f);
                        } catch (Throwable ex) {
                            onError(ex);
                        }
                    }
                }
            };
        }
    }).observeOn(Schedulers.io()).map(new Func1<ListenableFuture<Boolean>, Boolean>() {

        @Override
        public Boolean call(ListenableFuture<Boolean> f) {
            try {
                return f.get();
            } catch (Throwable e) {
                throw Exceptions.propagate(e);
            }
        }
    });
}

@Override
public int execute(Configuration conf) throws IOException {
    KMeansInitStrategy initStrategy = KMeansInitStrategy.valueOf(initStrategyName);
    KMeans kmeans = new KMeans(initStrategy, getUpdateStrategy());
    ListeningExecutorService exec;
    if (numThreads <= 1) {
        exec = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
    } else {
        exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads));
    }
    List<MLWeightedCenters> mlwc = AvroIO.read(MLWeightedCenters.class, new File(sketchFile));
    List<List<Weighted<Vector>>> sketches = toSketches(mlwc);
    List<Weighted<Vector>> allPoints = Lists.newArrayList();
    for (List<Weighted<Vector>> sketch : sketches) {
        allPoints.addAll(sketch);
    }
    List<Centers> centers = getClusters(exec, allPoints, kmeans);
    AvroIO.write(Lists.transform(centers, VectorConvert.FROM_CENTERS), new File(centersOutputFile));
    if (sketches.size() > 1) {
        // Perform the prediction strength calculations on the folds
        List<Weighted<Vector>> train = Lists.newArrayList();
        for (int i = 0; i < sketches.size() - 1; i++) {
            train.addAll(sketches.get(i));
        }
        List<Weighted<Vector>> test = sketches.get(sketches.size() - 1);
        List<Centers> trainCenters = getClusters(exec, train, kmeans);
        List<Centers> testCenters = getClusters(exec, test, kmeans);
        KMeansEvaluation eval = new KMeansEvaluation(testCenters, test, trainCenters, detailsFileName);
        eval.writeStatsToFile(new File(statsFileName));
        eval.writeStats(System.out);
    }
    return 0;
}

@Override
protected void execute(TaskListener listener) {
    final ImmutableList.Builder<ListenableFuture<?>> deletedNodesBuilder = ImmutableList.builder();
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Computer.threadPoolForRemoting);
    final ImmutableList.Builder<MesosComputer> computersToDeleteBuilder = ImmutableList.builder();
    for (final Computer c : getJenkins().getComputers()) {
        if (MesosComputer.class.isInstance(c)) {
            MesosSlave mesosSlave = (MesosSlave) c.getNode();
            if (mesosSlave != null && mesosSlave.isPendingDelete()) {
                final MesosComputer comp = (MesosComputer) c;
                computersToDeleteBuilder.add(comp);
                logger.log(Level.INFO, "Marked " + comp.getName() + " for deletion");
                ListenableFuture<?> f = executor.submit(new Runnable() {

                    public void run() {
                        logger.log(Level.INFO, "Deleting pending node " + comp.getName());
                        try {
                            MesosSlave node = comp.getNode();
                            if (node != null) {
                                node.terminate();
                            }
                        } catch (RuntimeException e) {
                            logger.log(Level.WARNING, "Failed to disconnect and delete " + comp.getName() + ": " + e.getMessage());
                            throw e;
                        }
                    }
                });
                deletedNodesBuilder.add(f);
            } else {
                logger.log(Level.FINE, c.getName() + " with slave " + mesosSlave + " is not pending deletion or the slave is null");
            }
        } else {
            logger.log(Level.FINER, c.getName() + " is not a mesos computer, it is a " + c.getClass().getName());
        }
    }
    Futures.getUnchecked(Futures.successfulAsList(deletedNodesBuilder.build()));
    for (MesosComputer c : computersToDeleteBuilder.build()) {
        try {
            c.deleteSlave();
        } catch (IOException e) {
            logger.log(Level.WARNING, "Failed to disconnect and delete " + c.getName() + ": " + e.getMessage());
        } catch (InterruptedException e) {
            logger.log(Level.WARNING, "Failed to disconnect and delete " + c.getName() + ": " + e.getMessage());
        }
    }
}

/**
     * Prevent further downloads from being queued and download queued files.
     *
     * @throws InterruptedException thrown on interruption
     * @throws IOException thrown on I/O error
     */
public void execute() throws InterruptedException, IOException {
    synchronized (this) {
        queue = Collections.unmodifiableList(queue);
    }
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threadCount));
    try {
        List<ListenableFuture<?>> futures = new ArrayList<ListenableFuture<?>>();
        synchronized (this) {
            for (HttpDownloadJob job : queue) {
                futures.add(executor.submit(job));
            }
        }
        try {
            Futures.allAsList(futures).get();
        } catch (ExecutionException e) {
            throw new IOException("Something went wrong", e);
        }
        synchronized (this) {
            if (failed.size() > 0) {
                throw new IOException(failed.size() + " file(s) could not be downloaded");
            }
        }
    } finally {
        executor.shutdownNow();
    }
}

@Test
public void setFromFunctionBlocks() throws ExecutionException, InterruptedException, TimeoutException {
    ListeningExecutorService executor = listeningDecorator(newFixedThreadPool(2));
    try {
        final CountDownLatch latch = new CountDownLatch(1);
        ListenableFuture<Boolean> t1 = executor.submit(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                latch.await();
                return revisions.setHead(Functions.<RecordId>identity());
            }
        });
        try {
            t1.get(500, MILLISECONDS);
            fail("SetHead from function should block");
        } catch (TimeoutException expected) {
        }
        ListenableFuture<Boolean> t2 = executor.submit(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                latch.countDown();
                return revisions.setHead(Functions.<RecordId>identity());
            }
        });
        assertTrue(t2.get(500, MILLISECONDS));
        assertTrue(t1.get(500, MILLISECONDS));
    } finally {
        executor.shutdown();
    }
}

@Test
public void concurrentSetHeadFromFunction() throws IOException, InterruptedException, ExecutionException, TimeoutException {
    ListeningExecutorService executor = listeningDecorator(newFixedThreadPool(2));
    try {
        ListenableFuture<Boolean> t1 = executor.submit(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return revisions.setHead(new Function<RecordId, RecordId>() {

                    @Nullable
                    @Override
                    public RecordId apply(RecordId headId) {
                        return addChild(reader.readNode(headId), "a").getRecordId();
                    }
                });
            }
        });
        ListenableFuture<Boolean> t2 = executor.submit(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return revisions.setHead(new Function<RecordId, RecordId>() {

                    @Nullable
                    @Override
                    public RecordId apply(RecordId headId) {
                        return addChild(reader.readNode(headId), "b").getRecordId();
                    }
                });
            }
        });
        assertTrue(t1.get(500, MILLISECONDS));
        assertTrue(t2.get(500, MILLISECONDS));
        SegmentNodeState root = reader.readNode(revisions.getHead());
        assertTrue(root.hasChildNode("a"));
        assertTrue(root.hasChildNode("b"));
    } finally {
        executor.shutdown();
    }
}

@Test
public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        umbilical.signalSendShouldDie();
        umbilical.awaitRegisteredEvent();
        // Not signaling an actual start to verify task interruption
        boolean result = taskRunnerFuture.get();
        assertFalse(result);
        TestProcessor.awaitCompletion();
        assertTrue(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        // TODO Is this statement correct ?
        // No completion events since shouldDie was requested by the AM, which should have killed the
        // task.
        umbilical.verifyNoCompletionEvents();
    } finally {
        executor.shutdownNow();
    }
}

@Test
public void testHeartbeatException() throws IOException, InterruptedException, TezException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        umbilical.signalThrowException();
        umbilical.awaitRegisteredEvent();
        // Not signaling an actual start to verify task interruption
        try {
            taskRunnerFuture.get();
            fail("Expecting the task to fail");
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            assertTrue(cause instanceof IOException);
            assertTrue(cause.getMessage().contains(HEARTBEAT_EXCEPTION_STRING));
        }
        TestProcessor.awaitCompletion();
        assertTrue(TestProcessor.wasInterrupted());
        assertNull(taskReporter.currentCallable);
        // No completion events since umbilical communication already failed.
        umbilical.verifyNoCompletionEvents();
    } finally {
        executor.shutdownNow();
    }
}

@Test
public void testFailedTask() throws IOException, InterruptedException, TezException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_THROW_TEZ_EXCEPTION);
        // Setup the executor
        Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
        // Signal the processor to go through
        TestProcessor.awaitStart();
        TestProcessor.signal();
        try {
            taskRunnerFuture.get();
            fail("Expecting the task to fail");
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            LOG.info(cause.getClass().getName());
            assertTrue(cause instanceof TezException);
        }
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskFailedEvent();
    } finally {
        executor.shutdownNow();
    }
}

@Test
public void testMultipleSuccessfulTasks() throws IOException, InterruptedException, TezException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
        TaskReporter taskReporter = createTaskReporter(appId, umbilical);
        TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        boolean result = taskRunnerFuture.get();
        assertTrue(result);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
        umbilical.resetTrackedEvents();
        taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY);
        // Setup the executor
        taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
        // Signal the processor to go through
        TestProcessor.signal();
        result = taskRunnerFuture.get();
        assertTrue(result);
        assertNull(taskReporter.currentCallable);
        umbilical.verifyTaskSuccessEvent();
    } finally {
        executor.shutdownNow();
    }
}

@Test(timeOut = 30_000)
public void testOnlyShadowCellsUpdateIsExecuted(ITestContext context) throws Exception {
    CommitTable.Client commitTableClient = getCommitTable(context).getClient();
    PostCommitActions syncPostCommitter = spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
    ListeningExecutorService postCommitExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
    PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
    TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
    final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Do not invoke real method simulating a fail of the async clean of commit table entry
            removeCommitTableEntryCalledLatch.countDown();
            return null;
        }
    }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
    try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
        // Execute tx with async post commit actions
        Transaction tx1 = tm.begin();
        Put put1 = new Put(row1);
        put1.add(family, qualifier, Bytes.toBytes("hey!"));
        txTable.put(tx1, put1);
        Put put2 = new Put(row2);
        put2.add(family, qualifier, Bytes.toBytes("hou!"));
        txTable.put(tx1, put2);
        tm.commit(tx1);
        long tx1Id = tx1.getTransactionId();
        // We continue when the unsuccessful call of the method for cleaning commit table has been invoked
        removeCommitTableEntryCalledLatch.await();
        // We check that the shadow cells are there (because the update of the shadow cells should precede
        // the cleaning of the commit table entry) ...
        assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // ... and the commit table entry has NOT been cleaned
        Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        // Final checks
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
    }
}

@Test(timeOut = 30_000)
public void testNoAsyncPostActionsAreCalled(ITestContext context) throws Exception {
    CommitTable.Client commitTableClient = getCommitTable(context).getClient();
    PostCommitActions syncPostCommitter = spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
    ListeningExecutorService postCommitExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
    PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
    TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
    final CountDownLatch updateShadowCellsCalledLatch = new CountDownLatch(1);
    final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
    // Simulate shadow cells are not updated and commit table is not clean
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Do not invoke real method simulating a fail of the shadow cells update
            updateShadowCellsCalledLatch.countDown();
            return null;
        }
    }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Do not invoke real method simulating a fail of the async clean of commit table entry
            removeCommitTableEntryCalledLatch.countDown();
            return null;
        }
    }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
    try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
        // Execute tx with async post commit actions
        Transaction tx1 = tm.begin();
        Put put1 = new Put(row1);
        put1.add(family, qualifier, Bytes.toBytes("hey!"));
        txTable.put(tx1, put1);
        Put put2 = new Put(row2);
        put2.add(family, qualifier, Bytes.toBytes("hou!"));
        txTable.put(tx1, put2);
        tm.commit(tx1);
        long tx1Id = tx1.getTransactionId();
        // The shadow cells shouldn't be there...
        assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // ... and the should NOT have been cleaned
        Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        updateShadowCellsCalledLatch.await();
        // Not even after waiting for the method call on the shadow cells update...
        assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        removeCommitTableEntryCalledLatch.await();
        // ... and after waiting for the method call for cleaning the commit table entry
        commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        // Final checks
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
    }
}

@Test(timeOut = 30_000)
public void testPostCommitActionsAreCalledAsynchronously(ITestContext context) throws Exception {
    CommitTable.Client commitTableClient = getCommitTable(context).getClient();
    PostCommitActions syncPostCommitter = spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
    ListeningExecutorService postCommitExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
    PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
    TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
    final CountDownLatch beforeUpdatingShadowCellsLatch = new CountDownLatch(1);
    final CountDownLatch afterUpdatingShadowCellsLatch = new CountDownLatch(1);
    final CountDownLatch beforeRemovingCTEntryLatch = new CountDownLatch(1);
    final CountDownLatch afterRemovingCTEntryLatch = new CountDownLatch(1);
    doAnswer(new Answer<ListenableFuture<Void>>() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            try {
                beforeUpdatingShadowCellsLatch.await();
                invocation.callRealMethod();
                afterUpdatingShadowCellsLatch.countDown();
            } catch (Throwable throwable) {
                throwable.printStackTrace();
            }
            return SettableFuture.create();
        }
    }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
    doAnswer(new Answer<ListenableFuture<Void>>() {

        public ListenableFuture<Void> answer(InvocationOnMock invocation) {
            try {
                beforeRemovingCTEntryLatch.await();
                LOG.info("We are here");
                invocation.callRealMethod();
                afterRemovingCTEntryLatch.countDown();
            } catch (Throwable throwable) {
                throwable.printStackTrace();
            }
            return SettableFuture.create();
        }
    }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
    try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
        // Execute tx with async post commit actions
        Transaction tx1 = tm.begin();
        Put put1 = new Put(row1);
        put1.add(family, qualifier, Bytes.toBytes("hey!"));
        txTable.put(tx1, put1);
        Put put2 = new Put(row2);
        put2.add(family, qualifier, Bytes.toBytes("hou!"));
        txTable.put(tx1, put2);
        tm.commit(tx1);
        long tx1Id = tx1.getTransactionId();
        // As we have paused the update of shadow cells, the shadow cells shouldn't be there yet
        assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // Commit Table should contain an entry for the transaction
        Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
        // Read from row1 and row2 in a different Tx and check that result is the data written by tx1 despite the
        // post commit actions have not been executed yet (the shadow cells healing process should make its work)
        Transaction tx2 = tm.begin();
        Get get1 = new Get(row1);
        Result result = txTable.get(tx2, get1);
        byte[] value = result.getValue(family, qualifier);
        assertNotNull(value);
        assertEquals("hey!", Bytes.toString(value));
        Get get2 = new Get(row2);
        result = txTable.get(tx2, get2);
        value = result.getValue(family, qualifier);
        assertNotNull(value);
        assertEquals("hou!", Bytes.toString(value));
        // Then, we continue with the update of shadow cells and we wait till completed
        beforeUpdatingShadowCellsLatch.countDown();
        afterUpdatingShadowCellsLatch.await();
        // Now we can check that the shadow cells are there...
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
        // ...and the transaction entry is still in the Commit Table
        commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertTrue(commitTimestamp.isPresent());
        assertTrue(commitTimestamp.get().isValid());
        assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
        // Finally, we continue till the Commit Table cleaning process is done...
        beforeRemovingCTEntryLatch.countDown();
        afterRemovingCTEntryLatch.await();
        // ...so now, the Commit Table should NOT contain the entry for the transaction anymore
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
        commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
        assertFalse(commitTimestamp.isPresent());
        // Final checks
        verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
        verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
    }
}

@Test
public void testEventQueue() throws Exception {
    // initialize the queue
    ClusterEventBlockingQueue queue = new ClusterEventBlockingQueue();
    // add an event
    ClusterEvent event1 = new ClusterEvent("event1");
    queue.put(event1);
    Assert.assertEquals(queue.size(), 1);
    // add an event with a different name
    ClusterEvent event2 = new ClusterEvent("event2");
    queue.put(event2);
    Assert.assertEquals(queue.size(), 2);
    // add an event with the same name as event1 (should not change queue size)
    ClusterEvent newEvent1 = new ClusterEvent("event1");
    newEvent1.addAttribute("attr", 1);
    queue.put(newEvent1);
    Assert.assertEquals(queue.size(), 2);
    // test peek
    ClusterEvent peeked = queue.peek();
    Assert.assertEquals(peeked.getName(), "event1");
    Assert.assertEquals(peeked.getAttribute("attr"), 1);
    Assert.assertEquals(queue.size(), 2);
    // test take the head
    ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
    ClusterEvent takenEvent1 = safeTake(queue, service);
    Assert.assertEquals(takenEvent1.getName(), "event1");
    Assert.assertEquals(takenEvent1.getAttribute("attr"), 1);
    Assert.assertEquals(queue.size(), 1);
    // test take the tail
    ClusterEvent takenEvent2 = safeTake(queue, service);
    Assert.assertEquals(takenEvent2.getName(), "event2");
    Assert.assertEquals(queue.size(), 0);
}

private void executeChangeOps(boolean parallel) throws UpdateException, RestApiException {
    ListeningExecutorService executor = parallel ? changeUpdateExector : MoreExecutors.newDirectExecutorService();
    List<ChangeTask> tasks = new ArrayList<>(ops.keySet().size());
    try {
        if (!ops.isEmpty() && notesMigration.failChangeWrites()) {
            // this is a programmer error.
            throw new OrmException(NoteDbUpdateManager.CHANGES_READ_ONLY);
        }
        List<ListenableFuture<?>> futures = new ArrayList<>(ops.keySet().size());
        for (Map.Entry<Change.Id, Collection<Op>> e : ops.asMap().entrySet()) {
            ChangeTask task = new ChangeTask(e.getKey(), e.getValue(), Thread.currentThread());
            tasks.add(task);
            futures.add(executor.submit(task));
        }
        Futures.allAsList(futures).get();
        if (notesMigration.commitChangeWrites()) {
            executeNoteDbUpdates(tasks);
        }
    } catch (ExecutionExceptionInterruptedException |  e) {
        Throwables.propagateIfInstanceOf(e.getCause(), UpdateException.class);
        Throwables.propagateIfInstanceOf(e.getCause(), RestApiException.class);
        throw new UpdateException(e);
    } catch (OrmException e) {
        throw new UpdateException(e);
    }
    // Reindex changes.
    for (ChangeTask task : tasks) {
        if (task.deleted) {
            indexFutures.add(indexer.deleteAsync(task.id));
        } else {
            indexFutures.add(indexer.indexAsync(project, task.id));
        }
    }
}

@Override
public int run() throws Exception {
    mustHaveValidSite();
    dbInjector = createDbInjector(MULTI_USER);
    threads = ThreadLimiter.limitThreads(dbInjector, threads);
    LifecycleManager dbManager = new LifecycleManager();
    dbManager.add(dbInjector);
    dbManager.start();
    sysInjector = createSysInjector();
    sysInjector.injectMembers(this);
    if (!notesMigration.enabled()) {
        throw die("NoteDb is not enabled.");
    }
    LifecycleManager sysManager = new LifecycleManager();
    sysManager.add(sysInjector);
    sysManager.start();
    ListeningExecutorService executor = newExecutor();
    System.out.println("Rebuilding the NoteDb");
    final ImmutableMultimap<Project.NameKey, Change.Id> changesByProject = getChangesByProject();
    boolean ok;
    Stopwatch sw = Stopwatch.createStarted();
    try (Repository allUsersRepo = repoManager.openRepository(allUsersName)) {
        deleteRefs(RefNames.REFS_DRAFT_COMMENTS, allUsersRepo);
        List<ListenableFuture<Boolean>> futures = new ArrayList<>();
        List<Project.NameKey> projectNames = Ordering.usingToString().sortedCopy(changesByProject.keySet());
        for (final Project.NameKey project : projectNames) {
            ListenableFuture<Boolean> future = executor.submit(new Callable<Boolean>() {

                @Override
                public Boolean call() {
                    try (ReviewDb db = unwrapDb(schemaFactory.open())) {
                        return rebuilder.rebuildProject(db, changesByProject, project, allUsersRepo);
                    } catch (Exception e) {
                        log.error("Error rebuilding project " + project, e);
                        return false;
                    }
                }
            });
            futures.add(future);
        }
        try {
            ok = Iterables.all(Futures.allAsList(futures).get(), Predicates.equalTo(true));
        } catch (InterruptedExceptionExecutionException |  e) {
            log.error("Error rebuilding projects", e);
            ok = false;
        }
    }
    double t = sw.elapsed(TimeUnit.MILLISECONDS) / 1000d;
    System.out.format("Rebuild %d changes in %.01fs (%.01f/s)\n", changesByProject.size(), t, changesByProject.size() / t);
    return ok ? 0 : 1;
}

@Test
public void anyFutureTest() throws ExecutionException, InterruptedException {
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(new ThreadPoolExecutor(16, 16, 1L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()));
    AnyFuture<Integer> anyFuture = new AnyFuture<Integer>() {

        @Override
        protected void postProcess(Integer integer) {
            System.out.println("FutureTest.postProcess:" + "integer = [" + integer + "]");
        }
    };
    for (int i = 0; i < 4; i++) {
        final int ii = i;
        ListenableFuture<Integer> future = executor.submit(new Callable<Integer>() {

            @Override
            public Integer call() throws Exception {
                System.out.println("Waiting " + ii);
                Thread.sleep(5000 - ii * 500);
                System.out.println("Complete " + ii);
                return ii;
            }
        });
        anyFuture.add(future);
    }
    Thread.sleep(1000);
    anyFuture.cancel(true);
    System.out.println("Getting anyFuture...");
    System.out.println("anyFuture: " + anyFuture.isCancelled() + ", " + anyFuture.isDone());
    anyFuture = new AnyFuture<Integer>() {

        @Override
        protected void postProcess(Integer integer) {
            System.out.println("FutureTest.postProcess:" + "integer = [" + integer + "]");
        }
    };
    for (int i = 0; i < 4; i++) {
        final int ii = i;
        ListenableFuture<Integer> future = executor.submit(new Callable<Integer>() {

            @Override
            public Integer call() throws Exception {
                System.out.println("Waiting " + ii);
                Thread.sleep(5000 - ii * 500);
                System.out.println("Complete " + ii);
                return ii;
            }
        });
        anyFuture.add(future);
    }
    System.out.println("Getting anyFuture...");
    System.out.println("anyFuture.get(): " + anyFuture.get() + ", " + anyFuture.isCancelled() + ", " + anyFuture.isDone());
}

@Test
public void interruptTest() throws InterruptedException {
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
    final ListenableFuture<Object> future = executor.submit(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            //                try {
            System.out.println("Waiting");
            Thread.sleep(10000);
            System.out.println("Complete");
            return null;
        //                } catch (Exception e) {
        //                    e.printStackTrace();
        //                    throw e;
        //                }
        }
    });
    future.addListener(new Runnable() {

        @Override
        public void run() {
            System.out.println("Listener: " + future.isCancelled() + ", " + future.isDone());
            try {
                future.get();
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            } catch (ExecutionException e) {
                throw new RuntimeException(e);
            }
        }
    }, MoreExecutors.sameThreadExecutor());
    Thread.sleep(1000);
    future.cancel(true);
}

@Test
public void testSameTimeZnode() throws Exception {
    final int numThreads = INITIAL_SEGMENTS / 10;
    final ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads));
    segmentAnnouncer.announceSegments(testSegments);
    waitForSync(batchServerInventoryView, testSegments);
    DruidServer server = Iterables.get(batchServerInventoryView.getInventory(), 0);
    final Set<DataSegment> segments = Sets.newHashSet(server.getSegments().values());
    Assert.assertEquals(testSegments, segments);
    final CountDownLatch latch = new CountDownLatch(numThreads);
    final List<ListenableFuture<BatchDataSegmentAnnouncer>> futures = new ArrayList<>();
    for (int i = 0; i < numThreads; ++i) {
        final int ii = i;
        futures.add(executor.submit(new Callable<BatchDataSegmentAnnouncer>() {

            @Override
            public BatchDataSegmentAnnouncer call() {
                BatchDataSegmentAnnouncer segmentAnnouncer = new BatchDataSegmentAnnouncer(new DruidServerMetadata("id", "host", Long.MAX_VALUE, "type", "tier", 0), new BatchDataSegmentAnnouncerConfig() {

                    @Override
                    public int getSegmentsPerNode() {
                        return 50;
                    }
                }, new ZkPathsConfig() {

                    @Override
                    public String getBase() {
                        return testBasePath;
                    }
                }, announcer, jsonMapper);
                segmentAnnouncer.start();
                List<DataSegment> segments = new ArrayList<DataSegment>();
                try {
                    for (int j = 0; j < INITIAL_SEGMENTS / numThreads; ++j) {
                        segments.add(makeSegment(INITIAL_SEGMENTS + ii + numThreads * j));
                    }
                    latch.countDown();
                    latch.await();
                    segmentAnnouncer.announceSegments(segments);
                    testSegments.addAll(segments);
                } catch (Exception e) {
                    throw Throwables.propagate(e);
                }
                return segmentAnnouncer;
            }
        }));
    }
    final List<BatchDataSegmentAnnouncer> announcers = Futures.<BatchDataSegmentAnnouncer>allAsList(futures).get();
    Assert.assertEquals(INITIAL_SEGMENTS * 2, testSegments.size());
    waitForSync(batchServerInventoryView, testSegments);
    Assert.assertEquals(testSegments, Sets.newHashSet(server.getSegments().values()));
    for (int i = 0; i < INITIAL_SEGMENTS; ++i) {
        final DataSegment segment = makeSegment(100 + i);
        segmentAnnouncer.unannounceSegment(segment);
        testSegments.remove(segment);
    }
    waitForSync(batchServerInventoryView, testSegments);
    Assert.assertEquals(testSegments, Sets.newHashSet(server.getSegments().values()));
}

@Test(timeout = 60000)
public void testKnownSizeConcurrency() throws Exception {
    final int numThreads = 20;
    ListeningExecutorService threadPoolExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads));
    List<ListenableFuture<?>> results = new ArrayList<>();
    for (int i = 0; i < numThreads; ++i) {
        results.add(threadPoolExecutor.submit(new Runnable() {

            @Override
            public void run() {
                ByteBuffer compressed = ByteBuffer.wrap(compressionStrategy.getCompressor().compress(originalData));
                ByteBuffer output = ByteBuffer.allocate(originalData.length);
                // TODO: Lambdas would be nice here whenever we use Java 8
                compressionStrategy.getDecompressor().decompress(compressed, compressed.array().length, output, originalData.length);
                byte[] checkArray = new byte[DATA_SIZER];
                output.get(checkArray);
                Assert.assertArrayEquals("Uncompressed data does not match", originalData, checkArray);
            }
        }));
    }
    Futures.allAsList(results).get();
}

@Override
public QueryRunner<SegmentAnalysis> mergeRunners(ExecutorService exec, Iterable<QueryRunner<SegmentAnalysis>> queryRunners) {
    final ListeningExecutorService queryExecutor = MoreExecutors.listeningDecorator(exec);
    return new ConcatQueryRunner<SegmentAnalysis>(Sequences.map(Sequences.simple(queryRunners), new Function<QueryRunner<SegmentAnalysis>, QueryRunner<SegmentAnalysis>>() {

        @Override
        public QueryRunner<SegmentAnalysis> apply(final QueryRunner<SegmentAnalysis> input) {
            return new QueryRunner<SegmentAnalysis>() {

                @Override
                public Sequence<SegmentAnalysis> run(final Query<SegmentAnalysis> query, final Map<String, Object> responseContext) {
                    final int priority = BaseQuery.getContextPriority(query, 0);
                    ListenableFuture<Sequence<SegmentAnalysis>> future = queryExecutor.submit(new AbstractPrioritizedCallable<Sequence<SegmentAnalysis>>(priority) {

                        @Override
                        public Sequence<SegmentAnalysis> call() throws Exception {
                            return Sequences.simple(Sequences.toList(input.run(query, responseContext), new ArrayList<SegmentAnalysis>()));
                        }
                    });
                    try {
                        queryWatcher.registerQuery(query, future);
                        final Number timeout = query.getContextValue(QueryContextKeys.TIMEOUT, (Number) null);
                        return timeout == null ? future.get() : future.get(timeout.longValue(), TimeUnit.MILLISECONDS);
                    } catch (InterruptedException e) {
                        log.warn(e, "Query interrupted, cancelling pending results, query id [%s]", query.getId());
                        future.cancel(true);
                        throw new QueryInterruptedException(e);
                    } catch (CancellationException e) {
                        throw new QueryInterruptedException(e);
                    } catch (TimeoutException e) {
                        log.info("Query timeout, cancelling pending results for query id [%s]", query.getId());
                        future.cancel(true);
                        throw new QueryInterruptedException(e);
                    } catch (ExecutionException e) {
                        throw Throwables.propagate(e.getCause());
                    }
                }
            };
        }
    }));
}

// This is very fast when run locally. Speed on Travis completely depends on noisy neighbors.
@Test(timeout = 600_000)
public void testConcurrentAddDelete() throws ExecutionException, InterruptedException, TimeoutException {
    final int threads = 10;
    final int deletesPerThread = 5;
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Execs.multiThreaded(threads, "concurrentTestingPool-%s"));
    final CountDownLatch latch = new CountDownLatch(threads);
    Collection<ListenableFuture<?>> futures = new ArrayList<>();
    for (int i = 0; i < threads; ++i) {
        final int ii = i;
        futures.add(executorService.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    latch.countDown();
                    if (!latch.await(5, TimeUnit.SECONDS)) {
                        throw new RuntimeException(new TimeoutException("Took too long to wait for more tasks"));
                    }
                    for (int j = 0; j < deletesPerThread; ++j) {
                        testDelete(String.format("ns-%d-%d", ii, j));
                    }
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw Throwables.propagate(e);
                }
            }
        }));
    }
    // Create an all-encompassing exception if any of them failed
    final Collection<Exception> exceptions = new ArrayList<>();
    try {
        for (ListenableFuture<?> future : futures) {
            try {
                future.get();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw e;
            } catch (Exception e) {
                exceptions.add(e);
            }
        }
        if (!exceptions.isEmpty()) {
            final RuntimeException e = new RuntimeException("Futures failed");
            for (Exception ex : exceptions) {
                e.addSuppressed(ex);
            }
        }
    } finally {
        executorService.shutdownNow();
    }
    checkNoMoreRunning();
}

/**
     * Similar to {@link #runWithAvailableThreads(ThreadPoolExecutor, Collection)}
     * but this function will return a Future that wraps the futures of each callable
     *
     * @param executor executor that is used to execute the callableList
     * @param poolSize the corePoolSize of the given executor
     * @param callableCollection a collection of callable that should be executed
     * @param mergeFunction function that will be applied to merge the results of multiple callable in case that they are
     *                      executed together if the threadPool is exhausted
     * @param <T> type of the final result
     * @return a future that will return a list of the results of the callableList
     * @throws RejectedExecutionException
     */
public static <T> ListenableFuture<List<T>> runWithAvailableThreads(ThreadPoolExecutor executor, int poolSize, Collection<Callable<T>> callableCollection, final Function<List<T>, T> mergeFunction) throws RejectedExecutionException {
    ListeningExecutorService listeningExecutorService = MoreExecutors.listeningDecorator(executor);
    List<ListenableFuture<T>> futures;
    int availableThreads = Math.max(poolSize - executor.getActiveCount(), 1);
    if (availableThreads < callableCollection.size()) {
        Iterable<List<Callable<T>>> partition = Iterables.partition(callableCollection, callableCollection.size() / availableThreads);
        futures = new ArrayList<>(availableThreads + 1);
        for (final List<Callable<T>> callableList : partition) {
            futures.add(listeningExecutorService.submit(new Callable<T>() {

                @Override
                public T call() throws Exception {
                    List<T> results = new ArrayList<T>(callableList.size());
                    for (Callable<T> tCallable : callableList) {
                        results.add(tCallable.call());
                    }
                    return mergeFunction.apply(results);
                }
            }));
        }
    } else {
        futures = new ArrayList<>(callableCollection.size());
        for (Callable<T> callable : callableCollection) {
            futures.add(listeningExecutorService.submit(callable));
        }
    }
    return Futures.allAsList(futures);
}

public BindingTestContext getTestContext() {
    Preconditions.checkState(executor != null, "Executor is not set.");
    ListeningExecutorService listenableExecutor = MoreExecutors.listeningDecorator(executor);
    return new BindingTestContext(listenableExecutor, getClassPool(), startWithParsedSchema);
}

@Before
public void setUp() {
    ListeningExecutorService executor = MoreExecutors.sameThreadExecutor();
    BindingBrokerTestFactory factory = new BindingBrokerTestFactory();
    factory.setExecutor(executor);
    factory.setStartWithParsedSchema(getStartWithSchema());
    testContext = factory.getTestContext();
    testContext.start();
    baDataService = testContext.getBindingDataBroker();
}

public ListenableFuture<Integer> loadSavedAsync() {
    final ListeningExecutorService es = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
    final long start = System.nanoTime();
    ListenableFuture<Integer> cacheLoad = es.submit(new Callable<Integer>() {

        @Override
        public Integer call() {
            return loadSaved();
        }
    });
    cacheLoad.addListener(new Runnable() {

        @Override
        public void run() {
            if (size() > 0)
                logger.info("Completed loading ({} ms; {} keys) {} cache", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start), CacheService.instance.keyCache.size(), cacheType);
            es.shutdown();
        }
    }, MoreExecutors.directExecutor());
    return cacheLoad;
}

@Override
public void run() throws InvalidBenchmarkException {
    ImmutableSet<Experiment> allExperiments = selector.selectExperiments();
    // TODO(lukes): move this standard-out handling into the ConsoleOutput class?
    stdout.println("Experiment selection: ");
    stdout.println("  Benchmark Methods:   " + FluentIterable.from(allExperiments).transform(new Function<Experiment, String>() {

        @Override
        public String apply(Experiment experiment) {
            return experiment.instrumentation().benchmarkMethod().getName();
        }
    }).toSet());
    stdout.println("  Instruments:   " + FluentIterable.from(selector.instruments()).transform(new Function<Instrument, String>() {

        @Override
        public String apply(Instrument instrument) {
            return instrument.name();
        }
    }));
    stdout.println("  User parameters:   " + selector.userParameters());
    stdout.println("  Virtual machines:  " + FluentIterable.from(selector.vms()).transform(new Function<VirtualMachine, String>() {

        @Override
        public String apply(VirtualMachine vm) {
            return vm.name;
        }
    }));
    stdout.println("  Selection type:    " + selector.selectionType());
    stdout.println();
    if (allExperiments.isEmpty()) {
        throw new InvalidBenchmarkException("There were no experiments to be performed for the class %s using the instruments %s", benchmarkClass.benchmarkClass().getSimpleName(), instruments);
    }
    stdout.format("This selection yields %s experiments.%n", allExperiments.size());
    stdout.flush();
    // always dry run first.
    ImmutableSet<Experiment> experimentsToRun = dryRun(allExperiments);
    if (experimentsToRun.size() != allExperiments.size()) {
        stdout.format("%d experiments were skipped.%n", allExperiments.size() - experimentsToRun.size());
    }
    if (experimentsToRun.isEmpty()) {
        throw new InvalidBenchmarkException("All experiments were skipped.");
    }
    if (options.dryRun()) {
        return;
    }
    stdout.flush();
    int totalTrials = experimentsToRun.size() * options.trialsPerScenario();
    Stopwatch stopwatch = Stopwatch.createStarted();
    List<ScheduledTrial> trials = createScheduledTrials(experimentsToRun, totalTrials);
    final ListeningExecutorService executor = executorProvider.get();
    List<ListenableFuture<TrialResult>> pendingTrials = scheduleTrials(trials, executor);
    ConsoleOutput output = new ConsoleOutput(stdout, totalTrials, stopwatch);
    try {
        // Process results as they complete.
        for (ListenableFuture<TrialResult> trialFuture : inCompletionOrder(pendingTrials)) {
            try {
                TrialResult result = trialFuture.get();
                output.processTrial(result);
                for (ResultProcessor resultProcessor : resultProcessors) {
                    resultProcessor.processTrial(result.getTrial());
                }
            } catch (ExecutionException e) {
                if (e.getCause() instanceof TrialFailureException) {
                    output.processFailedTrial((TrialFailureException) e.getCause());
                } else {
                    for (ListenableFuture<?> toCancel : pendingTrials) {
                        toCancel.cancel(true);
                    }
                    throw Throwables.propagate(e.getCause());
                }
            } catch (InterruptedException e) {
                for (ListenableFuture<?> toCancel : pendingTrials) {
                    toCancel.cancel(true);
                }
                throw new RuntimeException(e);
            }
        }
    } finally {
        executor.shutdown();
        output.close();
    }
    for (ResultProcessor resultProcessor : resultProcessors) {
        try {
            resultProcessor.close();
        } catch (IOException e) {
            logger.log(WARNING, "Could not close a result processor: " + resultProcessor, e);
        }
    }
}

@Test
public void workThatThrows() throws Exception {
    Cell cell = EasyMock.createMock(Cell.class);
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    final String exceptionMessage = "haha!";
    final AtomicBoolean throwWhileParsing = new AtomicBoolean(true);
    try (ProjectBuildFileParserPool parserPool = new ProjectBuildFileParserPool(/* maxParsers */
    2, createMockPaserFactory(new IAnswer<List<Map<String, Object>>>() {

        @Override
        public List<Map<String, Object>> answer() throws Throwable {
            if (throwWhileParsing.get()) {
                throw new Exception(exceptionMessage);
            }
            return ImmutableList.of();
        }
    }))) {
        ImmutableSet<ListenableFuture<?>> failedWork = scheduleWork(cell, parserPool, executorService, 5);
        for (ListenableFuture<?> failedFuture : failedWork) {
            try {
                failedFuture.get();
                fail("Expected ExecutionException to be thrown.");
            } catch (ExecutionException e) {
                assertThat(e.getCause().getMessage(), Matchers.equalTo(exceptionMessage));
            }
        }
        // Make sure it's still possible to do work.
        throwWhileParsing.set(false);
        Futures.allAsList(scheduleWork(cell, parserPool, executorService, 5)).get();
    } finally {
        executorService.shutdown();
    }
}

@Test
public void ignoresCancellation() throws Exception {
    Cell cell = EasyMock.createMock(Cell.class);
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    int numberOfJobs = 5;
    final CountDownLatch waitTillAllWorkIsDone = new CountDownLatch(numberOfJobs);
    final CountDownLatch waitTillCanceled = new CountDownLatch(1);
    try (ProjectBuildFileParserPool parserPool = new ProjectBuildFileParserPool(/* maxParsers */
    1, createMockPaserFactory(new IAnswer<List<Map<String, Object>>>() {

        @Override
        public List<Map<String, Object>> answer() throws Throwable {
            waitTillCanceled.await();
            waitTillAllWorkIsDone.countDown();
            return ImmutableList.of();
        }
    }))) {
        ImmutableSet<ListenableFuture<?>> futures = scheduleWork(cell, parserPool, executorService, numberOfJobs);
        for (ListenableFuture<?> future : futures) {
            future.cancel(true);
        }
        waitTillCanceled.countDown();
        // We're making sure cancel is ignored by the pool by waiting for the supposedly canceled
        // work to go through.
        waitTillAllWorkIsDone.await(1, TimeUnit.SECONDS);
    } finally {
        executorService.shutdown();
    }
}

@Test
public void fuzzForConcurrentAccess() throws Exception {
    final int parsersCount = 3;
    Cell cell = EasyMock.createMock(Cell.class);
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(4));
    try (ProjectBuildFileParserPool parserPool = new ProjectBuildFileParserPool(parsersCount, new Function<Cell, ProjectBuildFileParser>() {

        @Override
        public ProjectBuildFileParser apply(Cell input) {
            final AtomicInteger sleepCallCount = new AtomicInteger(0);
            return createMockPaser(new IAnswer<List<Map<String, Object>>>() {

                @Override
                public List<Map<String, Object>> answer() throws Throwable {
                    int numCalls = sleepCallCount.incrementAndGet();
                    Preconditions.checkState(numCalls == 1);
                    try {
                        Thread.sleep(10);
                    } finally {
                        sleepCallCount.decrementAndGet();
                    }
                    return ImmutableList.of();
                }
            });
        }
    })) {
        Futures.allAsList(scheduleWork(cell, parserPool, executorService, 142)).get();
    } finally {
        executorService.shutdown();
    }
}

@Test
public void closesCreatedParsers() throws Exception {
    final int parsersCount = 4;
    final AtomicInteger parserCount = new AtomicInteger(0);
    Cell cell = EasyMock.createMock(Cell.class);
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(parsersCount));
    final CountDownLatch createParserLatch = new CountDownLatch(parsersCount);
    try (ProjectBuildFileParserPool parserPool = new ProjectBuildFileParserPool(parsersCount, new Function<Cell, ProjectBuildFileParser>() {

        @Override
        public ProjectBuildFileParser apply(Cell input) {
            parserCount.incrementAndGet();
            final ProjectBuildFileParser parser = EasyMock.createMock(ProjectBuildFileParser.class);
            try {
                EasyMock.expect(parser.getAllRulesAndMetaRules(EasyMock.anyObject(Path.class))).andAnswer(new IAnswer<List<Map<String, Object>>>() {

                    @Override
                    public List<Map<String, Object>> answer() throws Throwable {
                        createParserLatch.countDown();
                        createParserLatch.await();
                        return ImmutableList.of();
                    }
                }).anyTimes();
                parser.close();
                EasyMock.expectLastCall().andAnswer(new IAnswer<Void>() {

                    @Override
                    public Void answer() throws Throwable {
                        parserCount.decrementAndGet();
                        return null;
                    }
                });
            } catch (Exception e) {
                Throwables.propagate(e);
            }
            EasyMock.replay(parser);
            return parser;
        }
    })) {
        Futures.allAsList(scheduleWork(cell, parserPool, executorService, parsersCount * 2)).get();
        assertThat(parserCount.get(), Matchers.is(4));
    } finally {
        executorService.shutdown();
    }
    // Parser shutdown is async.
    for (int i = 0; i < 10; ++i) {
        if (parserCount.get() == 0) {
            break;
        }
        Thread.sleep(100);
    }
    assertThat(parserCount.get(), Matchers.is(0));
}

@Test
public void createsConstrainedNumberOfParsers() throws Exception {
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    assertHowManyParserInstancesAreCreated(/* executor */
    executorService, /* maxParsers */
    2, /* requests */
    3, /* expectedCreateCount */
    2);
}

@Test
public void whenSeparateTestFailsThenBuildFails() throws Exception {
    CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
    final TestResults failingTestResults = FakeTestResults.of(ImmutableList.of(new TestCaseSummary("TestCase", ImmutableList.of(new TestResultSummary("TestCaseResult", "failTest", ResultType.FAILURE, 5000, null, null, null, null)))));
    BuildTarget failingTestTarget = BuildTargetFactory.newInstance("//:failingtest");
    FakeTestRule failingTest = new FakeTestRule(new FakeBuildRuleParamsBuilder(failingTestTarget).build(), new SourcePathResolver(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer())), ImmutableSet.<Label>of(), Optional.of(Paths.get("failingTestStep1OutputDir")), // runTestSeparately
    true, ImmutableList.<Step>of(), new Callable<TestResults>() {

        @Override
        public TestResults call() {
            return failingTestResults;
        }
    });
    ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
    FakeBuildEngine fakeBuildEngine = new FakeBuildEngine(ImmutableMap.of(failingTestTarget, BuildResult.success(failingTest, BUILT_LOCALLY, CacheResult.miss())), ImmutableMap.of(failingTestTarget, new RuleKey("00")));
    ExecutionContext fakeExecutionContext = TestExecutionContext.newInstance();
    DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
    int ret = TestRunning.runTests(commandRunnerParams, ImmutableList.<TestRule>of(failingTest), fakeExecutionContext, DEFAULT_OPTIONS, service, fakeBuildEngine, stepRunner);
    assertThat(ret, equalTo(TestRunning.TEST_FAILURES_EXIT_CODE));
}

@Test
public void whenAllTestsAreSeparateTestsRunInOrder() throws Exception {
    CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
    AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
    ExecutionOrderAwareFakeStep separateTestStep1 = new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
    final TestResults fakeTestResults = FakeTestResults.of(ImmutableList.of(new TestCaseSummary("TestCase", ImmutableList.of(new TestResultSummary("TestCaseResult", "passTest", ResultType.SUCCESS, 5000, null, null, null, null)))));
    BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
    FakeTestRule separateTest1 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest1Target).build(), new SourcePathResolver(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer())), ImmutableSet.<Label>of(), Optional.of(Paths.get("separateTestStep1OutputDir")), // runTestSeparately
    true, ImmutableList.<Step>of(separateTestStep1), new Callable<TestResults>() {

        @Override
        public TestResults call() {
            return fakeTestResults;
        }
    });
    ExecutionOrderAwareFakeStep separateTestStep2 = new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
    BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
    FakeTestRule separateTest2 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest2Target).build(), new SourcePathResolver(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer())), ImmutableSet.<Label>of(), Optional.of(Paths.get("separateTestStep2OutputDir")), // runTestSeparately
    true, ImmutableList.<Step>of(separateTestStep2), new Callable<TestResults>() {

        @Override
        public TestResults call() {
            return fakeTestResults;
        }
    });
    ExecutionOrderAwareFakeStep separateTestStep3 = new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
    BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
    FakeTestRule separateTest3 = new FakeTestRule(new FakeBuildRuleParamsBuilder(separateTest3Target).build(), new SourcePathResolver(new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer())), ImmutableSet.<Label>of(), Optional.of(Paths.get("separateTestStep3OutputDir")), // runTestSeparately
    true, ImmutableList.<Step>of(separateTestStep3), new Callable<TestResults>() {

        @Override
        public TestResults call() {
            return fakeTestResults;
        }
    });
    // We explicitly use an actual thread pool here; the logic should ensure the
    // separate tests are run in the correct order.
    ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
    FakeBuildEngine fakeBuildEngine = new FakeBuildEngine(ImmutableMap.of(separateTest1Target, BuildResult.success(separateTest1, BUILT_LOCALLY, CacheResult.miss()), separateTest2Target, BuildResult.success(separateTest2, BUILT_LOCALLY, CacheResult.miss()), separateTest3Target, BuildResult.success(separateTest3, BUILT_LOCALLY, CacheResult.miss())), ImmutableMap.of(separateTest1Target, new RuleKey("00"), separateTest2Target, new RuleKey("00"), separateTest3Target, new RuleKey("00")));
    ExecutionContext fakeExecutionContext = TestExecutionContext.newInstance();
    DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
    int ret = TestRunning.runTests(commandRunnerParams, ImmutableList.<TestRule>of(separateTest1, separateTest2, separateTest3), fakeExecutionContext, DEFAULT_OPTIONS, service, fakeBuildEngine, stepRunner);
    assertThat(ret, equalTo(0));
    assertThat(separateTestStep1.getExecutionBeginOrder(), equalTo(Optional.of(0)));
    assertThat(separateTestStep1.getExecutionEndOrder(), equalTo(Optional.of(1)));
    assertThat(separateTestStep2.getExecutionBeginOrder(), equalTo(Optional.of(2)));
    assertThat(separateTestStep2.getExecutionEndOrder(), equalTo(Optional.of(3)));
    assertThat(separateTestStep3.getExecutionBeginOrder(), equalTo(Optional.of(4)));
    assertThat(separateTestStep3.getExecutionEndOrder(), equalTo(Optional.of(5)));
}

private void testStoreCall(int expectStoreCallCount, Optional<Long> maxArtifactSizeBytes, int... artifactBytes) throws InterruptedException, IOException, ExecutionException {
    final AtomicInteger storeCallCount = new AtomicInteger(0);
    FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
    ListeningExecutorService service = new FakeListeningExecutorService() {

        @Override
        public void execute(Runnable command) {
            command.run();
        }
    };
    AbstractNetworkCache cache = new AbstractNetworkCache(NetworkCacheArgs.builder().setCacheName("AbstractNetworkCacheTest").setRepository("some_repository").setFetchClient(EasyMock.createMock(HttpService.class)).setStoreClient(EasyMock.createMock(HttpService.class)).setDoStore(true).setProjectFilesystem(filesystem).setBuckEventBus(EasyMock.createMock(BuckEventBus.class)).setHttpWriteExecutorService(service).setErrorTextTemplate("super error message").setMaxStoreSizeBytes(maxArtifactSizeBytes).build()) {

        @Override
        protected CacheResult fetchImpl(RuleKey ruleKey, LazyPath output, HttpArtifactCacheEvent.Finished.Builder eventBuilder) throws IOException {
            return null;
        }

        @Override
        protected void storeImpl(ArtifactInfo info, Path file, HttpArtifactCacheEvent.Finished.Builder eventBuilder) throws IOException {
            storeCallCount.incrementAndGet();
        }
    };
    for (int bytes : artifactBytes) {
        Path path = filesystem.getPathForRelativePath("topspin_" + this.getClass().getName());
        filesystem.writeBytesToPath(new byte[bytes], path);
        ListenableFuture<Void> future = cache.store(ArtifactInfo.builder().build(), BorrowablePath.notBorrowablePath(path));
        future.get();
    }
    Assert.assertEquals(expectStoreCallCount, storeCallCount.get());
}

public ListeningExecutorService getExecutorService(ExecutorPool p) {
    ListeningExecutorService executorService = getExecutors().get(p);
    Preconditions.checkNotNull(executorService);
    return executorService;
}

private ImmutableSetMultimap<Path, Prebuilt> downloadArtifacts(final MutableDirectedGraph<Artifact> graph) throws ExecutionException, InterruptedException {
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new MostExecutors.NamedThreadFactory("artifact download")));
    @SuppressWarnings("unchecked") List<ListenableFuture<Map.Entry<Path, Prebuilt>>> results = (List<ListenableFuture<Map.Entry<Path, Prebuilt>>>) (List<?>) exec.invokeAll(FluentIterable.from(graph.getNodes()).transform(new Function<Artifact, Callable<Map.Entry<Path, Prebuilt>>>() {

        @Override
        public Callable<Map.Entry<Path, Prebuilt>> apply(final Artifact artifact) {
            return new Callable<Map.Entry<Path, Prebuilt>>() {

                @Override
                public Map.Entry<Path, Prebuilt> call() throws Exception {
                    return downloadArtifact(artifact, graph);
                }
            };
        }
    }).toList());
    try {
        return ImmutableSetMultimap.<Path, Prebuilt>builder().orderValuesBy(Ordering.natural()).putAll(Futures.allAsList(results).get()).build();
    } finally {
        exec.shutdown();
    }
}

/**
   * Execute an {@link AdbCallable} for all matching devices. This functions performs device
   * filtering based on three possible arguments:
   *
   *  -e (emulator-only) - only emulators are passing the filter
   *  -d (device-only) - only real devices are passing the filter
   *  -s (serial) - only device/emulator with specific serial number are passing the filter
   *
   *  If more than one device matches the filter this function will fail unless multi-install
   *  mode is enabled (-x). This flag is used as a marker that user understands that multiple
   *  devices will be used to install the apk if needed.
   */
@SuppressWarnings("PMD.EmptyCatchBlock")
@SuppressForbidden
public boolean adbCall(AdbCallable adbCallable, boolean quiet) throws InterruptedException {
    List<IDevice> devices;
    try (TraceEventLogger ignored = TraceEventLogger.start(buckEventBus, "set_up_adb_call")) {
        devices = getDevices(quiet);
        if (devices.size() == 0) {
            return false;
        }
    }
    int adbThreadCount = options.getAdbThreadCount();
    if (adbThreadCount <= 0) {
        adbThreadCount = devices.size();
    }
    // Start executions on all matching devices.
    List<ListenableFuture<Boolean>> futures = Lists.newArrayList();
    ListeningExecutorService executorService = listeningDecorator(newMultiThreadExecutor(new CommandThreadFactory(getClass().getSimpleName()), adbThreadCount));
    for (final IDevice device : devices) {
        futures.add(executorService.submit(adbCallable.forDevice(device)));
    }
    // Wait for all executions to complete or fail.
    List<Boolean> results = null;
    try {
        results = Futures.allAsList(futures).get();
    } catch (ExecutionException ex) {
        console.printBuildFailure("Failed: " + adbCallable);
        ex.printStackTrace(console.getStdErr());
        return false;
    } catch (InterruptedException e) {
        try {
            Futures.allAsList(futures).cancel(true);
        } catch (CancellationException ignored) {
        }
        Thread.currentThread().interrupt();
        throw e;
    } finally {
        MostExecutors.shutdownOrThrow(executorService, 10, TimeUnit.MINUTES, new InterruptionFailedException("Failed to shutdown ExecutorService."));
    }
    int successCount = 0;
    for (Boolean result : results) {
        if (result) {
            successCount++;
        }
    }
    int failureCount = results.size() - successCount;
    // Report results.
    if (successCount > 0 && !quiet) {
        console.printSuccess(String.format("Successfully ran %s on %d device(s)", adbCallable, successCount));
    }
    if (failureCount > 0) {
        console.printBuildFailure(String.format("Failed to %s on %d device(s).", adbCallable, failureCount));
    }
    return failureCount == 0;
}

@Test
public void sValue() throws Exception {
    // Check that we never generate an S value that is larger than half the curve order. This avoids a malleability
    // issue that can allow someone to change a transaction [hash] without invalidating the signature.
    final int ITERATIONS = 10;
    ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(ITERATIONS));
    List<ListenableFuture<ECKey.ECDSASignature>> sigFutures = Lists.newArrayList();
    final ECKey key = new ECKey();
    for (byte i = 0; i < ITERATIONS; i++) {
        final Sha256Hash hash = Sha256Hash.of(new byte[] { i });
        sigFutures.add(executor.submit(new Callable<ECKey.ECDSASignature>() {

            @Override
            public ECKey.ECDSASignature call() throws Exception {
                return key.sign(hash);
            }
        }));
    }
    List<ECKey.ECDSASignature> sigs = Futures.allAsList(sigFutures).get();
    for (ECKey.ECDSASignature signature : sigs) {
        assertTrue(signature.isCanonical());
    }
    final ECDSASignature first = sigs.get(0);
    final ECKey.ECDSASignature duplicate = new ECKey.ECDSASignature(first.r, first.s);
    assertEquals(first, duplicate);
    assertEquals(first.hashCode(), duplicate.hashCode());
    final ECKey.ECDSASignature highS = new ECKey.ECDSASignature(first.r, ECKey.CURVE.getN().subtract(first.s));
    assertFalse(highS.isCanonical());
}

@Nonnull
@VisibleForTesting
public Map<ArtifactLocation, String> parsePackageStrings(@Nonnull List<ArtifactLocation> sources) throws Exception {
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()));
    Map<ArtifactLocation, ListenableFuture<String>> futures = Maps.newHashMap();
    for (final ArtifactLocation source : sources) {
        futures.put(source, executorService.submit(new Callable<String>() {

            @Override
            public String call() throws Exception {
                return getDeclaredPackageOfJavaFile(source);
            }
        }));
    }
    Map<ArtifactLocation, String> map = Maps.newHashMap();
    for (Entry<ArtifactLocation, ListenableFuture<String>> entry : futures.entrySet()) {
        String value = entry.getValue().get();
        if (value != null) {
            map.put(entry.getKey(), value);
        }
    }
    return map;
}

@Nullable
public SymbolLoader loadResourceSymbolTable(List<SymbolFileProvider> libraries, String appPackageName, Path primaryRTxt, Multimap<String, SymbolLoader> libMap) throws IOException {
    // The reported availableProcessors may be higher than the actual resources
    // (on a shared system). On the other hand, a lot of the work is I/O, so it's not completely
    // CPU bound. As a compromise, divide by 2 the reported availableProcessors.
    int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2);
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads));
    try (Closeable closeable = ExecutorServiceCloser.createWith(executorService)) {
        // Load the package names from the manifest files.
        Map<SymbolFileProvider, ListenableFuture<String>> packageJobs = new HashMap<>();
        for (final SymbolFileProvider lib : libraries) {
            packageJobs.put(lib, executorService.submit(new PackageParsingTask(lib.getManifest())));
        }
        Map<SymbolFileProvider, String> packageNames = new HashMap<>();
        try {
            for (Map.Entry<SymbolFileProvider, ListenableFuture<String>> entry : packageJobs.entrySet()) {
                packageNames.put(entry.getKey(), entry.getValue().get());
            }
        } catch (InterruptedExceptionExecutionException |  e) {
            throw new IOException("Failed to load package name: ", e);
        }
        // Associate the packages with symbol files.
        for (SymbolFileProvider lib : libraries) {
            String packageName = packageNames.get(lib);
            // stored in the primaryRTxt file.
            if (appPackageName.equals(packageName)) {
                continue;
            }
            File rFile = lib.getSymbolFile();
            // If the library has no resource, this file won't exist.
            if (rFile.isFile()) {
                SymbolLoader libSymbols = new SymbolLoader(rFile, stdLogger);
                libMap.put(packageName, libSymbols);
            }
        }
        // Even if there are no libraries, load fullSymbolValues, in case we only have resources
        // defined for the binary.
        File primaryRTxtFile = primaryRTxt.toFile();
        SymbolLoader fullSymbolValues = null;
        if (primaryRTxtFile.isFile()) {
            fullSymbolValues = new SymbolLoader(primaryRTxtFile, stdLogger);
        }
        // Now load the symbol files in parallel.
        List<ListenableFuture<?>> loadJobs = new ArrayList<>();
        Iterable<SymbolLoader> toLoad = fullSymbolValues != null ? Iterables.concat(libMap.values(), ImmutableList.of(fullSymbolValues)) : libMap.values();
        for (final SymbolLoader loader : toLoad) {
            loadJobs.add(executorService.submit(new SymbolLoadingTask(loader)));
        }
        try {
            Futures.allAsList(loadJobs).get();
        } catch (InterruptedExceptionExecutionException |  e) {
            throw new IOException("Failed to load SymbolFile: ", e);
        }
        return fullSymbolValues;
    }
}

@Override
public List<InputSplit> getSplits(final JobContext job) throws IOException {
    // TODO switch to Java 8 and replace this with streams
    List<FileStatus> allFiles = listStatus(job);
    List<FileStatus> dataFiles = Lists.newLinkedList();
    final Queue<InputSplit> allSplits = new ConcurrentLinkedQueue<>();
    for (FileStatus file : allFiles) {
        String name = file.getPath().getName();
        if (name.endsWith("-Data.db")) {
            dataFiles.add(file);
        }
    }
    LOG.info("Calculating splits for {} input files of which {} are data files", allFiles.size(), dataFiles.size());
    // TODO make the number of threads configurable
    ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(20));
    for (final FileStatus file : dataFiles) {
        ListenableFuture<List<InputSplit>> future = service.submit(new Callable<List<InputSplit>>() {

            public List<InputSplit> call() throws IOException {
                List<InputSplit> splitsForFile = getSSTableSplitsForFile(job, file);
                LOG.info("Split '{}' into {} splits", file.getPath(), splitsForFile.size());
                return splitsForFile;
            }
        });
        Futures.addCallback(future, new FutureCallback<List<InputSplit>>() {

            public void onFailure(Throwable thrown) {
                throw Throwables.propagate(thrown);
            }

            public void onSuccess(List<InputSplit> splits) {
                allSplits.addAll(splits);
            }
        });
    }
    try {
        service.shutdown();
        // TODO timeout configurable
        service.awaitTermination(2, TimeUnit.HOURS);
    } catch (InterruptedException e) {
        throw Throwables.propagate(e);
    }
    ImmutableList<InputSplit> inputSplits = ImmutableList.copyOf(allSplits);
    LOG.info("Split {} input data files into {} parts", dataFiles.size(), inputSplits.size());
    return inputSplits;
}