org.apache.pulsar.shade.org.apache.bookkeeper.mledger.ManagedLedgerConfig

Here are the examples of the java api org.apache.pulsar.shade.org.apache.bookkeeper.mledger.ManagedLedgerConfig taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

1 Examples 7

9 Source : InputSplitReader.java
with Apache License 2.0
from streamnative

/**
 * Segment reader for a input split.
 * @param <T> the type of emit data.
 */
@Slf4j
public abstract clreplaced InputSplitReader<T> {

    private ConnectorConfig connectorConfig;

    private int parreplacedionId;

    private List<InputLedger> ledgersToRead;

    private CachedClients cachedClients;

    private BookKeeper bkClient;

    private ManagedLedgerConfig mlConfig;

    private LedgerOffloader offloader;

    private Executor executor;

    private SpscArrayQueue<RawMessage> messageQueue;

    private SpscArrayQueue<Entry> entryQueue;

    private Thread deserializerThread;

    private RawMessage currentMessage;

    private AtomicLong outstandingLedgerReads = new AtomicLong(0);

    private int ledgerSize;

    private int currentLedgerIdx = 0;

    private long parreplacedionSize;

    private long entriesProcessed = 0;

    private Map<Long, TopicName> ledger2Topic = new ConcurrentHashMap<>();

    public InputSplitReader(ConnectorConfig connectorConfig, int parreplacedionId, List<InputLedger> ledgersToRead) throws Exception {
        this.connectorConfig = connectorConfig;
        this.parreplacedionId = parreplacedionId;
        this.ledgersToRead = ledgersToRead;
        this.cachedClients = CachedClients.getInstance(connectorConfig);
        this.bkClient = cachedClients.getManagedLedgerFactory().getBookKeeper();
        this.mlConfig = cachedClients.getManagedLedgerConfig();
        this.offloader = mlConfig.getLedgerOffloader();
        this.executor = Executors.newSingleThreadExecutor();
        this.messageQueue = new SpscArrayQueue<>(connectorConfig.getMaxSplitMessageQueueSize());
        this.entryQueue = new SpscArrayQueue<>(connectorConfig.getMaxSplitEntryQueueSize());
        this.ledgerSize = ledgersToRead.size();
        this.parreplacedionSize = ledgersToRead.stream().mapToLong(InputLedger::ledgerSize).sum();
    }

    public boolean next() throws IOException {
        if (deserializerThread == null) {
            deserializerThread = new DeserializeEntries();
            deserializerThread.start();
            getEntries(ledgersToRead.get(currentLedgerIdx));
            currentLedgerIdx++;
        }
        if (currentMessage != null) {
            currentMessage.release();
            currentMessage = null;
        }
        while (true) {
            if (messageQueue.isEmpty() && entriesProcessed >= parreplacedionSize) {
                return false;
            }
            if (currentLedgerIdx < ledgerSize && outstandingLedgerReads.get() == 0) {
                getEntries(ledgersToRead.get(currentLedgerIdx));
                currentLedgerIdx++;
            }
            currentMessage = messageQueue.poll();
            if (currentMessage != null) {
                return true;
            } else {
                try {
                    Thread.sleep(1);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
        }
    }

    public abstract T deserialize(RawMessage currentMessage) throws IOException;

    public T get() throws IOException {
        return deserialize(currentMessage);
    }

    public void close() throws Exception {
        if (currentMessage != null) {
            currentMessage.release();
        }
        if (messageQueue != null) {
            messageQueue.drain(RawMessage::release);
        }
        if (entryQueue != null) {
            entryQueue.drain(Entry::release);
        }
        if (deserializerThread != null) {
            deserializerThread.interrupt();
        }
    }

    CompletableFuture<Object> getEntries(InputLedger info) {
        outstandingLedgerReads.incrementAndGet();
        return getLedgerHandle(info).thenComposeAsync(readHandle -> {
            try (LedgerEntries entries = readHandle.read(info.getStartEntryId(), info.getEndEntryId())) {
                entryQueue.fill(new MessagePreplacedingQueue.Supplier<Entry>() {

                    private int i = 0;

                    @Override
                    public Entry get() {
                        EntryImpl impl = EntryImpl.create(entries.getEntry(i));
                        i++;
                        return impl;
                    }
                }, Lists.newArrayList(entries.iterator()).size());
            } catch (Exception e) {
                throw new CompletionException(e);
            }
            return null;
        }, executor).whenComplete((t, throwable) -> {
            if (throwable != null) {
                log.error(String.format("Get entry failed due to %s", throwable.getMessage()), throwable);
            } else {
                log.info(String.format("Finished extracting entries for ledger %s", info.toString()));
                outstandingLedgerReads.decrementAndGet();
            }
        });
    }

    CompletableFuture<ReadHandle> getLedgerHandle(InputLedger ledger) {
        ledger2Topic.put(ledger.getLedgerId(), TopicName.get(ledger.getTopic()));
        if (ledger.getUuid() != null) {
            return offloader.readOffloaded(ledger.getLedgerId(), ledger.getUuid(), ledger.getOffloaderDrvierMeta());
        } else {
            return bkClient.newOpenLedgerOp().withRecovery(false).withLedgerId(ledger.getLedgerId()).withDigestType(mlConfig.getDigestType()).withPreplacedword(mlConfig.getPreplacedword()).execute();
        }
    }

    clreplaced DeserializeEntries extends Thread {

        protected boolean isRunning = false;

        DeserializeEntries() {
            super("derserialize-thread-split-" + parreplacedionId);
        }

        @Override
        public void interrupt() {
            isRunning = false;
        }

        @Override
        public void run() {
            isRunning = true;
            while (isRunning) {
                int read = entryQueue.drain(entry -> {
                    TopicName tp = ledger2Topic.getOrDefault(entry.getLedgerId(), TopicName.get("DUMMY"));
                    try {
                        MessageParser.parseMessage(tp, entry.getLedgerId(), entry.getEntryId(), entry.getDataBuffer(), (message) -> {
                            try {
                                // enqueue deserialize message from this entry
                                while (!messageQueue.offer(message)) {
                                    Thread.sleep(1);
                                }
                            } catch (InterruptedException e) {
                            // no-op
                            }
                        }, connectorConfig.getMaxMessageSize());
                    } catch (IOException e) {
                        log.error(String.format("Failed to parse message from pulsar topic %s", tp.toString()), e);
                    } finally {
                        entriesProcessed++;
                        entry.release();
                    }
                });
                if (read <= 0) {
                    try {
                        Thread.sleep(1);
                    } catch (InterruptedException e) {
                        return;
                    }
                }
            }
        }
    }
}