org.slf4j.Logger

Here are the examples of the java api org.slf4j.Logger taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

88575 Examples 7

19 Source : PropsUtil.java
with MIT License
from zzzzbw

/**
 * 属性文件助手
 *
 * @author zbw
 * @create 2017/11/20 11:23
 */
public final clreplaced PropsUtil {

    private static final Logger logger = LoggerFactory.getLogger(PropsUtil.clreplaced);

    synchronized static public Properties loadProps(String propsName) {
        String jarUrl = JarUtil.getJarDir();
        String fileUrl = jarUrl + File.separator + propsName;
        File propFile = new File(fileUrl);
        if (!propFile.exists()) {
            try {
                propFile.createNewFile();
            } catch (IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
        Properties props = new Properties();
        InputStream in = null;
        try {
            in = new FileInputStream(propFile);
            props.load(in);
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
        } finally {
            try {
                if (null != in) {
                    in.close();
                }
            } catch (IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
        return props;
    }

    public static void saveString(Properties props, String propsName, String key, String value) {
        props.setProperty(key, value);
        try {
            String jarUrl = JarUtil.getJarDir();
            String fileUrl = jarUrl + File.separator + propsName;
            FileOutputStream fos = new FileOutputStream(new File(fileUrl));
            props.store(fos, "This is the properties for gitPic");
            fos.close();
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
        }
    }

    /**
     * 获取字符型属性
     */
    public static String getString(Properties props, String key) {
        String value = "";
        if (props.containsKey(key)) {
            value = props.getProperty(key);
        }
        return value;
    }

    /**
     * 获取字符型属性(带有默认值)
     */
    public static String getString(Properties props, String key, String defalutValue) {
        String value = defalutValue;
        if (props.containsKey(key)) {
            value = props.getProperty(key);
        }
        return value;
    }

    /**
     * 获取数值型属性
     */
    public static int getNumber(Properties props, String key) {
        int value = 0;
        if (props.containsKey(key)) {
            value = CastUtil.castInt(props.getProperty(key));
        }
        return value;
    }

    /**
     * 获取数值型属性(带有默认值)
     *
     * @param props
     * @param key
     * @param defaultValue
     * @return
     */
    public static int getNumber(Properties props, String key, int defaultValue) {
        int value = defaultValue;
        if (props.containsKey(key)) {
            value = CastUtil.castInt(props.getProperty(key));
        }
        return value;
    }

    /**
     * 获取布尔型属性
     */
    public static boolean getBoolean(Properties props, String key) {
        return getBoolean(props, key, false);
    }

    /**
     * 获取布尔型属性(带有默认值)
     */
    public static boolean getBoolean(Properties props, String key, boolean defalutValue) {
        boolean value = defalutValue;
        if (props.containsKey(key)) {
            value = CastUtil.castBoolean(props.getProperty(key));
        }
        return value;
    }
}

19 Source : GitUtils.java
with MIT License
from zzzzbw

/**
 * @author zbw
 * @create 2018/2/27 20:15
 */
public clreplaced GitUtils {

    private static final Logger logger = LoggerFactory.getLogger(GitUtils.clreplaced);

    /**
     * 加载git项目
     *
     * @param gitPath
     * @return
     */
    public static Repository init(String gitPath) {
        try {
            return new FileRepositoryBuilder().setGitDir(findGitRepositoryPath(gitPath)).build();
        } catch (IOException e) {
            e.printStackTrace();
            logger.error(e.getMessage());
            throw new TipException("初始化git异常");
        }
    }

    /**
     * 获取当前分支
     *
     * @param rep
     * @return
     */
    public static String getBranch(Repository rep) {
        try {
            return rep.getBranch();
        } catch (IOException e) {
            e.printStackTrace();
            logger.error(e.getMessage());
            throw new TipException("获取Branch异常");
        }
    }

    /**
     * 获取remote URI
     *
     * @param rep
     * @return
     */
    public static String getRemoteUri(Repository rep) {
        Git git = new Git(rep);
        List<RemoteConfig> remoteConfigList;
        try {
            remoteConfigList = git.remoteList().call();
        } catch (GitAPIException e) {
            e.printStackTrace();
            logger.error(e.getMessage());
            throw new TipException("获取RemoteUri异常");
        }
        if (null != remoteConfigList && remoteConfigList.size() > 0) {
            if (remoteConfigList.get(0).getURIs().size() <= 0) {
                throw new TipException("该分支不存在远程仓库");
            }
            return remoteConfigList.get(0).getURIs().get(0).toString();
        }
        return "";
    }

    /**
     * 获取验证方式
     *
     * @param uri
     * @return SSH/HTTPS
     */
    public static String authType(String uri) {
        if (uri.contains(Constants.GIT_SSH)) {
            return Constants.GIT_SSH;
        }
        if (uri.contains(Constants.GIT_HTTPS)) {
            return Constants.GIT_HTTPS;
        }
        throw new TipException("不合法的uri");
    }

    /**
     * git commit
     *
     * @param repository
     */
    public static void commitAll(Repository repository) {
        commitAll(repository, Constants.GIT_DEFAULT_COMMIT_MESSAGE);
    }

    /**
     * git commit
     *
     * @param repository
     * @param commitMsg
     */
    public static void commitAll(Repository repository, String commitMsg) {
        Git git = new Git(repository);
        try {
            git.add().addFilepattern(".").call();
            git.commit().setMessage(commitMsg).call();
        } catch (GitAPIException e) {
            logger.error(e.getMessage(), e);
            throw new TipException("git commit 异常");
        }
    }

    public static void pull(Repository repository) {
        Git git = new Git(repository);
        try {
            PullResult result = git.pull().call();
            FetchResult fetchResult = result.getFetchResult();
            MergeResult mergeResult = result.getMergeResult();
            if (fetchResult.getMessages() != null && !fetchResult.getMessages().isEmpty()) {
                logger.info(fetchResult.getMessages());
            }
            logger.info(mergeResult.getMergeStatus().toString());
            if (!mergeResult.getMergeStatus().isSuccessful()) {
                throw new TipException(mergeResult.getMergeStatus().toString());
            }
        } catch (GitAPIException e) {
            logger.error(e.getMessage(), e);
            throw new TipException("git commit 异常");
        }
    }

    /**
     * git push
     *
     * @param repository
     */
    public static void push(Repository repository) {
        Git git = new Git(repository);
        try {
            Iterable<PushResult> results = git.push().call();
            PushResult result = results.iterator().next();
            validPushResult(result);
        } catch (GitAPIException e) {
            logger.error(e.getMessage(), e);
            throw new TipException("git push 异常, message:" + e.getMessage());
        }
    }

    /**
     * git push
     *
     * @param repository
     * @param username
     * @param preplacedword
     */
    public static void push(Repository repository, String username, String preplacedword) {
        Git git = new Git(repository);
        try {
            CredentialsProvider cp = new UsernamePreplacedwordCredentialsProvider(username, preplacedword);
            Iterable<PushResult> results = git.push().setCredentialsProvider(cp).call();
            PushResult result = results.iterator().next();
            validPushResult(result);
        } catch (TransportException e) {
            logger.error(e.getMessage(), e);
            throw new AuthorizedException("验证失败");
        } catch (GitAPIException e) {
            logger.error(e.getMessage(), e);
            throw new TipException("git push 异常, message:" + e.getMessage());
        }
    }

    /**
     * 验证push结果
     *
     * @param result
     */
    public static void validPushResult(PushResult result) {
        String msg = "未知原因";
        if (null == result) {
            throw new TipException(("push失败: " + msg));
        }
        RemoteRefUpdate.Status status = result.getRemoteUpdate(Constants.GIT_MASTER_HEAD).getStatus();
        switch(status) {
            case OK:
                return;
            case NOT_ATTEMPTED:
                msg = "Push process hasn't yet attempted to update this ref. This is the default status, prior to push process execution.";
                break;
            case UP_TO_DATE:
                msg = "Remote ref was up to date, there was no need to update anything.";
                break;
            case REJECTED_NONFASTFORWARD:
                msg = "Remote ref update was rejected, as it would cause non fast-forward  update.";
                break;
            case REJECTED_NODELETE:
                msg = "Remote ref update was rejected, because remote side doesn't support/allow deleting refs.";
                break;
            case REJECTED_REMOTE_CHANGED:
                msg = "Remote ref update was rejected, because old object id on remote repository wasn't the same as defined expected old object.";
                break;
            case REJECTED_OTHER_REASON:
                msg = "Remote ref update was rejected for other reason";
                break;
            case NON_EXISTING:
                msg = "Remote ref didn't exist. Can occur on delete request of a non existing ref.";
                break;
            case AWAITING_REPORT:
                msg = "Push process is awaiting update report from remote repository. This is a temporary state or state after critical error in push process.";
                break;
            default:
                msg = "未知原因";
                break;
        }
        throw new TipException("push失败: " + msg);
    }

    /**
     * 获取git文件夹
     *
     * @param projectPath
     * @return
     */
    public static File findGitRepositoryPath(String projectPath) {
        File file = new File(projectPath);
        if (!file.isDirectory()) {
            throw new TipException("git项目必须为文件夹");
        }
        int len = projectPath.length();
        if (Constants.GIT_PATH.equals(projectPath.substring(len - 4, len))) {
            return file;
        } else {
            // 不是.git文件夹则为git项目的文件夹
            projectPath += File.separator + Constants.GIT_PATH;
            file = new File(projectPath);
            if (file.exists() && file.isDirectory()) {
                return file;
            }
        }
        throw new TipException("该目录不存在git项目");
    }

    /**
     * 创建github文件链接
     *
     * @param uri
     * @param branchName
     * @param folder
     * @param fileName
     * @return
     */
    public static String createGitBlobUrl(String uri, String branchName, String folder, String fileName) {
        uri = uri.replace(Constants.GIT_PATH, "");
        if (Constants.GIT_SSH.equals(authType(uri))) {
            uri = uri.replace("[email protected]:", "https://github.com/");
        }
        return (uri + "/blob/" + branchName + folder + "/" + fileName).replace("\\", "/");
    }

    /**
     * 创建raw cdn链接
     *
     * @param blobUrl
     * @return
     */
    public static String createGitCdnUrl(String blobUrl) {
        return blobUrl.replace("github", "raw.githubusercontent").replace("blob/", "");
    }
}

19 Source : MainController.java
with MIT License
from zzzzbw

/**
 * @author zbw
 * @create 2018/2/26 16:36
 */
public clreplaced MainController extends StackPane implements Initializable {

    private static Logger logger = LoggerFactory.getLogger(MainController.clreplaced);

    private Stage stage;

    private String uploadImgFilePath;

    private boolean isGitInit = false;

    private Repository repository = null;

    @FXML
    private StackPane root;

    @FXML
    private JFXTextField projectPathTextField;

    @FXML
    private JFXTextField imgPathTextField;

    @FXML
    private JFXTextField rawUrlTextField;

    @FXML
    private JFXButton commitButton;

    @FXML
    private JFXSpinner promptSpinner;

    @FXML
    private Label promptLabel;

    @FXML
    private JFXSpinner gitSpinner;

    @FXML
    private Label gitLabel;

    @FXML
    private JFXDialog dialog;

    @FXML
    private JFXTextField usernameTextField;

    @FXML
    private JFXPreplacedwordField preplacedwordTextField;

    @Override
    public void initialize(URL location, ResourceBundle resources) {
        // 先从根组件移除弹窗,否则无法手动初始化弹窗
        root.getChildren().remove(dialog);
        stage = new Stage();
        promptSpinner.setVisible(false);
        gitSpinner.setVisible(false);
        dialog.setTransitionType(JFXDialog.DialogTransition.CENTER);
        String projectPath = Preference.getInstance().getProjectPath();
        if (CastUtil.isNotEmpty(projectPath)) {
            projectPathTextField.setText(projectPath);
            initGit(projectPath);
        }
        imgPathTextField.setText(Preference.getInstance().getPicPath());
    }

    /**
     * 主面板onDragOver事件
     *
     * @param event
     */
    @FXML
    protected void setTransferMode(DragEvent event) {
        event.acceptTransferModes(TransferMode.ANY);
    }

    /**
     * 主面板onDragDropped事件
     *
     * @param event
     */
    @FXML
    protected void setUploadImgPath(DragEvent event) {
        Dragboard dragboard = event.getDragboard();
        if (dragboard.hasFiles()) {
            File file = dragboard.getFiles().get(0);
            if (file != null) {
                uploadImgFilePath = file.getAbsolutePath();
            }
        }
        copyAndGenerate();
    }

    /**
     * 选择项目根目录
     */
    @FXML
    protected void chooseProjectPath() {
        DirectoryChooser directoryChooser = new DirectoryChooser();
        directoryChooser.setreplacedle("选择项目目录");
        File file = directoryChooser.showDialog(stage);
        if (file == null || !file.isDirectory()) {
            return;
        }
        projectPathTextField.setText(file.getAbsolutePath());
        if (CastUtil.isEmpty(projectPathTextField.getText())) {
            return;
        }
        String projectPath = projectPathTextField.getText();
        Preference.getInstance().saveProjectPath(projectPath);
        initGit(projectPath);
    }

    /**
     * 选择图片保存目录
     */
    @FXML
    protected void chooseImgPath() {
        DirectoryChooser directoryChooser = new DirectoryChooser();
        String projectPath = projectPathTextField.getText();
        if (CastUtil.isNotEmpty(projectPath)) {
            File file = new File(projectPath);
            directoryChooser.setInitialDirectory(file);
        }
        directoryChooser.setreplacedle("选择图片保存目录");
        File file = directoryChooser.showDialog(stage);
        if (file == null || !file.isDirectory()) {
            return;
        }
        imgPathTextField.setText(file.getAbsolutePath());
        Preference.getInstance().savePicPath(imgPathTextField.getText());
    }

    /**
     * 选择要上传的图片
     */
    @FXML
    protected void chooseUploadImg() {
        FileChooser fileChooser = new FileChooser();
        File file = fileChooser.showOpenDialog(stage);
        if (null == file) {
            return;
        }
        uploadImgFilePath = file.getAbsolutePath();
        copyAndGenerate();
    }

    /**
     * 从剪贴板复制
     */
    @FXML
    protected void copyByClipboard() {
        Clipboard clipboard = Clipboard.getSystemClipboard();
        if (clipboard.hasImage()) {
            Image image = clipboard.getImage();
            BufferedImage bImage = SwingFXUtils.fromFXImage(image, null);
            try {
                Path tempDirectory = Files.createTempDirectory(Constants.GITPIC_PREFIX);
                String tempFile = tempDirectory.toString() + File.separator + Constants.GITPIC_PREFIX + System.currentTimeMillis() + ".png";
                File file = new File(tempFile);
                ImageIO.write(bImage, "png", file);
                uploadImgFilePath = file.getAbsolutePath();
                copyAndGenerate();
                // 删除临时图片
                file.delete();
                // 删除临时目录
                Files.delete(tempDirectory);
            } catch (IOException e) {
                logger.error(e.getMessage(), e);
                showErrorMessage("从剪切板拷贝图片异常", e);
            }
        } else {
            showErrorMessage("剪切板中没有图片");
        }
    }

    /**
     * commit push 按钮
     */
    @FXML
    protected void commitAndPush() {
        commitButton.setDisable(true);
        ThreadPool.getInstance().execute(() -> {
            try {
                showNormalMessage("commit中...");
                GitUtils.commitAll(repository);
                showSuccessMessage("commit成功!");
                pushGit();
            } catch (TipException e) {
                showErrorMessage(e);
                commitButton.setDisable(false);
            }
        });
    }

    /**
     * Dialog面板Accept
     */
    @FXML
    protected void dialogAccept() {
        if (!usernameTextField.validate() || !preplacedwordTextField.validate()) {
            return;
        }
        String username = this.usernameTextField.getText();
        String preplacedword = this.preplacedwordTextField.getText();
        Preference.getInstance().saveGitUsername(username);
        Preference.getInstance().saveGitPreplacedword(preplacedword);
        pushGit();
        dialog.close();
    }

    /**
     * Dialog面板Cancel
     */
    @FXML
    protected void dialogCancel() {
        dialog.close();
        commitButton.setDisable(false);
        showErrorMessage("push到github失败");
    }

    /**
     * 加载Git项目
     *
     * @param projectPath
     */
    private void initGit(String projectPath) {
        ThreadPool.getInstance().execute(() -> {
            try {
                Platform.runLater(() -> {
                    gitSpinner.setVisible(true);
                    gitLabel.setText("初始化git项目中...");
                });
                repository = GitUtils.init(projectPath);
                isGitInit = true;
                Platform.runLater(() -> {
                    gitSpinner.setVisible(false);
                    gitLabel.setText("项目已加载");
                    gitLabel.getStyleClreplaced().add("text-success");
                });
                showPromptSpinner();
                showNormalMessage("从Github中pull项目...");
                GitUtils.pull(repository);
                hidePromptSpinner();
                showNormalMessage("Pull项目成功");
                commitButton.setDisable(false);
            } catch (TipException e) {
                showErrorMessage(e);
                Platform.runLater(() -> {
                    gitSpinner.setVisible(false);
                    gitLabel.setText("");
                });
            }
        });
    }

    /**
     * 复制文件到git项目下并生成raw链接
     */
    private void copyAndGenerate() {
        if (!isGitInit) {
            showErrorMessage("请先初始化git");
            return;
        }
        try {
            copyToProject();
        } catch (TipException e) {
            showErrorMessage(e);
            return;
        }
        try {
            generateGitRawPath();
        } catch (TipException e) {
            showErrorMessage(e);
        }
    }

    /**
     * 复制图片到git项目下
     */
    private void copyToProject() {
        if (CastUtil.isEmpty(imgPathTextField.getText())) {
            throw new TipException("请先设置保存图片文件夹");
        }
        File file = new File(imgPathTextField.getText());
        if (!file.isDirectory() || !file.exists()) {
            throw new TipException("保存图片文件夹路径不存在");
        }
        File pic = new File(uploadImgFilePath);
        if (!pic.exists() || !pic.isFile()) {
            throw new TipException("保存图片文件夹路径不存在");
        }
        String gitImgPath = imgPathTextField.getText() + File.separator + pic.getName();
        File gitPic = new File(gitImgPath);
        try {
            Files.copy(pic.toPath(), gitPic.toPath());
        } catch (FileAlreadyExistsException e) {
            throw new TipException("项目中有相同文件名文件");
        } catch (IOException e) {
            e.printStackTrace();
            throw new TipException("复制文件失败");
        }
    }

    /**
     * 生成raw链接
     */
    private void generateGitRawPath() {
        File pic = new File(uploadImgFilePath);
        if (!pic.exists() || pic.isDirectory()) {
            throw new TipException("请选择正确的图片文件");
        }
        if (null == repository) {
            throw new TipException("请先初始化git项目");
        }
        String uri = GitUtils.getRemoteUri(repository);
        String branch = GitUtils.getBranch(repository);
        String folder = imgPathTextField.getText().replace(projectPathTextField.getText(), "");
        String name = uploadImgFilePath.substring(uploadImgFilePath.lastIndexOf(File.separator) + 1);
        String url = GitUtils.createGitBlobUrl(uri, branch, folder, name);
        rawUrlTextField.setText(GitUtils.createGitCdnUrl(url));
        rawUrlTextField.requestFocus();
        rawUrlTextField.selectAll();
        rawUrlTextField.copy();
        showSuccessMessage("已复制图片路径到剪切板");
    }

    /**
     * push到git
     */
    private void pushGit() {
        ThreadPool.getInstance().execute(() -> {
            showPromptSpinner();
            showNormalMessage("push到github中...");
            try {
                String uri = GitUtils.getRemoteUri(repository);
                if (Constants.GIT_SSH.equals(GitUtils.authType(uri))) {
                    GitUtils.push(repository);
                    showSuccessMessage("push成功!");
                    hidePromptSpinner();
                    commitButton.setDisable(false);
                } else {
                    String username = Preference.getInstance().getGitUsername();
                    String preplacedword = Preference.getInstance().getGitPreplacedword();
                    if (CastUtil.isNotEmpty(username) && CastUtil.isNotEmpty(preplacedword)) {
                        try {
                            GitUtils.push(repository, username, preplacedword);
                            showSuccessMessage("push成功!");
                            hidePromptSpinner();
                        } catch (AuthorizedException e) {
                            showErrorMessage(e);
                            hidePromptSpinner();
                            Platform.runLater(() -> dialog.show(root));
                        }
                        commitButton.setDisable(false);
                    } else {
                        Platform.runLater(() -> dialog.show(root));
                    }
                }
            } catch (TipException e) {
                commitButton.setDisable(false);
                hidePromptSpinner();
                showErrorMessage(e);
            }
        });
    }

    /**
     * 显示成功信息
     *
     * @param msg
     */
    private void showSuccessMessage(String msg) {
        this.setGitMessageLabel(msg, "text-success");
    }

    /**
     * 显示错误信息
     *
     * @param msg
     */
    private void showErrorMessage(String msg) {
        this.setGitMessageLabel(msg, "text-error");
    }

    /**
     * 显示异常的信息
     *
     * @param e
     */
    private void showErrorMessage(Exception e) {
        this.setGitMessageLabel(e.getMessage(), "text-dark");
    }

    /**
     * 显示带异常信息的错误信息
     *
     * @param msg
     * @param e
     */
    private void showErrorMessage(String msg, Exception e) {
        this.setGitMessageLabel(msg + e.getMessage(), "text-error");
    }

    /**
     * 显示提示信息
     *
     * @param msg
     */
    private void showNormalMessage(String msg) {
        this.setGitMessageLabel(msg, "text-dark");
    }

    /**
     * 显示信息
     *
     * @param msg
     * @param styleClreplaced
     */
    private void setGitMessageLabel(String msg, String... styleClreplaced) {
        Platform.runLater(() -> {
            promptLabel.getStyleClreplaced().clear();
            promptLabel.getStyleClreplaced().addAll(styleClreplaced);
            promptLabel.setText(msg);
        });
    }

    private void showPromptSpinner() {
        Platform.runLater(() -> promptSpinner.setVisible(true));
    }

    private void hidePromptSpinner() {
        Platform.runLater(() -> promptSpinner.setVisible(false));
    }
}

19 Source : Bootstrap.java
with MIT License
from zzzzbw

/**
 * @author zbw
 * @create 2018/2/24 17:28
 */
public clreplaced Bootstrap extends Application {

    private static final Logger logger = LoggerFactory.getLogger(Bootstrap.clreplaced);

    @Override
    public void start(Stage primaryStage) throws Exception {
        logger.info("app start!");
        Parent root = FXMLLoader.load(getClreplaced().getResource("/fxml/main.fxml"));
        final Scene scene = new Scene(root, 400, 400);
        scene.getStylesheets().add(Bootstrap.clreplaced.getResource("/css/jfoenix-components.css").toExternalForm());
        scene.getStylesheets().add(Bootstrap.clreplaced.getResource("/css/jfoenix-components-style.css").toExternalForm());
        primaryStage.setreplacedle("gitPic");
        primaryStage.setScene(scene);
        primaryStage.setResizable(false);
        primaryStage.getIcons().add(new Image(Bootstrap.clreplaced.getClreplacedLoader().getResourcereplacedtream("images/github_logo.png")));
        primaryStage.show();
    }

    @Override
    public void stop() throws Exception {
        logger.info("app stop!");
        ThreadPool.getInstance().shutdown();
        super.stop();
    }

    public static void main(String[] args) {
        launch(args);
    }
}

19 Source : SyncUtil.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced SyncUtil {

    private static final Logger logger = LoggerFactory.getLogger(SyncUtil.clreplaced);

    private static final Gson gson = new Gson();

    public static String toJson(Object o) {
        if (o == null) {
            return null;
        }
        return gson.toJson(o);
    }

    public static <T> T fromJson(String json, Clreplaced<T> clazz) {
        if (json == null) {
            return null;
        }
        try {
            return gson.fromJson(json, clazz);
        } catch (JsonSyntaxException e) {
            logger.error("Fail to parse json string {} to {}", json, clazz);
            return null;
        }
    }

    public static <T> T fromJson(String json, TypeToken<T> token) {
        if (json == null) {
            return null;
        }
        try {
            return gson.fromJson(json, token.getType());
        } catch (JsonSyntaxException e) {
            logger.error("Fail to parse json string {} to {}", json, token.getType());
            return null;
        }
    }

    public static Map fromJson(String json) {
        return fromJson(json, Map.clreplaced);
    }

    public static void underscoreToCamel(SyncData data) {
        HashMap<String, Object> fields = data.getFields();
        HashMap<String, Object> tmp = new HashMap<>();
        for (Map.Entry<String, Object> e : fields.entrySet()) {
            String from = e.getKey();
            String to = underscoreToCamel(from);
            logger.debug("Rename field: {} -> {}", from, to);
            tmp.put(to, e.getValue());
        }
        fields.clear();
        fields.putAll(tmp);
    }

    public static String underscoreToCamel(String from) {
        char[] cs = from.toCharArray();
        StringBuilder sb = new StringBuilder(cs.length);
        boolean lastIsUnderscore = false;
        for (char c : cs) {
            if (c == '_') {
                lastIsUnderscore = true;
            } else if (Character.isAlphabetic(c)) {
                if (lastIsUnderscore && Character.isLowerCase(c)) {
                    sb.append(Character.toUpperCase(c));
                } else {
                    sb.append(c);
                }
                lastIsUnderscore = false;
            } else {
                logger.warn("Unsupported {} in {}", c, from);
            }
        }
        return sb.toString();
    }

    /**
     * @param key name for field which is byte[] in Java, which may come from blob type in db
     * @param sync SyncData
     */
    @Deprecated
    public static void toStr(SyncData sync, String key) {
        Object value = sync.getField(key);
        if (value != null) {
            sync.updateField(key, new String((byte[]) value, java.nio.charset.StandardCharsets.UTF_8));
        }
    }

    public static void unsignedByte(SyncData sync, String key) {
        Object field = sync.getField(key);
        if (field != null) {
            sync.updateField(key, Byte.toUnsignedInt((byte) (int) field));
        }
    }
}

19 Source : CopyOnWriteListTest.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced CopyOnWriteListTest {

    private static final Logger logger = LoggerFactory.getLogger(CopyOnWriteListTest.clreplaced);

    private static final ExecutorService service = Executors.newFixedThreadPool(2);

    @Test
    public void removeNotVisitedInFor() throws Exception {
        CopyOnWriteArrayList<Integer> l = prepareList();
        int c = l.size();
        for (Integer integer : l) {
            if (integer == 2) {
                l.remove(2 + 1);
            }
            c--;
        }
        replacedert.replacedertEquals(c, 0);
    }

    @Test
    public void removeVisitedInFor() throws Exception {
        CopyOnWriteArrayList<Integer> l = prepareList();
        int c = l.size();
        for (Integer integer : l) {
            if (integer == 2) {
                l.remove(2 - 1);
            }
            c--;
        }
        replacedert.replacedertEquals(c, 0);
    }

    @Test
    public void removeCurrentInFor() throws Exception {
        CopyOnWriteArrayList<Integer> l = prepareList();
        int c = l.size();
        for (Integer integer : l) {
            if (integer == 2) {
                l.remove(2);
            }
            c--;
        }
        replacedert.replacedertEquals(c, 0);
    }

    @Test
    public void removeThenFor() throws Exception {
        CopyOnWriteArrayList<Integer> l = prepareList();
        for (Integer integer : l) {
            if (integer == 2) {
                l.remove(2);
            }
        }
        int c = l.size();
        for (Integer integer : l) {
            c--;
        }
        replacedert.replacedertEquals(c, 0);
    }

    @Test
    public void removeMulreplacedhread() throws Exception {
        Random random = new Random();
        CopyOnWriteArrayList<Integer> l = prepareList();
        service.submit(() -> {
            while (true) {
                for (Integer integer : l) {
                    logger.info("{}", integer);
                    Thread.sleep(random.nextInt(5) * 100 + 200);
                }
            }
        });
        service.submit(() -> {
            while (true) {
                for (Integer integer : l) {
                    logger.info("{}", integer);
                    Thread.sleep(random.nextInt(5) * 100 + 100);
                    if (integer == 2) {
                        Integer remove = l.remove(2);
                        logger.info("remove {}", remove);
                    }
                }
            }
        });
        service.shutdown();
        service.awaitTermination(5, TimeUnit.SECONDS);
    }

    private CopyOnWriteArrayList<Integer> prepareList() {
        return new CopyOnWriteArrayList<>(Lists.newArrayList(0, 1, 2, 3, 4));
    }
}

19 Source : MethodFilterTemplate.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

public clreplaced MethodFilterTemplate implements SyncFilter<SyncData> {

    private final Logger logger = LoggerFactory.getLogger(MethodFilter.clreplaced);

    public void filter(List<SyncData> list) {
    /*TODO*/
    }
}

19 Source : ShutDownCenter.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

public clreplaced ShutDownCenter {

    public static final int SHUTDOWN_TIMEOUT = 5;

    public static final int SHUTDOWN_MAX_TRY = 30;

    private static AtomicBoolean shutdown = new AtomicBoolean(false);

    private static final Logger logger = LoggerFactory.getLogger(ShutDownCenter.clreplaced);

    public static void initShutDown(Throwable e) {
        boolean first = shutdown.compareAndSet(false, true);
        if (first) {
            new Thread(() -> {
                if (e instanceof InvalidConfigException) {
                    logger.error("[Shutting down] Init: {}", e.getMessage());
                } else {
                    logger.error("[Shutting down] Init", e);
                }
                System.exit(1);
            }, "syncer-shutdown-starter").start();
        } else {
            logger.warn("[Shutting down]", e);
            throw new ShutDownException(e);
        }
    }

    public static void inShutdown() {
        shutdown.set(true);
    }
}

19 Source : LocalConsumerRegistry.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * todo split it to every producer?
 * @author zzt
 */
public clreplaced LocalConsumerRegistry implements ConsumerRegistry {

    private Logger logger = LoggerFactory.getLogger(LocalConsumerRegistry.clreplaced);

    private ConcurrentHashMap<Connection, BinlogInfo> olderBinlog = new ConcurrentHashMap<>();

    private ConcurrentHashMap<Connection, DocTimestamp> smallerId = new ConcurrentHashMap<>();

    private ConcurrentSkipListSet<Connection> voted = new ConcurrentSkipListSet<>();

    private ConcurrentHashMap<Connection, Set<ConsumerSource>> consumerSources = new ConcurrentHashMap<>();

    @Override
    public boolean register(Connection connection, ConsumerSource source) {
        if (voted.contains(connection)) {
            logger.warn("Output sink is already started, fail to register");
            return false;
        }
        if (source instanceof MysqlInputSource) {
            BinlogInfo syncInitMeta = ((MysqlInputSource) source).getSyncInitMeta();
            olderBinlog.compute(connection, (k, v) -> v == null ? syncInitMeta : v.compareTo(syncInitMeta) <= 0 ? v : syncInitMeta);
        } else if (source instanceof MongoInputSource) {
            DocTimestamp syncInitMeta = ((MongoInputSource) source).getSyncInitMeta();
            smallerId.compute(connection, (k, v) -> v == null ? syncInitMeta : v.compareTo(syncInitMeta) <= 0 ? v : syncInitMeta);
        } else {
            checkState(false);
        }
        final boolean[] add = new boolean[1];
        consumerSources.compute(connection, (k, v) -> {
            if (v == null) {
                add[0] = true;
                return Sets.newHashSet(source);
            } else {
                add[0] = v.add(source);
                if (!add[0]) {
                    logger.warn("Duplicate input source {}", source.clientId());
                }
                return v;
            }
        });
        return add[0];
    }

    @Override
    public BinlogInfo votedBinlogInfo(Connection connection) {
        checkState(olderBinlog.containsKey(connection), "no input source registered");
        voted.add(connection);
        BinlogInfo binlogInfo = olderBinlog.get(connection);
        logger.info("Voted {} for {}", binlogInfo, connection);
        return binlogInfo;
    }

    @Override
    public DocTimestamp votedMongoId(Connection connection) {
        checkState(smallerId.containsKey(connection), "no input source registered");
        voted.add(connection);
        DocTimestamp docTimestamp = smallerId.get(connection);
        logger.info("Voted {} for {}", docTimestamp, connection);
        return docTimestamp;
    }

    @Override
    public HashMap<Consumer, ProducerSink> outputSink(Connection connection) {
        HashMap<Consumer, ProducerSink> res = new HashMap<>();
        if (!consumerSources.containsKey(connection))
            return res;
        for (ConsumerSource consumerSource : consumerSources.get(connection)) {
            res.put(new Consumer(consumerSource), new LocalProducerSink(consumerSource));
        }
        return res;
    }

    @Override
    public Set<Connection> wantedSource() {
        return new HashSet<>(consumerSources.keySet());
    }
}

19 Source : ProducerStarter.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced ProducerStarter implements Starter {

    private static ProducerStarter starter;

    private final Logger logger = LoggerFactory.getLogger(ProducerStarter.clreplaced);

    private final Set<ProducerMaster> masterSources;

    private final ExecutorService service;

    private final ConsumerRegistry consumerRegistry;

    private final LinkedList<MasterConnector> connectors = new LinkedList<>();

    private ProducerStarter(ProducerInput input, SyncerInput syncerConfigInput, ConsumerRegistry consumerRegistry) {
        masterSources = fromPipelineConfig(input);
        int size = masterSources.stream().mapToInt(p -> p.getRealConnection().getReals().size()).sum();
        if (size > Runtime.getRuntime().availableProcessors()) {
            logger.warn("Too many master source: {} > cores", size);
        }
        service = Executors.newFixedThreadPool(size, new NamedThreadFactory("syncer-producer"));
        this.consumerRegistry = consumerRegistry;
    }

    public static ProducerStarter getInstance(ProducerInput input, SyncerInput syncerConfigInput, ConsumerRegistry consumerRegistry) {
        if (starter == null) {
            starter = new ProducerStarter(input, syncerConfigInput, consumerRegistry);
        }
        return starter;
    }

    @Override
    public Starter start() throws IOException {
        logger.info("Start handling [{}]", masterSources);
        if (masterSources.size() > 1) {
            logger.warn("Connect to multiple masters, not suggested usage");
        }
        Set<Connection> wanted = consumerRegistry.wantedSource();
        for (ProducerMaster masterSource : masterSources) {
            Connection mayClusterConnection = masterSource.getRealConnection();
            for (Connection real : mayClusterConnection.getReals()) {
                wanted.remove(real);
                addConnector(masterSource, real);
            }
        }
        if (!wanted.isEmpty()) {
            logger.warn("Some consumer wanted source is not configured in `producer`: {}", wanted);
        }
        return this;
    }

    private void addConnector(ProducerMaster masterSource, Connection connection) {
        if (consumerRegistry.outputSink(connection).isEmpty()) {
            logger.warn("Skip {} because no consumer registered", masterSource);
            return;
        }
        try {
            MasterConnector masterConnector = null;
            switch(masterSource.getType()) {
                case MySQL:
                    masterConnector = new MysqlMasterConnector(new MysqlConnection(connection), masterSource.getFile(), consumerRegistry, masterSource.isOnlyUpdated());
                    break;
                case Mongo:
                    masterConnector = new MongoMasterConnectorFactory(new MongoConnection(connection), consumerRegistry).getMongoConnectorByServerVersion(masterSource);
                    break;
                default:
                    throw new IllegalStateException("Not implemented type");
            }
            connectors.add(masterConnector);
            service.submit(masterConnector);
        } catch (InvalidConfigException e) {
            logger.error("Invalid config for {}", masterSource);
            ShutDownCenter.initShutDown(e);
        } catch (IOException | SchemaUnavailableException e) {
            logger.error("Fail to connect to master source: {}", masterSource);
            ShutDownCenter.initShutDown(e);
        }
    }

    private Set<ProducerMaster> fromPipelineConfig(ProducerInput input) {
        return input.getMasterSet();
    }

    @Override
    public void close() throws Exception {
        for (MasterConnector connector : connectors) {
            connector.close();
        }
        service.shutdownNow();
        while (!service.awaitTermination(ShutDownCenter.SHUTDOWN_TIMEOUT, TimeUnit.SECONDS)) {
            logger.warn("[Shutting down] producer");
            service.shutdownNow();
        }
    }

    @Override
    public void registerToHealthCenter() {
        for (ProducerMaster source : masterSources) {
            Connection connection = source.getRealConnection();
            for (Connection real : connection.getReals()) {
                if (consumerRegistry.outputSink(real).isEmpty()) {
                    SyncerHealth.producer(real.connectionIdentifier(), Health.inactive("No consumer registered"));
                } else {
                    SyncerHealth.producer(real.connectionIdentifier(), Health.green());
                }
            }
        }
    }
}

19 Source : SyncListener.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced SyncListener implements BinaryLogClient.EventListener {

    /**
     * @see com.github.zzt93.syncer.producer.input.mysql.connect.SyncListener#onEvent(Event)
     */
    private static Map<EventType, SimpleEventType> map = new EnumMap<>(EventType.clreplaced);

    static {
        // unify multiple update/delete/write into one single type
        // to avoid checking wrong type
        map.put(EventType.PRE_GA_WRITE_ROWS, SimpleEventType.WRITE);
        map.put(EventType.WRITE_ROWS, SimpleEventType.WRITE);
        map.put(EventType.EXT_WRITE_ROWS, SimpleEventType.WRITE);
        map.put(EventType.PRE_GA_UPDATE_ROWS, SimpleEventType.UPDATE);
        map.put(EventType.UPDATE_ROWS, SimpleEventType.UPDATE);
        map.put(EventType.EXT_UPDATE_ROWS, SimpleEventType.UPDATE);
        map.put(EventType.PRE_GA_DELETE_ROWS, SimpleEventType.DELETE);
        map.put(EventType.DELETE_ROWS, SimpleEventType.DELETE);
        map.put(EventType.EXT_DELETE_ROWS, SimpleEventType.DELETE);
    }

    private final Logger logger = LoggerFactory.getLogger(SyncListener.clreplaced);

    private final MysqlDispatcher mysqlDispatcher;

    private final String connectorIdentifier;

    private final MysqlConnection connection;

    private Event last;

    public SyncListener(MysqlDispatcher mysqlDispatcher, MysqlConnection connection) {
        this.mysqlDispatcher = mysqlDispatcher;
        this.connection = connection;
        this.connectorIdentifier = connection.connectionIdentifier();
    }

    /**
     * May return null, should be handled
     *
     * @see #onEvent(Event)
     */
    private static SimpleEventType toSimpleEvent(EventType type) {
        return map.getOrDefault(type, null);
    }

    @Override
    public void onEvent(Event event) {
        EventType eventType = event.getHeader().getEventType();
        logger.trace("Receive binlog event: {}", event);
        switch(eventType) {
            case TABLE_MAP:
                this.last = event;
                break;
            case QUERY:
                // Event{header=EventHeaderV4{timestamp=1574766674000, eventType=QUERY, serverId=1, headerLength=19, dataLength=162, nextPosition=1747640, flags=0}, data=QueryEventData{threadId=585243, executionTime=0, errorCode=0, database='copy_0', sql='/* ApplicationName=IntelliJ IDEA 2019.1 */ alter table toCopy modify replacedle varchar(254) default '' null'}}
                // Event{header=EventHeaderV4{timestamp=1574766674000, eventType=QUERY,serverId=1, headerLength=19, dataLength=169, nextPosition=1747893, flags=0}, data=QueryEventData{threadId=585243, executionTime=0, errorCode=0, database='copy_0', sql='/* ApplicationName=IntelliJ IDEA 2019.1 */ alter table copy_0.toCopy modify replacedle varchar(255) default '' null'}}
                QueryEventData data = event.getData();
                String sql = data.getSql();
                // if SQL like alter xx after yy
                // trigger retrieve meta info
                // add new TableMeta, remove old TableMeta from SchemaMeta
                // ((EventHeaderV4) event.getHeader()).getFlags()
                AlterMeta alterMeta = SQLHelper.alterMeta(data.getDatabase(), sql);
                if (alterMeta != null) {
                    try {
                        mysqlDispatcher.updateSchemaMeta(alterMeta.setConnection(connection));
                    } catch (Throwable e) {
                        logger.error("Fail to update meta {}, {}", event, alterMeta, e);
                    }
                }
                break;
            default:
                SimpleEventType type = toSimpleEvent(eventType);
                if (type == null) {
                    break;
                }
                try {
                    mysqlDispatcher.dispatch(type, last, event);
                } catch (InvalidConfigException e) {
                    SyncerHealth.producer(connectorIdentifier, Health.red(e.getMessage()));
                    ShutDownCenter.initShutDown(e);
                } catch (Throwable e) {
                    logger.error("Fail to dispatch {}", event);
                    SyncerHealth.producer(connectorIdentifier, Health.red(e.getMessage()));
                    ShutDownCenter.initShutDown(e);
                }
        }
    }
}

19 Source : LogLifecycleListener.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced LogLifecycleListener implements BinaryLogClient.LifecycleListener {

    private Logger logger = LoggerFactory.getLogger(LogLifecycleListener.clreplaced);

    @Override
    public void onConnect(BinaryLogClient client) {
        logger.info("Connected {}@{}", client.getBinlogFilename(), client.getBinlogPosition());
    }

    @Override
    public void onCommunicationFailure(BinaryLogClient client, Exception ex) {
        if (binlogDeprecated(ex)) {
            if (dupServerId(ex)) {
                throw new DupServerIdException(ex);
            }
            throw new InvalidBinlogException(ex, client.getBinlogFilename(), client.getBinlogPosition());
        } else {
            logger.error("Communication failure", ex);
        }
    }

    private boolean dupServerId(Exception ex) {
        return ex.getMessage().startsWith("A slave with the same server_uuid/server_id as this slave has connected to the master");
    }

    private boolean binlogDeprecated(Exception ex) {
        return ex instanceof ServerException && ((ServerException) ex).getErrorCode() == 1236;
    }

    @Override
    public void onEventDeserializationFailure(BinaryLogClient client, Exception ex) {
        logger.error("Deserialization failure", ex);
    }

    @Override
    public void onDisconnect(BinaryLogClient client) {
        logger.warn("Disconnect {}@{}", client.getBinlogFilename(), client.getBinlogPosition());
    }
}

19 Source : MongoV4MasterConnector.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced MongoV4MasterConnector extends MongoConnectorBase {

    private static final Logger logger = LoggerFactory.getLogger(MongoV4MasterConnector.clreplaced);

    private static final String NS = "ns";

    private static final int MONGO_CHANGE_STREAM_BATCH_SIZE = 100;

    private final boolean bsonConversion;

    private MongoCursor<ChangeStreamDoreplacedent<Doreplacedent>> cursor;

    private MongoDispatcher mongoDispatcher;

    private ChangeStreamIterable<Doreplacedent> changeStreamDoreplacedents;

    MongoV4MasterConnector(MongoConnection connection, ConsumerRegistry registry, ProducerMaster.MongoV4Option mongoV4Option) {
        super(connection);
        configDispatch(connection, registry);
        configQuery(connection, registry, mongoV4Option.isUpdateLookUp());
        bsonConversion = mongoV4Option.isBsonConversion();
    }

    private List<BsonDoreplacedent> getNamespaces(MongoConnection connection, ConsumerRegistry registry) {
        return getNamespaces(connection, registry, (repoEnreplacedy) -> new BsonDoreplacedent("db", new BsonString(repoEnreplacedy[0])).append("coll", new BsonString(repoEnreplacedy[1]))).collect(Collectors.toList());
    }

    private void configQuery(MongoConnection connection, ConsumerRegistry registry, boolean updateLookUp) {
        List<Bson> pipeline = singletonList(Aggregates.match(Filters.and(Filters.in(NS, getNamespaces(connection, registry)), Filters.in("operationType", asList("insert", "delete", "replace", "update")))));
        changeStreamDoreplacedents = client.watch(pipeline).batchSize(MONGO_CHANGE_STREAM_BATCH_SIZE);
        DocTimestamp docTimestamp = registry.votedMongoId(connection);
        if (DocTimestamp.earliest == docTimestamp) {
            MongoCursor<Doreplacedent> firstLog = client.getDatabase(LOCAL).getCollection(OPLOG_RS).find(new Doreplacedent()).limit(1).iterator();
            if (firstLog.hasNext()) {
                Doreplacedent next = firstLog.next();
                logger.info("Connect to earliest oplog time: {}", next.get(TS));
                changeStreamDoreplacedents.startAtOperationTime(((BsonTimestamp) next.get(TS)));
            } else {
                logger.info("Doreplacedent not found in local.oplog.rs -- is this a new and empty db instance?");
                changeStreamDoreplacedents.startAtOperationTime(docTimestamp.getTimestamp());
            }
        } else {
            /*
      Optional. The starting point for the change stream.
      If the specified starting point is in the past, it must be in the time range of the oplog.
      To check the time range of the oplog, see rs.printReplicationInfo().
       */
            changeStreamDoreplacedents.startAtOperationTime(docTimestamp.getTimestamp());
        }
        // UPDATE_LOOKUP: return the most current majority-committed version of the updated doreplacedent.
        // i.e. run at different time, will have different fullDoreplacedent
        changeStreamDoreplacedents.fullDoreplacedent(updateLookUp ? FullDoreplacedent.UPDATE_LOOKUP : FullDoreplacedent.DEFAULT);
    }

    private void configDispatch(MongoConnection connection, ConsumerRegistry registry) {
        HashMap<Consumer, ProducerSink> schemaSinkMap = registry.outputSink(connection);
        mongoDispatcher = new MongoDispatcher(schemaSinkMap);
    }

    @Override
    public void configCursor() {
        this.cursor = changeStreamDoreplacedents.iterator();
    }

    @Override
    public void closeCursor() {
        if (cursor != null) {
            cursor.close();
        }
    }

    @Override
    public void eventLoop() {
        // here hasNext is not blocking, may change to reactive
        // https://mongodb.github.io/mongo-java-driver-reactivestreams/1.13/getting-started/quick-tour-primer/
        while (cursor.hasNext()) {
            ChangeStreamDoreplacedent<Doreplacedent> d = cursor.next();
            MongoDataId dataId = DataId.fromDoreplacedent(d.getClusterTime());
            MDC.put(LogbackLoggingField.EID, dataId.eventId());
            try {
                mongoDispatcher.dispatch(null, fromChangeStream(d, dataId));
            } catch (InvalidConfigException e) {
                ShutDownCenter.initShutDown(e);
            } catch (Throwable e) {
                logger.error("Fail to dispatch this doc {}", d);
                Throwables.throwIfUnchecked(e);
            }
        }
    }

    private SyncData fromChangeStream(ChangeStreamDoreplacedent<Doreplacedent> d, MongoDataId dataId) {
        MongoNamespace namespace = d.getNamespace();
        HashMap<String, Object> full = new HashMap<>(), updated = null;
        SimpleEventType type;
        switch(d.getOperationType()) {
            case UPDATE:
                // e.g. members.6.state -> {BsonInt32@9194} "BsonInt32{value=1}"
                // e.g. addToSet  -> all elements in bson array
                type = SimpleEventType.UPDATE;
                replacedert d.getUpdateDescription() != null;
                UpdateDescription updateDescription = d.getUpdateDescription();
                updated = new HashMap<>(getUpdatedFields(d.getFullDoreplacedent(), updateDescription.getUpdatedFields(), bsonConversion));
                if (d.getFullDoreplacedent() != null) {
                    full.putAll(getFullDoreplacedent(d));
                // use UpdateDescription to overrides latest version
                } else {
                    full.put(ID, getId(d));
                }
                full.putAll(updated);
                addRemovedFields(updated, d.getUpdateDescription().getRemovedFields());
                break;
            case DELETE:
                type = SimpleEventType.DELETE;
                full.put(ID, getId(d));
                break;
            case INSERT:
            case // write will overwrite for ES, not suitable for other output
            REPLACE:
                type = SimpleEventType.WRITE;
                full.putAll(getFullDoreplacedent(d));
                break;
            case OTHER:
            case INVALIDATE:
            case RENAME:
            case DROP:
            case DROP_DATABASE:
            default:
                return null;
        }
        return new SyncData(dataId, type, namespace.getDatabaseName(), namespace.getCollectionName(), ID, full.get(ID), new NamedChangeStream(full, updated));
    }

    static Object getId(ChangeStreamDoreplacedent<Doreplacedent> d) {
        BsonDoreplacedent doreplacedentKey = d.getDoreplacedentKey();
        BsonValue o = doreplacedentKey.get(ID);
        return MongoTypeUtil.convertBson(o);
    }

    private Map getFullDoreplacedent(ChangeStreamDoreplacedent<Doreplacedent> d) {
        return (Map) MongoTypeUtil.convertBsonTypes(d.getFullDoreplacedent());
    }

    static Map getUpdatedFields(Doreplacedent fullDoreplacedent, BsonDoreplacedent updatedFields, boolean bsonConversion) {
        if (bsonConversion) {
            if (fullDoreplacedent == null) {
                return (Map) MongoTypeUtil.convertBson(updatedFields);
            }
            HashMap<String, Object> res = new HashMap<>();
            for (String key : updatedFields.keySet()) {
                res.put(key, fullDoreplacedent.get(key));
            }
            return res;
        }
        return updatedFields;
    }

    private void addRemovedFields(HashMap<String, Object> updated, List<String> removedFields) {
        if (removedFields == null) {
            return;
        }
        for (String removedField : removedFields) {
            updated.put(removedField, null);
        }
    }
}

19 Source : MongoMasterConnectorFactory.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced MongoMasterConnectorFactory {

    private static final String LOCAL = "local";

    private static final int DEPLOYMENT_CHANGE_STREAM_VERSION = 4;

    private final Logger logger = LoggerFactory.getLogger(MongoMasterConnectorFactory.clreplaced);

    private final int mainVersion;

    private final ConsumerRegistry registry;

    private final MongoConnection connection;

    public MongoMasterConnectorFactory(MongoConnection connection, ConsumerRegistry registry) {
        this.registry = registry;
        this.connection = connection;
        MongoClient client = new MongoClient(new MongoClientURI(connection.toConnectionUrl(null)));
        mainVersion = getMainVersion(client);
    }

    private int getMainVersion(MongoClient client) {
        String version = client.getDatabase(LOCAL).runCommand(new BsonDoreplacedent("buildinfo", new BsonString(""))).get("version").toString();
        return Integer.parseInt(version.split("\\.")[0]);
    }

    public MongoConnectorBase getMongoConnectorByServerVersion(ProducerMaster producerMaster) {
        if (mainVersion >= DEPLOYMENT_CHANGE_STREAM_VERSION) {
            // https://docs.mongodb.com/manual/changeStreams/#watch-collection-database-deployment
            return new MongoV4MasterConnector(connection, registry, producerMaster.mongoV4Option());
        }
        return new MongoMasterConnector(connection, registry);
    }
}

19 Source : MongoConnectorBase.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public abstract clreplaced MongoConnectorBase implements MasterConnector {

    static final String OPLOG_RS = "oplog.rs";

    static final String LOCAL = "local";

    static final String TS = "ts";

    private final String identifier;

    private final Logger logger = LoggerFactory.getLogger(MongoConnectorBase.clreplaced);

    protected MongoClient client;

    MongoConnectorBase(MongoConnection connection) {
        client = new MongoClient(new MongoClientURI(connection.toConnectionUrl(null)));
        identifier = connection.connectionIdentifier();
    }

    <T> Stream<T> getNamespaces(MongoConnection connection, ConsumerRegistry registry, Function<String[], T> f) {
        Set<String> producerDbName = new HashSet<>();
        for (String dbName : client.listDatabaseNames()) {
            producerDbName.add(dbName);
        }
        checkOplog(producerDbName);
        Set<Consumer> consumers = registry.outputSink(connection).keySet();
        return consumers.stream().map(Consumer::getRepos).flatMap(Set::stream).flatMap(s -> {
            if (!producerDbName.contains(s.getName())) {
                throw new InvalidConfigException("No such repo(" + s.getName() + ") in " + connection);
            }
            Set<String> producerCollectionName = new HashSet<>();
            for (String collectionName : client.getDatabase(s.getName()).listCollectionNames()) {
                producerCollectionName.add(collectionName);
            }
            List<Enreplacedy> enreplacedies = s.getEnreplacedies();
            ArrayList<T> res = new ArrayList<>(enreplacedies.size());
            for (Enreplacedy enreplacedy : enreplacedies) {
                if (!producerCollectionName.contains(enreplacedy.getName())) {
                    throw new InvalidConfigException("No such collection(" + s.getName() + "." + enreplacedy.getName() + ") in " + connection);
                }
                res.add(f.apply(new String[] { s.getName(), enreplacedy.getName() }));
            }
            return res.stream();
        });
    }

    private void checkOplog(Set<String> producerDbName) {
        if (!producerDbName.contains(LOCAL)) {
            throw new InvalidConfigException("Replication not detected. Enable by: rs.initiate()");
        }
        HashSet<String> names = new HashSet<>();
        for (String collectionName : client.getDatabase(LOCAL).listCollectionNames()) {
            names.add(collectionName);
        }
        if (!names.contains(OPLOG_RS)) {
            throw new InvalidConfigException("Replication not detected. Enable by: rs.initiate()");
        }
    }

    @Override
    public void close() {
        try {
            closeCursor();
            client.close();
        } catch (Throwable e) {
            logger.error("[Shutting down] failed", e);
            return;
        }
        MasterConnector.super.close();
    }

    @Override
    public void loop() {
        Thread.currentThread().setName(identifier);
        logger.info("Start export from [{}]", identifier);
        long sleepInSecond = 1;
        while (!Thread.interrupted()) {
            try {
                configCursor();
                eventLoop();
            } catch (MongoTimeoutException | MongoSocketException e) {
                logger.error("Fail to connect to remote: {}, retry in {} second", identifier, sleepInSecond);
                sleepInSecond = FallBackPolicy.POW_2.sleep(sleepInSecond);
            } catch (MongoInterruptedException e) {
                logger.warn("Mongo master interrupted");
                throw new ShutDownException(e);
            }
        }
    }

    public abstract void closeCursor();

    public abstract void configCursor();

    public abstract void eventLoop();
}

19 Source : RowsEvent.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * <a href="https://dev.mysql.com/doc/internals/en/binlog-row-image.html">binlog row image
 * format</a>
 *
 * @author zzt
 */
public abstract clreplaced RowsEvent {

    private static final Logger logger = LoggerFactory.getLogger(RowsEvent.clreplaced);

    public static List<NamedFullRow> getNamedRows(List<IndexedFullRow> indexedRow, List<Integer> interestedAndPkIndex, Map<Integer, String> indexToName) {
        List<NamedFullRow> res = new ArrayList<>(indexedRow.size());
        for (IndexedFullRow indexedFullRow : indexedRow) {
            try {
                res.add(indexedFullRow.toNamed(interestedAndPkIndex, indexToName));
            } catch (ArrayIndexOutOfBoundsException e) {
                logger.error("Current schema({}) does not match old binlog record({}), fail to parse it. Try to connect to latest binlog.", interestedAndPkIndex, indexedFullRow.length());
                throw new MismatchedSchemaException("Current schema does not match old binlog record, fail to parse it. Try to connect to latest binlog", e);
            }
        }
        return res;
    }

    public static String getPrimaryKey(Map<Integer, String> indexToName, Set<Integer> primaryKeys) {
        Iterator<Integer> iterator = primaryKeys.iterator();
        Integer key = iterator.next();
        return indexToName.get(key);
    }

    public static List<IndexedFullRow> getIndexedRows(SimpleEventType eventType, EventData data, Set<Integer> primaryKeys) {
        switch(eventType) {
            case UPDATE:
                return UpdateRowsEvent.getIndexedRows((UpdateRowsEventData) data);
            case WRITE:
                WriteRowsEventData write = (WriteRowsEventData) data;
                return getIndexedRows(write.getRows(), write.getIncludedColumns());
            case DELETE:
                DeleteRowsEventData delete = (DeleteRowsEventData) data;
                return getIndexedRows(delete.getRows(), delete.getIncludedColumns());
            default:
                throw new IllegalArgumentException("Unsupported event type");
        }
    }

    private static List<IndexedFullRow> getIndexedRows(List<Serializable[]> rows, BitSet includedColumns) {
        List<IndexedFullRow> res = new LinkedList<>();
        for (Serializable[] row : rows) {
            res.add(new IndexedFullRow(row));
        }
        return res;
    }
}

19 Source : MongoDispatcher.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced MongoDispatcher implements Dispatcher {

    private final Logger logger = LoggerFactory.getLogger(MongoDispatcher.clreplaced);

    private final HashMap<String, List<JsonKeyFilter>> directOutput = new HashMap<>();

    private final HashMap<Pattern, JsonKeyFilter> regexOutput = new HashMap<>();

    public MongoDispatcher(HashMap<Consumer, ProducerSink> schemaSinkMap) {
        for (Entry<Consumer, ProducerSink> entry : schemaSinkMap.entrySet()) {
            for (Repo repo : entry.getKey().getRepos()) {
                if (repo.noNamePattern()) {
                    directOutput.computeIfAbsent(repo.getName(), k -> new ArrayList<>()).add(new JsonKeyFilter(repo, entry.getValue()));
                } else {
                    regexOutput.put(repo.getNamePattern(), new JsonKeyFilter(repo, entry.getValue()));
                }
            }
        }
    }

    @Override
    public boolean dispatch(SimpleEventType simpleEventType, Object... data) {
        SyncData syncData = (SyncData) data[0];
        if (syncData == null) {
            return false;
        }
        String db = syncData.getRepo();
        if (directOutput.containsKey(db)) {
            for (JsonKeyFilter keyFilter : directOutput.get(db)) {
                keyFilter.output(syncData);
            }
        } else {
            for (Entry<Pattern, JsonKeyFilter> entry : regexOutput.entrySet()) {
                if (entry.getKey().matcher(db).find()) {
                    entry.getValue().output(syncData);
                } else {
                    logger.warn("Unknown syncData {}", syncData);
                }
            }
        }
        return true;
    }
}

19 Source : FailureLog.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced FailureLog<T> implements Resource {

    private static final Gson gson = new GsonBuilder().disableHtmlEscaping().create();

    private final Logger logger = LoggerFactory.getLogger(FailureLog.clreplaced);

    private final Type type;

    private final AtomicInteger itemCount = new AtomicInteger(0);

    private final BufferedWriter writer;

    private final int countLimit;

    private final ScheduledExecutorService service;

    private FailureLog(Path path, FailureLogConfig limit, TypeToken token) throws FileNotFoundException {
        countLimit = limit.getCountLimit();
        service = Executors.newScheduledThreadPool(1, new NamedThreadFactory("syncer-failure-log-timer"));
        service.scheduleWithFixedDelay(() -> itemCount.set(0), limit.getTimeLimit(), limit.getTimeLimit(), limit.getUnit());
        type = token.getType();
        replacedert token.getRawType() == FailureEntry.clreplaced;
        FileUtil.createFile(path, (e) -> logger.error("Fail to create failure log file [{}]", path, e));
        writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path.toFile(), true)));
    }

    private List<T> recover(Path path) {
        List<T> res = new LinkedList<>();
        // TODO 18/2/1 @see SyncDeserializer
        return res;
    }

    public boolean log(T data, String errorMsg) {
        itemCount.incrementAndGet();
        try {
            FailureEntry<T> failureEntry = new FailureEntry<>(data, LocalDateTime.now(), errorMsg);
            writer.write(gson.toJson(failureEntry, type));
            writer.newLine();
            writer.flush();
        } catch (IOException e) {
            logger.error("Fail to convert to json {}", data, e);
        } catch (StackOverflowError e) {
            logger.error("Fail to convert to json {}", data);
            throw e;
        }
        int count = itemCount.get();
        if (count > countLimit) {
            logger.error("Too many failure: {} > {}", count, countLimit);
        // throw new FailureException("Too many failed items, abort and need human influence");
        }
        return true;
    }

    @Override
    public void cleanup() throws IOException {
        service.shutdown();
        writer.close();
    }

    public static <T> FailureLog<T> getLogger(Path path, FailureLogConfig limit, TypeToken token) {
        try {
            return new FailureLog<>(path, limit, token);
        } catch (FileNotFoundException e) {
            throw new IllegalStateException("Impossible state", e);
        }
    }
}

19 Source : RedisChannel.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced RedisChannel implements BufferedChannel<RedisCallback> {

    private final Logger logger = LoggerFactory.getLogger(RedisChannel.clreplaced);

    private final BatchBuffer<SyncWrapper<RedisCallback>> batchBuffer;

    private final PipelineBatchConfig batch;

    private final Ack ack;

    private final FailureLog<SyncData> request;

    private final RedisTemplate<String, Object> template;

    private final OperationMapper operationMapper;

    private final String id;

    private Expression expression;

    public RedisChannel(Redis redis, SyncerOutputMeta outputMeta, Ack ack) {
        this.batch = redis.getBatch();
        id = redis.connectionIdentifier();
        this.batchBuffer = new BatchBuffer<>(batch);
        this.ack = ack;
        FailureLogConfig failureLog = redis.getFailureLog();
        Path path = Paths.get(outputMeta.getFailureLogDir(), id);
        request = FailureLog.getLogger(path, failureLog, new TypeToken<FailureEntry<SyncWrapper<String>>>() {
        });
        template = new RedisTemplate<>();
        LettuceConnectionFactory factory = redis.getConnectionFactory();
        factory.afterPropertiesSet();
        template.setConnectionFactory(factory);
        template.afterPropertiesSet();
        operationMapper = new OperationMapper(redis.getMapping());
        SpelExpressionParser parser = new SpelExpressionParser();
        if (!StringUtils.isEmpty(redis.conditionExpr())) {
            try {
                expression = parser.parseExpression(redis.conditionExpr());
            } catch (ParseException e) {
                throw new InvalidConfigException("Fail to parse [condition] for [redis] output channel", e);
            }
        }
    }

    @Override
    public long getDelay() {
        return batch.getDelay();
    }

    @Override
    public TimeUnit getDelayUnit() {
        return batch.getDelayTimeUnit();
    }

    @Override
    public boolean flush() {
        List<SyncWrapper<RedisCallback>> aim = batchBuffer.flush();
        send(aim);
        return aim != null;
    }

    private void send(List<SyncWrapper<RedisCallback>> aim) {
        if (aim != null) {
            try {
                template.executePipelined((RedisCallback<Void>) connection -> {
                    for (SyncWrapper<RedisCallback> wrapper : aim) {
                        wrapper.getData().doInRedis(connection);
                    }
                    return null;
                });
                ackSuccess(aim);
            } catch (Exception e) {
                retryFailed(aim, e);
            }
        }
    }

    @Override
    public boolean flushIfReachSizeLimit() {
        List<SyncWrapper<RedisCallback>> wrappers = batchBuffer.flushIfReachSizeLimit();
        send(wrappers);
        return wrappers != null;
    }

    @Override
    public void setFlushDone() {
        batchBuffer.flushDone();
    }

    @Override
    public void ackSuccess(List<SyncWrapper<RedisCallback>> aim) {
        for (SyncWrapper wrapper : aim) {
            ack.remove(wrapper.getSourceId(), wrapper.getSyncDataId());
        }
    }

    @Override
    public void retryFailed(List<SyncWrapper<RedisCallback>> aim, Throwable e) {
        for (SyncWrapper wrapper : aim) {
            wrapper.inc();
            if (wrapper.retryCount() > batch.getMaxRetry()) {
                request.log(wrapper.getEvent(), e.getMessage());
            }
        }
        logger.error("{}", aim, e);
    }

    @Override
    public boolean checkpoint() {
        return ack.flush();
    }

    @Override
    public boolean output(SyncData event) {
        throw new UnsupportedOperationException("Not implemented");
    // TODO 18/4/16 add flushIfReachSizeLimit
    // if (expression == null) {
    // return batchBuffer.add(new SyncWrapper<>(event, operationMapper.map(event)));
    // }
    // Boolean value = expression.getValue(event.getContext(), Boolean.clreplaced);
    // if (value == null || !value) {
    // ack.remove(event.getSourceIdentifier(), event.getDataId());
    // return false;
    // }
    // return batchBuffer.add(new SyncWrapper<>(event, operationMapper.map(event)));
    // BufferedChannel.super.flushAndSetFlushDone(true);
    }

    @Override
    public String des() {
        return "RedisChannel{" + "template=" + template + '}';
    }

    @Override
    public void close() {
    }

    @Override
    public String id() {
        return id;
    }
}

19 Source : OperationMapper.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced OperationMapper implements Mapper<SyncData, RedisCallback> {

    private final Logger logger = LoggerFactory.getLogger(OperationMapper.clreplaced);

    private final OperationMapping mapping;

    private Expression keyExpr;

    private Expression valueExpr;

    public OperationMapper(OperationMapping mapping) {
        this.mapping = mapping;
        SpelExpressionParser parser = new SpelExpressionParser();
        try {
            keyExpr = parser.parseExpression(mapping.getKey());
            valueExpr = parser.parseExpression(mapping.getValue());
        } catch (ParseException | IllegalStateException e) {
            throw new InvalidConfigException("Fail to parse [mapping] config for [redis] output", e);
        }
    }

    @Override
    public RedisCallback map(SyncData data) {
        StandardEvaluationContext context = data.getContext();
        String key = keyExpr.getValue(context, String.clreplaced);
        Object value = valueExpr.getValue(context);
        final byte[] rawKey = rawKey(key);
        final byte[] rawValue = rawValue(value);
        return connection -> {
            mapping.operationMethod(connection).apply(rawKey, rawValue);
            return null;
        };
    }

    private byte[] rawKey(String key) {
        return key.getBytes(StandardCharsets.UTF_8);
    }

    private byte[] rawValue(Object value) {
        return ((String) value).getBytes(StandardCharsets.UTF_8);
    }
}

19 Source : KVMapper.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced KVMapper implements Mapper<SyncData, HashMap<String, Object>> {

    public static final String FAKE_KEY = "any.Key";

    private final Logger logger = LoggerFactory.getLogger(KVMapper.clreplaced);

    private final Map<String, Object> mapping;

    public KVMapper(Map<String, Object> mapping) {
        HashMap<String, Object> tmp = new HashMap<>();
        initMapping(mapping, tmp, new SpelExpressionParser());
        this.mapping = Collections.unmodifiableMap(tmp);
    }

    public KVMapper(HashMap<String, Object> mapping) {
        HashMap<String, Object> tmp = new HashMap<>();
        initMapping(mapping, tmp, new SpelExpressionParser());
        this.mapping = Collections.unmodifiableMap(tmp);
    }

    public HashMap<String, Object> map(SyncData data) {
        HashMap<String, Object> res = new HashMap<>();
        SyncDataTypeUtil.mapToJson(data, mapping, res, true);
        logger.debug("SyncData json: {}", res);
        return res;
    }

    private void initMapping(Map<String, Object> mapping, Map<String, Object> res, SpelExpressionParser parser) {
        for (Entry<String, Object> entry : mapping.entrySet()) {
            String key = entry.getKey();
            Object value = entry.getValue();
            if (value instanceof Map) {
                Map map = (Map) value;
                initMapping(map, res, parser);
            } else if (value instanceof String) {
                String expr = (String) value;
                switch(expr) {
                    case SyncDataTypeUtil.ROW_ALL:
                    case SyncDataTypeUtil.EXTRA_ALL:
                    case SyncDataTypeUtil.ROW_FLATTEN:
                    case SyncDataTypeUtil.EXTRA_FLATTEN:
                        res.put(key, expr);
                        break;
                    default:
                        Expression expression = parser.parseExpression(expr);
                        res.put(key, expression);
                        break;
                }
            } else {
                res.put(key, value);
            }
        }
    }
}

19 Source : KafkaChannel.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced KafkaChannel implements OutputChannel, AckChannel<String> {

    private final Logger logger = LoggerFactory.getLogger(KafkaChannel.clreplaced);

    private final KafkaTemplate<String, Object> kafkaTemplate;

    private final Ack ack;

    private final FailureLog<SyncWrapper<String>> request;

    private final String consumerId;

    private final MsgMapper msgMapper;

    private final String identifier;

    public KafkaChannel(Kafka kafka, SyncerOutputMeta outputMeta, Ack ack) {
        DefaultKafkaProducerFactory<String, Object> factory = new DefaultKafkaProducerFactory<>(kafka.buildProperties());
        this.kafkaTemplate = new KafkaTemplate<>(factory);
        this.ack = ack;
        this.consumerId = kafka.getConsumerId();
        this.msgMapper = new MsgMapper(kafka.getMsgMapping());
        ClusterConnection connection = kafka.getConnection();
        FailureLogConfig failureLog = kafka.getFailureLog();
        identifier = connection.connectionIdentifier();
        Path path = Paths.get(outputMeta.getFailureLogDir(), identifier);
        this.request = FailureLog.getLogger(path, failureLog, new TypeToken<FailureEntry<SyncWrapper<String>>>() {
        });
    }

    @Override
    public void ackSuccess(List<SyncWrapper<String>> aim) {
        for (SyncWrapper<String> wrapper : aim) {
            ack.remove(wrapper.getSourceId(), wrapper.getSyncDataId());
        }
    }

    @Override
    public void retryFailed(List<SyncWrapper<String>> aim, Throwable e) {
        SyncWrapper<String> wrapper = aim.get(0);
        ErrorLevel level = level(e, wrapper, wrapper.retryCount());
        if (level.retriable()) {
            doSend(wrapper.getData(), wrapper);
            return;
        }
        // TODO 18/11/14 test producer retry
        // because kafka template has configured to retry
        logger.error("Max retry exceed, write '{}' to failure log", wrapper, e);
        request.log(wrapper, e.getMessage());
        ack.remove(wrapper.getSourceId(), wrapper.getSyncDataId());
    }

    @Override
    public ErrorLevel level(Throwable e, SyncWrapper wrapper, int maxTry) {
        if (e instanceof KafkaProducerException) {
            e = e.getCause();
        }
        if (e instanceof TimeoutException || e instanceof NotLeaderForParreplacedionException) {
            return ErrorLevel.RETRIABLE_ERROR;
        }
        return ErrorLevel.MAX_TRY_EXCEED;
    }

    @Override
    public boolean checkpoint() {
        return ack.flush();
    }

    @Override
    public boolean output(SyncData event) throws InterruptedException {
        String topic = msgMapper.map(event);
        SyncWrapper<String> wrapper = new SyncWrapper<>(event, topic);
        doSend(topic, wrapper);
        return true;
    }

    private void doSend(String topic, SyncWrapper<String> wrapper) {
        final SyncData event = wrapper.getEvent();
        // require that messages with the same key (for instance, a unique id) are always seen in the correct order,
        // attaching a key to messages will ensure messages with the same key always go to the same parreplacedion in a topic
        ListenableFuture<SendResult<String, Object>> future;
        Long parreplacedionId = event.getParreplacedionId();
        if (parreplacedionId != null) {
            future = kafkaTemplate.send(topic, parreplacedionId.toString(), event.getResult());
        } else {
            logger.warn("Send {} to {} without key", event, topic);
            future = kafkaTemplate.send(topic, event.getResult());
        }
        ListenableFutureCallback<SendResult<String, Object>> callback = new ListenableFutureCallback<SendResult<String, Object>>() {

            @Override
            public void onSuccess(final SendResult<String, Object> message) {
                ackSuccess(Lists.newArrayList(wrapper));
                logger.info("sent {} with offset {} ", event, message.getRecordMetadata().offset());
            }

            @Override
            public void onFailure(final Throwable throwable) {
                SyncerHealth.consumer(consumerId, identifier, Health.red(throwable.getMessage()));
                retryFailed(Lists.newArrayList(wrapper), throwable);
                logger.error("unable to send {} ", event, throwable);
            }
        };
        future.addCallback(callback);
    // no need to wait future, the order between batch is ensured by kafka client
    }

    @Override
    public String des() {
        return "KafkaChannel{" + "kafkaTemplate=" + kafkaTemplate + ", ack=" + ack + ", request=" + request + '}';
    }

    @Override
    public void close() {
        kafkaTemplate.flush();
    }

    @Override
    public String id() {
        return identifier;
    }
}

19 Source : SQLMapper.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced SQLMapper implements Mapper<SyncData, String> {

    // TODO 2019-11-23 ignore dup is right?
    static final String INSERT_INTO_VALUES = "insert into `?0`.`?1` (?2) values (?3) ON DUPLICATE KEY UPDATE ?4=?5";

    static final String DELETE_FROM_WHERE_ID = "delete from `?0`.`?1` where id = ?2";

    static final String UPDATE_SET_WHERE_ID = "update `?0`.`?1` set ?3 where id = ?2";

    static final String UPDATE_SET_WHERE = "update `?0`.`?1` set ?3 where ?2";

    private final Logger logger = LoggerFactory.getLogger(SQLMapper.clreplaced);

    private final KVMapper kvMapper;

    private final Expression schema;

    private final Expression table;

    private final Expression id;

    private final String idName;

    public SQLMapper(RowMapping rowMapping) {
        SpelExpressionParser parser = new SpelExpressionParser();
        schema = parser.parseExpression(rowMapping.getSchema());
        table = parser.parseExpression(rowMapping.getTable());
        id = parser.parseExpression(rowMapping.getId());
        idName = rowMapping.getId();
        kvMapper = new KVMapper(rowMapping.getRows());
    }

    @Override
    public String map(SyncData data) {
        StandardEvaluationContext context = data.getContext();
        String schema = evalString(this.schema, context);
        String table = evalString(this.table, context);
        String id = evalString(this.id, context);
        HashMap<String, Object> map = kvMapper.map(data);
        switch(data.getType()) {
            case WRITE:
                // add id for MySQL output, make it idempotent.
                map.put(idName, id);
                String[] entry = join(map, WRITE);
                return ParameterReplace.orderedParam(INSERT_INTO_VALUES, schema, table, entry[0], entry[1], idName, idName);
            case DELETE:
                return ParameterReplace.orderedParam(DELETE_FROM_WHERE_ID, schema, table, id);
            case UPDATE:
                if (id != null) {
                    return ParameterReplace.orderedParam(UPDATE_SET_WHERE_ID, schema, table, id, join(map, UPDATE)[0]);
                } else {
                    HashMap<String, Object> syncBy = data.getSyncBy();
                    if (syncBy == null) {
                        throw new InvalidSyncDataException("Ignore invalid SyncData: update row without [id] and/or [syncByQuery].", data);
                    }
                    String filterCondition = join(syncBy, UPDATE)[0];
                    return ParameterReplace.orderedParam(UPDATE_SET_WHERE, schema, table, filterCondition, join(map, data.getType())[0]);
                }
            default:
                throw new IllegalArgumentException("Unsupported row event type: " + data);
        }
    }

    private String[] join(HashMap<String, Object> map, SimpleEventType type) {
        switch(type) {
            case WRITE:
                StringJoiner keys = new StringJoiner(",");
                StringJoiner values = new StringJoiner(",");
                for (Entry<String, Object> entry : map.entrySet()) {
                    keys.add(SQLHelper.wrapCol(entry.getKey()));
                    values.add(SQLHelper.inSQL(entry.getValue()));
                }
                return new String[] { keys.toString(), values.toString() };
            case UPDATE:
                StringJoiner kv = new StringJoiner(",");
                for (Entry<String, Object> entry : map.entrySet()) {
                    String condition = "" + SQLHelper.wrapCol(entry.getKey()) + "=" + SQLHelper.inSQL(entry.getValue());
                    kv.add(condition);
                }
                return new String[] { kv.toString() };
            default:
                throw new IllegalArgumentException("Unsupported row event type: " + type);
        }
    }

    String evalString(Expression expr, StandardEvaluationContext context) {
        return expr.getValue(context, String.clreplaced);
    }
}

19 Source : NestedSQLMapper.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced NestedSQLMapper extends SQLMapper {

    /**
     * <a href="https://www.w3schools.com/sql/sql_insert_into_select.asp">insert into select</a> the
     * `?3` is a select clause
     */
    private static final String INSERT_INTO_SELECT = "insert into `?0`.`?1` (?2) (?3)";

    private final Logger logger = LoggerFactory.getLogger(NestedSQLMapper.clreplaced);

    private final KVMapper kvMapper;

    private final Expression schema;

    private final Expression table;

    private final JdbcNestedQueryMapper jdbcNestedQueryMapper;

    public NestedSQLMapper(RowMapping rowMapping, JdbcTemplate jdbcTemplate) {
        super(rowMapping);
        SpelExpressionParser parser = new SpelExpressionParser();
        schema = parser.parseExpression(rowMapping.getSchema());
        table = parser.parseExpression(rowMapping.getTable());
        jdbcNestedQueryMapper = new JdbcNestedQueryMapper();
        kvMapper = new KVMapper(rowMapping.getRows());
    }

    private String[] join(HashMap<String, Object> map) {
        StringJoiner keys = new StringJoiner(",");
        ParameterizedString parameterizedString = null;
        HashMap<String, String> tmp = new HashMap<>();
        for (Entry<String, Object> entry : map.entrySet()) {
            keys.add(SQLHelper.wrapCol(entry.getKey()));
            Object value = entry.getValue();
            if (value instanceof ParameterizedString && value != parameterizedString) {
                // TODO 18/3/8 multiple query, change to select .. a join b
                // JdbcNestedQueryMapper & ParameterizedString
                parameterizedString = (ParameterizedString) value;
            } else {
                tmp.put(entry.getKey(), SQLHelper.inSQL(value));
            }
        }
        replacedert.notNull(parameterizedString, "[Impossible to be null]");
        parameterizedString.nameToAlias(tmp);
        return new String[] { keys.toString(), parameterizedString.getSql() };
    }

    @Override
    public String map(SyncData data) {
        if (!data.hasExtra()) {
            return super.map(data);
        }
        jdbcNestedQueryMapper.parseExtraQueryContext(data.getExtraQueryContext());
        StandardEvaluationContext context = data.getContext();
        String schema = evalString(this.schema, context);
        String table = evalString(this.table, context);
        HashMap<String, Object> map = kvMapper.map(data);
        logger.debug("Convert SyncData to {}", map);
        switch(data.getType()) {
            case WRITE:
                String[] entry = join(map);
                return ParameterReplace.orderedParam(INSERT_INTO_SELECT, schema, table, entry[0], entry[1]);
            default:
                throw new IllegalArgumentException("Unsupported row event type: " + data);
        }
    }
}

19 Source : MysqlChannel.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced MysqlChannel implements BufferedChannel<String> {

    private final Logger logger = LoggerFactory.getLogger(MysqlChannel.clreplaced);

    private final BatchBuffer<SyncWrapper<String>> batchBuffer;

    private final PipelineBatchConfig batch;

    private final JdbcTemplate jdbcTemplate;

    private final SQLMapper sqlMapper;

    private final Ack ack;

    private final FailureLog<SyncWrapper<String>> sqlFailureLog;

    private final String output;

    private final String consumerId;

    private final AtomicBoolean closed = new AtomicBoolean(false);

    private final ExecutorService mysqlService;

    public MysqlChannel(Mysql mysql, SyncerOutputMeta outputMeta, Ack ack) {
        MysqlConnection connection = mysql.getConnection();
        jdbcTemplate = new JdbcTemplate(connection.dataSource());
        batchBuffer = new BatchBuffer<>(mysql.getBatch());
        sqlMapper = new NestedSQLMapper(mysql.getRowMapping(), jdbcTemplate);
        this.batch = mysql.getBatch();
        this.ack = ack;
        FailureLogConfig failureLog = mysql.getFailureLog();
        sqlFailureLog = FailureLog.getLogger(Paths.get(outputMeta.getFailureLogDir(), connection.connectionIdentifier()), failureLog, new TypeToken<FailureEntry<SyncWrapper<String>>>() {
        });
        output = connection.connectionIdentifier();
        consumerId = mysql.getConsumerId();
        mysqlService = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(10), new NamedThreadFactory("syncer-" + connection.connectionIdentifier() + "-output-mysql"), new ThreadPoolExecutor.DiscardPolicy());
    }

    @Override
    public boolean output(SyncData event) throws InterruptedException {
        if (closed.get()) {
            return false;
        }
        String sql = sqlMapper.map(event);
        boolean add = batchBuffer.add(new SyncWrapper<>(event, sql));
        logger.debug("Add {}, {}", sql, add);
        mysqlService.submit(() -> {
            try {
                BufferedChannel.super.flushAndSetFlushDone(true);
            } catch (InterruptedException e) {
                logger.warn("[Shutting down] Filter job interrupted");
            }
        });
        return add;
    }

    @Override
    public String des() {
        return "MysqlChannel{" + "jdbcTemplate=" + jdbcTemplate + '}';
    }

    @Override
    public void close() {
        if (!closed.compareAndSet(false, true)) {
            return;
        }
        BufferedChannel.super.close();
    }

    @Override
    public String id() {
        return output;
    }

    @Override
    public long getDelay() {
        return batch.getDelay();
    }

    @Override
    public TimeUnit getDelayUnit() {
        return batch.getDelayTimeUnit();
    }

    @Override
    public boolean flush() throws InterruptedException {
        List<SyncWrapper<String>> sqls = batchBuffer.flush();
        batchAndRetry(sqls);
        return sqls != null;
    }

    private void batchAndRetry(List<SyncWrapper<String>> sqls) throws InterruptedException {
        if (sqls == null) {
            return;
        }
        String[] sqlStatement = sqls.stream().map(SyncWrapper::getData).toArray(String[]::new);
        logger.info("Flush batch({})", sqls.size());
        if (logger.isDebugEnabled()) {
            logger.debug("Sending {}", Arrays.toString(sqlStatement));
        }
        long sleepInSecond = 1;
        while (!Thread.currentThread().isInterrupted()) {
            try {
                jdbcTemplate.batchUpdate(sqlStatement);
                ackSuccess(sqls);
                return;
            } catch (CannotGetJdbcConnectionException e) {
                String error = "Fail to connect to DB, will retry in {} second(s)";
                logger.error(error, sleepInSecond, e);
                SyncerHealth.consumer(consumerId, output, Health.red(error));
                sleepInSecond = FallBackPolicy.POW_2.sleep(sleepInSecond);
            } catch (DataAccessException e) {
                retryFailed(sqls, e);
                return;
            }
        }
    }

    @Override
    public boolean flushIfReachSizeLimit() throws InterruptedException {
        List<SyncWrapper<String>> sqls = batchBuffer.flushIfReachSizeLimit();
        batchAndRetry(sqls);
        return sqls != null;
    }

    @Override
    public void setFlushDone() {
        batchBuffer.flushDone();
    }

    @Override
    public void ackSuccess(List<SyncWrapper<String>> aim) {
        for (SyncWrapper wrapper : aim) {
            ack.remove(wrapper.getSourceId(), wrapper.getSyncDataId());
        }
    }

    @Override
    public void retryFailed(List<SyncWrapper<String>> sqls, Throwable e) {
        Throwable cause = e.getCause();
        /*
      After a command in a batch update fails to execute properly and a BatchUpdateException is thrown,
      the driver may or may not continue to process the remaining commands in the batch.
      If the driver continues processing after a failure,
      the array returned by the method BatchUpdateException.getUpdateCounts will have an element for every command in the batch
      rather than only elements for the commands that executed successfully before the error.
      In the case where the driver continues processing commands,
      the array element for any command that failed is Statement.EXECUTE_FAILED.
     */
        /*
      Mysql with `rewriteBatchedStatements=true` not continue processing and return [-1, Statement.EXECUTE_FAILED, ...] in case of BatchUpdateException
     */
        if (!(cause instanceof BatchUpdateException)) {
            logger.error("Unknown exception", e);
            throw new IllegalStateException();
        }
        LinkedList<SyncWrapper<String>> tmp = new LinkedList<>();
        int[] updateCounts = ((BatchUpdateException) cause).getUpdateCounts();
        for (int i = 0; i < updateCounts.length; i++) {
            SyncWrapper<String> stringSyncWrapper = sqls.get(i);
            if (succ(updateCounts[i])) {
                ack.remove(stringSyncWrapper.getSourceId(), stringSyncWrapper.getSyncDataId());
                continue;
            }
            // TODO 2019-10-28 maybe diff error reason!!!
            ErrorLevel level = level(e, stringSyncWrapper, batch.getMaxRetry());
            if (level.retriable()) {
                tmp.add(stringSyncWrapper);
                continue;
            } else {
                switch(level) {
                    case MAX_TRY_EXCEED:
                    case // count as failure then write a log, so no break
                    SYNCER_BUG:
                        sqlFailureLog.log(stringSyncWrapper, cause.getMessage());
                        logger.error("Met {} in {}", level, stringSyncWrapper, cause);
                        break;
                    case // not count WARN as failure item
                    WARN:
                        logger.error("Met [{}] in {}", cause.getMessage(), stringSyncWrapper);
                        break;
                }
            }
            ack.remove(stringSyncWrapper.getSourceId(), stringSyncWrapper.getSyncDataId());
        }
        batchBuffer.addAllInHead(tmp);
    }

    private boolean succ(long updateCount) {
        // TODO 2019-10-28 -1?
        return updateCount != Statement.EXECUTE_FAILED;
    }

    @Override
    public ErrorLevel level(Throwable e, SyncWrapper wrapper, int maxTry) {
        /*
     * Possible reasons for DuplicateKey
     * 1. the first failed, the second succeed. Then restart, then the second will send again and cause this
     * 2. duplicate entry in binlog file: load data into db multiple time
     * 3. the data is sync to mysql but not receive response before syncer shutdown
     */
        if (e instanceof DuplicateKeyException) {
            return ErrorLevel.WARN;
        }
        if (e instanceof BadSqlGrammarException) {
            return ErrorLevel.SYNCER_BUG;
        }
        return BufferedChannel.super.level(e, wrapper, maxTry);
    }

    @Override
    public boolean checkpoint() {
        return ack.flush();
    }
}

19 Source : HttpChannel.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * Should be thread safe
 *
 * @author zzt
 */
public clreplaced HttpChannel implements OutputChannel {

    private static final Gson gson = new Gson();

    private static final int MAX_TRY = 5;

    private final Logger logger = LoggerFactory.getLogger(HttpChannel.clreplaced);

    private final String connection;

    private final KVMapper mapper;

    private final Ack ack;

    private final String id;

    private final NettyHttpClient httpClient;

    private Expression pathExpr;

    public HttpChannel(Http http, Map<String, Object> jsonMapper, Ack ack) {
        HttpConnection connection = http.getConnection();
        if (http.getPath() != null) {
            SpelExpressionParser parser = new SpelExpressionParser();
            pathExpr = parser.parseExpression(http.getPath());
        }
        this.ack = ack;
        this.connection = connection.toConnectionUrl(null);
        httpClient = new NettyHttpClient(connection, new HttpClientInitializer());
        id = connection.connectionIdentifier();
        this.mapper = new KVMapper(jsonMapper);
    }

    /**
     * <a href="https://stackoverflow.com/questions/22989500/is-resttemplate-thread-safe">
     * RestTemplate is thread safe</a>
     *
     * @param event the data from filter module
     */
    @Override
    public boolean output(SyncData event) throws InterruptedException {
        // TODO 17/9/22 add batch worker
        HashMap<String, Object> map = mapper.map(event);
        String path = pathExpr.getValue(event.getContext(), String.clreplaced);
        boolean res = false;
        int count = 0;
        while (!res && count < MAX_TRY) {
            switch(event.getType()) {
                case UPDATE:
                    res = execute(POST, path, map);
                    break;
                case DELETE:
                    res = execute(DELETE, path, map);
                    break;
                case WRITE:
                    res = execute(PUT, path, map);
                    break;
                default:
                    logger.error("Unsupported event type: {}", event);
                    return false;
            }
            if (res) {
                ack.remove(event.getSourceIdentifier(), event.getDataId());
            } else {
                count++;
                if (count == MAX_TRY) {
                    logger.error("Fail to send {}", event);
                }
            }
        }
        return false;
    }

    private boolean execute(HttpMethod method, String path, HashMap<String, Object> content) throws InterruptedException {
        return httpClient.write(method, path, gson.toJson(content));
    }

    @Override
    public String des() {
        return "HttpChannel{" + "connection='" + connection + '\'' + '}';
    }

    @Override
    public void close() {
        httpClient.close();
    }

    @Override
    public String id() {
        return id;
    }
}

19 Source : ESRequestMapper.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 *
 * https://www.elastic.co/guide/en/elasticsearch/reference/5.4/painless-api-reference.html
 * https://www.elastic.co/guide/en/elasticsearch/painless/7.5/painless-api-reference-shared-java-util.html#painless-api-reference-shared-ArrayList
 */
public clreplaced ESRequestMapper implements Mapper<SyncData, Object> {

    private final Logger logger = LoggerFactory.getLogger(ElasticsearchChannel.clreplaced);

    private final ESRequestMapping esRequestMapping;

    // TODO use es7 branch to delete this dependency
    private final AbstractClient client;

    private final KVMapper requestBodyMapper;

    private final Expression indexExpr;

    private final Expression typeExpr;

    private final ESQueryMapper esQueryMapper;

    ESRequestMapper(AbstractClient client, ESRequestMapping esRequestMapping) {
        this.esRequestMapping = esRequestMapping;
        this.client = client;
        SpelExpressionParser parser = new SpelExpressionParser();
        indexExpr = parser.parseExpression(esRequestMapping.getIndex());
        typeExpr = parser.parseExpression(esRequestMapping.getType());
        esQueryMapper = new ESQueryMapper(client);
        requestBodyMapper = new KVMapper(esRequestMapping.getFieldsMapping());
    }

    @ThreadSafe(safe = { SpelExpressionParser.clreplaced, ESRequestMapping.clreplaced, TransportClient.clreplaced })
    @Override
    public Object map(SyncData data) {
        esQueryMapper.parseExtraQueryContext(data.getExtraQueryContext());
        StandardEvaluationContext context = data.getContext();
        String index = eval(indexExpr, context);
        String type = eval(typeExpr, context);
        String id = data.getId() == null ? null : data.getId().toString();
        switch(data.getType()) {
            case WRITE:
                if (esRequestMapping.getNoUseIdForIndex()) {
                    return client.prepareIndex(index, type).setSource(requestBodyMapper.map(data));
                }
                return client.prepareIndex(index, type, id).setSource(requestBodyMapper.map(data));
            case DELETE:
                logger.info("Deleting doc from Elasticsearch, may affect performance");
                if (id != null) {
                    return client.prepareDelete(index, type, id);
                }
                logger.warn("Deleting doc by query, may affect performance");
                return DeleteByQueryAction.INSTANCE.newRequestBuilder(client).source(index).filter(getFilter(data));
            case UPDATE:
                // Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html
                if (id != null) {
                    // update doc with `id`
                    if (needScript(data)) {
                        // scripted updates: update using script
                        HashMap<String, Object> map = requestBodyMapper.map(data);
                        UpdateRequestBuilder builder = client.prepareUpdate(index, type, id).setScript(getScript(data, map)).setRetryOnConflict(esRequestMapping.getRetryOnUpdateConflict());
                        if (esRequestMapping.isUpsert()) {
                            // scripted_upsert
                            builder.setUpsert(getUpsert(data)).setScriptedUpsert(true);
                        }
                        return builder;
                    } else {
                        // update with partial doc
                        return client.prepareUpdate(index, type, id).setDoc(requestBodyMapper.map(data)).setDocAsUpsert(// doc_as_upsert
                        esRequestMapping.isUpsert()).setRetryOnConflict(esRequestMapping.getRetryOnUpdateConflict());
                    }
                } else {
                    // update doc by `query`
                    logger.warn("Updating doc by query, may affect performance");
                    return UpdateByQueryAction.INSTANCE.newRequestBuilder(client).source(index).filter(getFilter(data)).script(getScript(data, data.getFields()));
                }
            default:
                throw new IllegalArgumentException("Unsupported row event type: " + data);
        }
    }

    private static boolean needScript(SyncData data) {
        ESScriptUpdate esScriptUpdate = data.getEsScriptUpdate();
        return esScriptUpdate != null && esScriptUpdate.needScript();
    }

    private String eval(Expression expr, StandardEvaluationContext context) {
        return expr.getValue(context, String.clreplaced);
    }

    /**
     * https://www.elastic.co/guide/en/elasticsearch/reference/5.4/painless-api-reference.html
     */
    private Script getScript(SyncData data, HashMap<String, Object> toSet) {
        HashMap<String, Object> params = new HashMap<>();
        StringBuilder code = new StringBuilder();
        ESScriptUpdate.makeScript(code, " = params.", ";", toSet, params);
        if (needScript(data)) {
            ESScriptUpdate esScriptUpdate = data.getEsScriptUpdate();
            esScriptUpdate.generateMergeScript(code, params);
        }
        return new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, code.toString(), params);
    }

    private QueryBuilder getFilter(SyncData data) {
        BoolQueryBuilder builder = boolQuery();
        HashMap<String, Object> syncBy = data.getSyncBy();
        if (syncBy.isEmpty()) {
            throw new InvalidConfigException("No data used to do sync(update/delete) filter");
        }
        for (Entry<String, Object> entry : syncBy.entrySet()) {
            String[] key = entry.getKey().split("\\.");
            if (key.length == 2) {
                builder.filter(nestedQuery(key[0], boolQuery().filter(getSingleFilter(entry)), ScoreMode.Avg));
            } else if (key.length == 1) {
                builder.filter(getSingleFilter(entry));
            } else {
                logger.error("Only support one level nested obj for the time being");
            }
        }
        return builder;
    }

    private QueryBuilder getSingleFilter(Entry<String, Object> entry) {
        Object value = entry.getValue();
        if (value instanceof Collection || value.getClreplaced().isArray()) {
            return termsQuery(entry.getKey(), value);
        }
        return termQuery(entry.getKey(), value);
    }

    static Map getUpsert(SyncData data) {
        replacedert needScript(data);
        HashMap<String, Object> upsert = new HashMap<>();
        ESScriptUpdate esScriptUpdate = data.getEsScriptUpdate();
        esScriptUpdate.upsert(upsert);
        return upsert;
    }
}

19 Source : ESQueryMapper.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced ESQueryMapper implements ExtraQueryMapper {

    private final AbstractClient client;

    private final Logger logger = LoggerFactory.getLogger(ESQueryMapper.clreplaced);

    public ESQueryMapper(AbstractClient client) {
        this.client = client;
    }

    @Override
    public Map<String, Object> map(ExtraQuery extraQuery) {
        String[] select = extraQuery.getSelect();
        SearchResponse response;
        try {
            response = client.prepareSearch(extraQuery.getIndexName()).setTypes(extraQuery.getTypeName()).setSearchType(SearchType.DEFAULT).setFetchSource(select, null).setQuery(getFilter(extraQuery)).execute().actionGet();
        } catch (Exception e) {
            logger.error("Fail to do the extra query {}", extraQuery, e);
            return Collections.emptyMap();
        }
        SearchHits hits = response.getHits();
        if (hits.totalHits > 1) {
            // todo toList
            logger.warn("Multiple query results exists, only use the first");
        } else if (hits.totalHits == 0) {
            logger.warn("Fail to find any match by {}", extraQuery);
            return Collections.emptyMap();
        }
        Map<String, Object> hit = hits.getAt(0).getSource();
        Map<String, Object> res = new HashMap<>();
        for (int i = 0; i < select.length; i++) {
            Object value = hit.get(select[i]);
            if (value == null) {
                logger.warn("No {} in {} by {}", select[i], hit, extraQuery);
            }
            res.put(extraQuery.getAs(i), value);
        }
        extraQuery.addQueryResult(res);
        return res;
    }

    private QueryBuilder getFilter(ExtraQuery extraQuery) {
        BoolQueryBuilder bool = new BoolQueryBuilder();
        for (Entry<String, Object> e : extraQuery.getQueryBy().entrySet()) {
            Object value = e.getValue();
            String key = e.getKey();
            Optional<Object> queryResult = getPlaceholderQueryResult(extraQuery, value);
            if (queryResult.isPresent()) {
                extraQuery.filter(key, queryResult.get());
                bool.filter(QueryBuilders.termQuery(key, queryResult.get()));
            } else {
                bool.filter(QueryBuilders.termQuery(key, value));
            }
        }
        return bool;
    }

    private Optional<Object> getPlaceholderQueryResult(ExtraQuery extraQuery, Object value) {
        if (value instanceof String) {
            String str = ((String) value);
            Optional<String> placeholderValue = getPlaceholderValue(str);
            if (placeholderValue.isPresent()) {
                String key = placeholderValue.get();
                Object record = extraQuery.getField(key);
                if (!(record instanceof ExtraQuery) || ((ExtraQuery) record).getQueryResult(key) == null) {
                    logger.error("Dependent extra query {} has no result of {} from {}", record, key, value);
                    return Optional.empty();
                } else {
                    return Optional.of(((ExtraQuery) record).getQueryResult(key));
                }
            }
        }
        return Optional.empty();
    }

    /**
     * @param str special placeholder: $userId$
     * means this key is the result of previous query
     */
    private Optional<String> getPlaceholderValue(String str) {
        if (str.startsWith("$") && str.endsWith("$")) {
            return Optional.of(str.substring(1, str.length() - 1));
        }
        return Optional.empty();
    }
}

19 Source : BatchJob.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced BatchJob implements EventLoop {

    private static final Logger logger = LoggerFactory.getLogger(BatchJob.clreplaced);

    private final BufferedChannel bufferedChannel;

    private final String consumerId;

    public BatchJob(String consumerId, BufferedChannel bufferedChannel) {
        this.bufferedChannel = bufferedChannel;
        this.consumerId = consumerId;
    }

    @Override
    public void loop() {
        try {
            bufferedChannel.flushAndSetFlushDone(false);
        } catch (InterruptedException e) {
            logger.warn("Batch job interrupted");
            throw new ShutDownException(e);
        } catch (FailureException e) {
            String err = FailureException.getErr(bufferedChannel, consumerId);
            logger.error(err, e);
            SyncerHealth.consumer(consumerId, bufferedChannel.id(), Health.red(err));
        }
    }
}

19 Source : BatchBuffer.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * <ul>
 *   <li>Filter thread should not share one buffer, otherwise may cause disorder: e.g.
 *   <pre>
 *    hh:ss.000: [thread2] add to buffer: insert (2, b)
 *    hh:ss.001: [thread1] add to buffer: insert (1, a)
 *    hh:ss.002: [thread1] flushIfReachSizeLimit(buffer size 2): thread1 insert (1, a)(2, b)
 *    hh:ss.003: [thread2] add to buffer: insert (4, c), update (2, d)
 *    hh:ss.004: [thread2] flushIfReachSizeLimit(buffer size 2): thread2 insert (4, c); update (2, d)
 *    // update may cause DoreplacedentMissing, should after thread1 return
 *   </pre>
 *   </li>
 *   <li>Flush by size and time should not invoked at same time, otherwise may cause disorder: e.g.
 *   <pre>
 *    hh:ss.000: [thread1] add to buffer: insert (1, a)(2, b)
 *    hh:ss.000: [thread1] flushIfReachSizeLimit(buffer size 2): thread1 insert (1, a)(2, b)
 *    hh:ss.001: [thread2] add to buffer: update (2, d)
 *    hh:ss.002: [thread2] flush(time reach): thread2 update (2, d)
 *   </pre>
 *   </li>
 * </ul>
 *
 * <h3>Solution</h3>
 *
 * <ul>
 *   <li>Every filter thread should have one buffer; Or, following option 3</li>
 *   <li>A flush timer thread flush all filter buffer</li>
 *   <li>A buffer have a `flushing` flag (set when `flushing == false` in flushXXX, unSet when finish remote request),
 *   a flushing buffer refuse other flush request</li>
 * </ul>
 *
 *  @author zzt
 */
public clreplaced BatchBuffer<T extends Retryable> {

    private final Logger logger = LoggerFactory.getLogger(BatchBuffer.clreplaced);

    private AtomicBoolean flushing = new AtomicBoolean(false);

    private final int limit;

    /**
     * <h3>Equation</h3>
     * deque.size() >= estimateSize
     * Notice: deque.size() may not accurate
     */
    private final ConcurrentLinkedDeque<T> deque = new ConcurrentLinkedDeque<>();

    private final AtomicInteger estimateSize = new AtomicInteger(0);

    public BatchBuffer(PipelineBatchConfig batch) {
        limit = batch.getSize() <= 0 ? Integer.MAX_VALUE : batch.getSize();
    }

    public boolean add(T data) {
        data.inc();
        deque.addLast(data);
        estimateSize.incrementAndGet();
        return true;
    }

    public boolean addAllInHead(List<T> data) {
        for (T datum : data) {
            datum.inc();
            deque.addFirst(datum);
        }
        estimateSize.addAndGet(data.size());
        return true;
    }

    @ThreadSafe(safe = { ConcurrentLinkedDeque.clreplaced, AtomicInteger.clreplaced })
    public List<T> flushIfReachSizeLimit() {
        // The function should be side-effect-free, since it may be
        // re-applied when attempted updates fail due to contention among threads
        logger.debug("{}, {}, {}", estimateSize, deque.size(), flushing);
        if (estimateSize.get() >= limit && flushing.compareAndSet(false, true)) {
            estimateSize.addAndGet(-limit);
            return flushContent(limit);
        }
        return null;
    }

    public List<T> flush() {
        int size;
        logger.debug("{}, {}, {}", estimateSize, deque.size(), flushing);
        if ((size = estimateSize.get()) > 0 && flushing.compareAndSet(false, true)) {
            int toFlush = Math.min(size, limit);
            estimateSize.addAndGet(-toFlush);
            return flushContent(toFlush);
        }
        return null;
    }

    private List<T> flushContent(int size) {
        Preconditions.checkArgument(flushing.get(), "Flushing");
        logger.debug("{}, {}, Flushing", estimateSize, deque.size() - size);
        ArrayList<T> res = new ArrayList<>(size);
        for (int i = 0; i < size; i++) {
            try {
                res.add(deque.removeFirst());
            } catch (NoSuchElementException e) {
                logger.error("Syncer Bug: {}, {}, {}", estimateSize, deque.size(), flushing, e);
                throw new IllegalStateException();
            }
        }
        return res;
    }

    public void flushDone() {
        flushing.set(false);
    }
}

19 Source : LocalConsumerSource.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public abstract clreplaced LocalConsumerSource implements ConsumerSource {

    private static final Logger logger = LoggerFactory.getLogger(LocalConsumerSource.clreplaced);

    private final BlockingQueue<SyncData> toFilter;

    private final Set<Repo> repos;

    private final Connection connection;

    private final SyncInitMeta syncInitMeta;

    private final String clientId;

    private final String connectionIdentifier;

    private final Ack ack;

    private boolean isSent = true;

    public LocalConsumerSource(String clientId, Connection connection, Set<Repo> repos, SyncInitMeta syncInitMeta, Ack ack, BlockingQueue<SyncData> toFilter) {
        this.repos = repos;
        this.connection = connection;
        this.syncInitMeta = syncInitMeta;
        this.clientId = clientId;
        this.ack = ack;
        this.toFilter = toFilter;
        connectionIdentifier = connection.connectionIdentifier();
    }

    @Override
    public Connection getRemoteConnection() {
        return connection;
    }

    @Override
    public abstract SyncInitMeta getSyncInitMeta();

    @Override
    public Set<Repo> copyRepos() {
        Set<Repo> res = new HashSet<>();
        for (Repo repo : repos) {
            res.add(new Repo(repo));
        }
        return res;
    }

    @Override
    public String clientId() {
        return clientId;
    }

    @Override
    public boolean input(SyncData data) {
        if (sent(data)) {
            logger.info("Consumer({}, {}) skip {} from {}", getSyncInitMeta(), clientId, data, connectionIdentifier);
            return false;
        }
        logger.debug("add single: data id: {}, {}, {}", data.getDataId(), data, data.hashCode());
        ack.append(connectionIdentifier, data.getDataId());
        return toFilter.add(data.setSourceIdentifier(connectionIdentifier));
    }

    @Override
    public boolean input(SyncData[] data) {
        boolean res = true;
        for (SyncData datum : data) {
            if (datum == null) {
                continue;
            }
            if (!sent(datum)) {
                ack.append(connectionIdentifier, datum.getDataId());
                logger.debug("Consumer({}, {}) receive: {}", getSyncInitMeta(), clientId, datum);
                // `toFilter.add` should be last call to avoid sharing SyncData in diff threads
                res = toFilter.add(datum.setSourceIdentifier(connectionIdentifier)) && res;
            } else {
                logger.info("Consumer({}, {}) skip {} from {}", getSyncInitMeta(), clientId, datum, connectionIdentifier);
            }
        }
        return res;
    }

    /**
     * Because {@link #input(SyncData)} is called only by one thread, so we use {@link #isSent} as a
     * simple boolean
     */
    @Override
    public boolean sent(SyncData data) {
        if (!isSent) {
            return false;
        }
        // remembered position is not synced in last run,
        // if syncInitMeta.compareTo(now) == 0, is not sent
        return isSent = getSyncInitMeta().compareTo(data.getDataId().getSyncInitMeta()) > 0;
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) {
            return true;
        }
        if (o == null || getClreplaced() != o.getClreplaced()) {
            return false;
        }
        LocalConsumerSource that = (LocalConsumerSource) o;
        return clientId.equals(that.clientId);
    }

    @Override
    public int hashCode() {
        return clientId.hashCode();
    }

    @Override
    public String toString() {
        return "LocalConsumerSource{" + "repos=" + repos + ", connection=" + connection + ", syncInitMeta=" + syncInitMeta + ", clientId='" + clientId + '\'' + '}';
    }
}

19 Source : Switch.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced Switch implements ConditionalStatement {

    private final static Gson gson = new Gson();

    private final Logger logger = LoggerFactory.getLogger(Switch.clreplaced);

    private final SwitchCondition switchCondition;

    private final Map<String, List<SyncFilter>> actionsMap;

    public Switch(SpelExpressionParser parser, Switcher filter) {
        switchCondition = new SwitchCondition(filter.getSwitch(), parser);
        Map<String, List<SyncFilter>> tmp = new HashMap<>();
        filter.getCase().forEach((k, v) -> {
            if (v == null) {
                throw new InvalidConfigException("Unsupport empty case: " + k);
            }
            tmp.put(k, v.stream().map(c -> gson.fromJson(gson.toJsonTree(c), FilterConfig.clreplaced).toFilter(parser)).collect(Collectors.toList()));
        });
        actionsMap = Collections.unmodifiableMap(tmp);
    }

    @Override
    public List<SyncFilter> conditional(SyncData syncData) {
        StandardEvaluationContext context = syncData.getContext();
        String conditionRes = switchCondition.execute(context);
        if (conditionRes == null) {
            logger.error("switch on `null`, skip {}", syncData);
            return null;
        }
        List<SyncFilter> caseClause = actionsMap.get(conditionRes);
        if (caseClause == null && actionsMap.containsKey(Switcher.DEFAULT)) {
            caseClause = actionsMap.get(Switcher.DEFAULT);
        }
        if (caseClause == null) {
            logger.error("Unknown switch result and no default config : {}", conditionRes);
            return null;
        }
        return caseClause;
    }
}

19 Source : JavaMethod.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

public clreplaced JavaMethod {

    private static final Logger logger = LoggerFactory.getLogger(JavaMethod.clreplaced);

    public static SyncFilter build(String consumerId, SyncerFilterMeta filterMeta, String method) {
        String source = // User write config using com.github.zzt93.syncer.data,
        // syncer run config using com.github.zzt93.syncer.common.data
        "import com.github.zzt93.syncer.data.*;\n" + "import com.github.zzt93.syncer.data.es.*;\n" + "import com.github.zzt93.syncer.data.util.*;\n" + "import java.util.*;\n" + "import java.math.BigDecimal;\n" + "import java.sql.Timestamp;\n" + "import org.slf4j.Logger;\n" + "import org.slf4j.LoggerFactory;\n" + "\n" + "public clreplaced MethodFilterTemplate implements SyncFilter<SyncData> {\n" + "\n" + "  private final Logger logger = LoggerFactory.getLogger(getClreplaced());\n" + "\n" + addNewline(method) + "\n" + "}\n";
        String clreplacedName = "Filter" + consumerId;
        source = source.replaceFirst("MethodFilterTemplate", clreplacedName);
        Path path = Paths.get(filterMeta.getSrc(), clreplacedName + ".java");
        try {
            Files.write(path, source.getBytes(StandardCharsets.UTF_8));
        } catch (IOException e) {
            logger.error("No permission", e);
        }
        compile(path.toString());
        Clreplaced<?> cls;
        try {
            URLClreplacedLoader clreplacedLoader = URLClreplacedLoader.newInstance(new URL[] { path.getParent().toUri().toURL() }, JavaMethod.clreplaced.getClreplacedLoader());
            cls = Clreplaced.forName(clreplacedName, true, clreplacedLoader);
            return (SyncFilter) cls.newInstance();
        } catch (ClreplacedNotFoundException | IllegalAccessException | InstantiationException | MalformedURLException e) {
            ShutDownCenter.initShutDown(e);
            return null;
        }
    }

    private static void compile(String path) {
        JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
        StandardJavaFileManager fm = compiler.getStandardFileManager(diagnostic -> logger.error("{}, {}", diagnostic.getLineNumber(), diagnostic.getSource().toUri()), null, null);
        try {
            fm.setLocation(StandardLocation.CLreplaced_PATH, Lists.newArrayList(new File(System.getProperty("java.clreplaced.path"))));
        } catch (IOException e) {
            logger.error("Fail to set location for compiler file manager", e);
        }
        if (compiler.run(null, null, null, path) != 0) {
            ShutDownCenter.initShutDown(new InvalidConfigException());
        }
    }

    private static String addNewline(String method) {
        char[] cs = method.toCharArray();
        StringBuilder sb = new StringBuilder(method.length() + 50);
        boolean inQuote = false;
        for (char c : cs) {
            sb.append(c);
            switch(c) {
                case '"':
                    inQuote = !inQuote;
                    break;
                case ';':
                case ':':
                case '{':
                case '}':
                    if (!inQuote)
                        sb.append('\n');
                    break;
            }
        }
        return sb.toString();
    }
}

19 Source : PositionFlusher.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced PositionFlusher implements EventLoop {

    private final Ack ack;

    private final Logger logger = LoggerFactory.getLogger(PositionFlusher.clreplaced);

    public PositionFlusher(Ack ack) {
        this.ack = ack;
    }

    @Override
    public void loop() {
        ack.flush();
    }
}

19 Source : Ack.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt TODO 18/1/17 change string identifier to Clreplaced?
 */
public clreplaced Ack {

    private static final Logger logger = LoggerFactory.getLogger(Ack.clreplaced);

    private final String metaDir;

    @ThreadSafe(sharedBy = { "main", "shutdown hook", "syncer-filter-output" }, des = "Thread start rule." + "main thread init ack before two other thread start")
    private Map<String, FileBasedMap<DataId>> ackMap = new HashMap<>();

    private final String consumerId;

    private final int outputSize;

    public static Ack build(ConsumerInitContext context, HashMap<String, SyncInitMeta> ackConnectionId2SyncInitMeta) {
        Set<MasterSource> masterSources = context.getInput().getMasterSet();
        String consumerId = context.getConsumerId();
        SyncerInputMeta syncerInputMeta = context.getSyncerInput().getInputMeta();
        Ack ack = new Ack(consumerId, syncerInputMeta, context.outputSize());
        for (MasterSource masterSource : masterSources) {
            Set<String> ids = masterSource.remoteIds();
            for (String id : ids) {
                SyncInitMeta initMeta = ack.addDatasource(id, masterSource.getType(), context);
                if (initMeta != null) {
                    ackConnectionId2SyncInitMeta.put(id, initMeta);
                }
            }
        }
        ack.ackMap = Collections.unmodifiableMap(ack.ackMap);
        return ack;
    }

    private Ack(String consumerId, SyncerInputMeta syncerInputMeta, int outputSize) {
        this.consumerId = consumerId;
        this.metaDir = syncerInputMeta.getLastRunMetadataDir();
        this.outputSize = outputSize;
    }

    private SyncInitMeta addDatasource(String identifier, MasterSourceType sourceType, ConsumerInitContext context) {
        Path path = Paths.get(metaDir, consumerId, identifier);
        FileBasedMap<DataId> fileBasedMap;
        if (context.hasEtcd()) {
            fileBasedMap = new FileBasedMap<>(path, context.getEtcd().setInputIdentifier(identifier));
        } else {
            fileBasedMap = new FileBasedMap<>(path);
        }
        ackMap.put(identifier, fileBasedMap);
        SyncInitMeta syncInitMeta = null;
        try {
            syncInitMeta = recoverSyncInitMeta(fileBasedMap, sourceType, syncInitMeta);
        } catch (IOException e) {
            logger.error("Impossible to run to here", e);
        }
        return syncInitMeta;
    }

    private SyncInitMeta recoverSyncInitMeta(FileBasedMap<DataId> fileBasedMap, MasterSourceType sourceType, SyncInitMeta syncInitMeta) throws IOException {
        AckMetaData bytes = fileBasedMap.readData();
        if (bytes.isEmpty()) {
            try {
                String data = bytes.toDataStr();
                switch(sourceType) {
                    case MySQL:
                        syncInitMeta = DataId.fromDataId(data);
                        break;
                    case Mongo:
                        syncInitMeta = DataId.fromMongoDataId(data);
                        break;
                    default:
                        throw new IllegalStateException("Not implemented type");
                }
            } catch (Exception e) {
                logger.warn("Meta file in {} crashed, take as fresh run", fileBasedMap);
            }
        } else {
            logger.warn("Meta file in {} crashed, take as fresh run", fileBasedMap);
        }
        return syncInitMeta;
    }

    /**
     * append `outputSize` at the beginning of consumer
     */
    public void append(String identifier, DataId dataId) {
        if (ackMap.get(identifier).append(dataId, outputSize)) {
            logger.debug("Append {} {} to ack log", identifier, dataId);
        } else {
            logger.error("Already append: {} {}", identifier, dataId);
        }
    }

    /**
     * remove one when ack is received from a output
     */
    public void remove(String identifier, DataId dataId) {
        boolean remove = false;
        try {
            remove = ackMap.get(identifier).remove(dataId, 1);
        } catch (Exception e) {
            logger.error("Fail to remove from ack log: {} {}", identifier, dataId, e);
        }
        if (remove) {
            logger.debug("Remove {} {} from ack log", identifier, dataId);
        }
    }

    public boolean flush() {
        // - add next dataId: seems hard to do
        // - set `lastRemoved` in FileBasedMap#remove
        boolean res = true;
        for (FileBasedMap<? super DataId> map : ackMap.values()) {
            res = map.flush() && res;
        }
        return res;
    }
}

19 Source : YamlEnvironmentPostProcessor.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced YamlEnvironmentPostProcessor {

    private static final Logger logger = LoggerFactory.getLogger(YamlEnvironmentPostProcessor.clreplaced);

    private static final String CONSUMER_CONFIG = "consumerConfig";

    private static final String PRODUCER_CONFIG = "producerConfig";

    private static final String CONFIG = "config";

    private static final String APPLICATION = "application.yml";

    public static SyncerApplication processEnvironment(HashMap<String, String> argKV) {
        String configFile = argKV.get(CONFIG);
        if (configFile == null) {
            configFile = "syncer.yml";
        }
        SyncerConfig syncerConfig = initConfig(configFile, SyncerConfig.clreplaced);
        String producer = argKV.get(PRODUCER_CONFIG);
        if (producer == null) {
            throw new IllegalArgumentException("No producer config file specified, try '--" + PRODUCER_CONFIG + "=producer.yml");
        }
        ProducerConfig producerConfig = initConfig(producer, ProducerConfig.clreplaced);
        Set<String> filePaths = getFilePaths(argKV.get(CONSUMER_CONFIG));
        ArrayList<ConsumerConfig> configs = new ArrayList<>();
        for (String fileName : filePaths) {
            configs.add(initConfig(fileName, ConsumerConfig.clreplaced));
        }
        String version = getVersion();
        return new SyncerApplication(producerConfig, syncerConfig, new LocalConsumerRegistry(), configs, new SyncerInfo(version));
    }

    private static String getVersion() {
        String str;
        try {
            str = FileUtil.readAll(FileUtil.getResource(APPLICATION).getInputStream());
        } catch (IOException e) {
            logger.error("Fail to load/parse {} file", APPLICATION, e);
            throw new InvalidConfigException(e);
        }
        Yaml yaml = new Yaml();
        Map map = yaml.loadAs(str, Map.clreplaced);
        return map.get("syncer.version").toString();
    }

    private static Set<String> getFilePaths(String pipelineNames) {
        if (pipelineNames == null) {
            throw new IllegalArgumentException("No consumer config file specified, try '--" + CONSUMER_CONFIG + "=sample.yml,sample2.yml'");
        }
        Set<String> res = new HashSet<>();
        for (String name : pipelineNames.split(",")) {
            Path path = Paths.get(name);
            if (Files.isDirectory(path)) {
                try (DirectoryStream<Path> paths = Files.newDirectoryStream(path, "*.{yml,yaml}")) {
                    paths.iterator().forEachRemaining(p -> res.add(p.toString()));
                } catch (IOException e) {
                    logger.error("Fail to travel {}", path, e);
                    throw new InvalidConfigException(e);
                }
            } else {
                res.add(name);
            }
        }
        return res;
    }

    private static <T> T initConfig(String fileName, Clreplaced<T> tClreplaced) {
        Resource path = FileUtil.getResource(fileName);
        Yaml yaml = getYaml(tClreplaced);
        try (InputStream in = path.getInputStream()) {
            String str = replaceEnv(in);
            return yaml.loadAs(str, tClreplaced);
        } catch (IOException e) {
            logger.error("Fail to load/parse {} as {}", fileName, tClreplaced, e);
            throw new InvalidConfigException(e);
        }
    }

    private static <T> Yaml getYaml(Clreplaced<T> tClreplaced) {
        Constructor c = new Constructor(tClreplaced);
        c.setPropertyUtils(new PropertyUtils() {

            @Override
            public Property getProperty(Clreplaced<?> type, String name) {
                if (name.indexOf('-') > -1) {
                    name = CaseFormat.LOWER_HYPHEN.to(CaseFormat.LOWER_CAMEL, name);
                }
                return super.getProperty(type, name);
            }
        });
        return new Yaml(c);
    }

    private static String replaceEnv(InputStream in) throws IOException {
        String str = FileUtil.readAll(in);
        Matcher matcher = RegexUtil.env().matcher(str);
        HashMap<String, String> rep = new HashMap<>();
        while (matcher.find()) {
            String group = matcher.group(1);
            String property = System.getenv(group);
            Preconditions.checkNotNull(property, "Fail to resolve env var: %s", group);
            rep.put(matcher.group(), property);
        }
        for (Entry<String, String> entry : rep.entrySet()) {
            str = str.replace(entry.getKey(), entry.getValue());
        }
        return str;
    }
}

19 Source : SyncerConfig.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
@Setter
@Getter
public clreplaced SyncerConfig {

    public static final String SERVER_PORT = "port";

    public static final String INSTANCE = "instance";

    public static final String DEFAULT_START = "40000";

    private static final Logger logger = LoggerFactory.getLogger(SyncerConfig.clreplaced);

    private static final String RETRY = "10";

    private int port;

    private SyncerAck ack;

    private SyncerInput input;

    private SyncerFilter filter;

    private SyncerOutput output;

    public void setPort(int port) {
        if (port <= 0 || port > 65535)
            throw new InvalidConfigException("Invalid port config " + port);
        this.port = port;
    }

    public EtcdConnection getEtcd() {
        return ack.getEtcd();
    }

    public boolean hasEtcd() {
        return ack.hasEtcd();
    }
}

19 Source : MongoConnection.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced MongoConnection extends Connection {

    private static final Logger logger = LoggerFactory.getLogger(MongoConnection.clreplaced);

    public MongoConnection(String addr, int port, String user, String preplaced) throws UnknownHostException {
        setAddress(addr);
        setPort(port);
        setUser(user);
        setPreplacedword(preplaced);
    }

    public MongoConnection(Connection connection) {
        try {
            setAddress(connection.getAddress());
        } catch (UnknownHostException ignored) {
            logger.error("Impossible", ignored);
        }
        setPort(connection.getPort());
        if (connection.getUser() == null) {
            return;
        }
        setUser(connection.getUser());
        setPreplacedword(connection.getPreplacedword());
        setPreplacedwordFile(connection.getPreplacedwordFile());
    }

    @Override
    public String toConnectionUrl(String path) {
        String s = super.toConnectionUrl(null);
        if (getUser() != null) {
            s = getUser() + ":" + getPreplacedword() + "@" + s + "/local?authSource=admin";
        }
        return "mongodb://" + s;
    }
}

19 Source : ElasticsearchConnection.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * Created by zzt on 9/11/17. <h3></h3>
 */
public clreplaced ElasticsearchConnection extends ClusterConnection {

    private final Logger logger = LoggerFactory.getLogger(ElasticsearchConnection.clreplaced);

    /**
     * @see org.elasticsearch.transport.TcpTransport#TCP_CONNECT_TIMEOUT
     */
    public AbstractClient esClient() throws Exception {
        // https://discuss.elastic.co/t/getting-availableprocessors-is-already-set-to-1-rejecting-1-illegalstateexception-exception/103082
        System.setProperty("es.set.netty.runtime.available.processors", "false");
        TransportClient client = new PreBuiltXPackTransportClient(settings());
        for (String clusterNode : getClusterNodes()) {
            String hostName = substringBeforeLast(clusterNode, COLON);
            String port = substringAfterLast(clusterNode, COLON);
            replacedert.hasText(hostName, "[replacedertion failed] missing host name in 'clusterNodes'");
            replacedert.hasText(port, "[replacedertion failed] missing port in 'clusterNodes'");
            logger.info("Adding transport node : {}, timeout in 30s", clusterNode);
            client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(hostName), Integer.valueOf(port)));
        }
        return client;
    }

    private Settings settings() {
        Builder builder = Settings.builder().put("cluster.name", getClusterName());
        if (getUser() == null && noPreplacedword()) {
            return builder.build();
        }
        if (getUser() == null || getPreplacedword() == null) {
            throw new IllegalArgumentException("Lacking user or preplacedword");
        }
        return builder.put("xpack.security.user", getUser() + COLON + getPreplacedword()).build();
    }

    @Override
    public boolean valid() {
        return !StringUtils.isEmpty(getClusterName()) && !getClusterNodes().isEmpty();
    }
}

19 Source : Connection.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced Connection implements Comparable<Connection> {

    private static final Logger logger = LoggerFactory.getLogger(Connection.clreplaced);

    private static final String COMMON = ":";

    private String address;

    private int port;

    private String user;

    private String preplacedwordFile;

    private String preplacedword;

    /**
     * identifier for this connection, used to distinguish other connection
     */
    private volatile String identifier;

    /**
     * utility field to replace address
     */
    private String ip;

    /**
     * special field for input module
     * @see com.github.zzt93.syncer.config.ConsumerConfig
     * @see MasterSource
     */
    private SyncMeta syncMeta;

    public Connection() {
    }

    public Connection(String address, int port, String user, String preplacedwordFile, String preplacedword, SyncMeta syncMeta) {
        try {
            setAddress(address);
        } catch (UnknownHostException e) {
            logger.error("Unknown host", e);
            throw new InvalidConfigException(e);
        }
        this.port = port;
        this.user = user;
        setPreplacedword(preplacedword);
        setPreplacedwordFile(preplacedwordFile);
        this.syncMeta = syncMeta;
    }

    public Connection(Connection connection) {
        address = connection.address;
        port = connection.port;
        user = connection.user;
        preplacedwordFile = connection.preplacedwordFile;
        preplacedword = connection.preplacedword;
        identifier = connection.identifier;
        ip = connection.ip;
        syncMeta = connection.syncMeta;
    }

    public SyncMeta[] getSyncMetas() {
        return new SyncMeta[] { syncMeta };
    }

    public void setSyncMeta(SyncMeta syncMeta) {
        this.syncMeta = syncMeta;
    }

    public String getAddress() {
        return address;
    }

    public void setAddress(String address) throws UnknownHostException {
        this.address = address;
        ip = NetworkUtil.toIp(getAddress());
    }

    public int getPort() {
        return port;
    }

    public void setPort(int port) {
        this.port = port;
    }

    public String getUser() {
        return user;
    }

    public void setUser(String user) {
        this.user = user;
    }

    public String getPreplacedwordFile() {
        return preplacedwordFile;
    }

    public void setPreplacedwordFile(String preplacedwordFile) {
        if (preplacedwordFile == null) {
            return;
        }
        this.preplacedwordFile = preplacedwordFile;
        try {
            List<String> lines = FileUtil.readLine(preplacedwordFile);
            if (lines.size() != 1) {
                throw new InvalidConfigException("Multiple line preplacedword in " + preplacedwordFile);
            }
            this.preplacedword = lines.get(0);
        } catch (Exception e) {
            logger.error("Fail to read preplacedword file from clreplacedpath, you may consider using absolute path", e);
        }
    }

    public void setPreplacedword(String preplacedword) {
        if (preplacedword == null) {
            return;
        }
        this.preplacedword = preplacedword;
    }

    public String getPreplacedword() {
        return preplacedword;
    }

    public boolean noPreplacedword() {
        return StringUtils.isEmpty(preplacedword);
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) {
            return true;
        }
        if (!(o instanceof Connection)) {
            return false;
        }
        Connection that = (Connection) o;
        return port == that.port && ip.equals(that.ip);
    }

    @Override
    public int hashCode() {
        int result = ip.hashCode();
        result = 31 * result + port;
        return result;
    }

    @Override
    public String toString() {
        return "Connection{" + "address='" + address + '\'' + ", port=" + port + ", user='" + user + '\'' + ", preplacedwordFile='" + preplacedwordFile + '\'' + '}';
    }

    public boolean valid() {
        return validConnection(address, port);
    }

    static boolean validConnection(String address, int port) {
        return address != null && port > 0 && port < 65536;
    }

    public String connectionIdentifier() {
        if (identifier == null) {
            identifier = ip + COMMON + getPort();
        }
        return identifier;
    }

    public String toConnectionUrl(String path) {
        return getAddress() + ":" + getPort();
    }

    @Override
    public int compareTo(Connection o) {
        int compare = ip.compareTo(o.ip);
        return compare != 0 ? compare : Integer.compare(port, o.port);
    }

    public Set<String> remoteIds() {
        return Sets.newHashSet(connectionIdentifier());
    }

    public List<Connection> getReals() {
        return Lists.newArrayList(this);
    }
}

19 Source : SQLHelper.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced SQLHelper {

    private static final Logger logger = LoggerFactory.getLogger(SQLHelper.clreplaced);

    public static String inSQL(Object value) {
        if (value == null) {
            return "NULL";
        }
        Clreplaced<?> aClreplaced = value.getClreplaced();
        if (ClreplacedUtils.isPrimitiveOrWrapper(aClreplaced) || CharSequence.clreplaced.isreplacedignableFrom(aClreplaced) || value instanceof Timestamp || value instanceof BigDecimal) {
            if (value instanceof String) {
                // TODO 2019/3/3 http://www.jguru.com/faq/view.jsp?EID=8881 {escape '/'} ?
                String replace = StringUtils.replace(StringUtils.replace(value.toString(), "'", "''"), "\\", "\\\\");
                value = "'" + replace + "'";
            } else if (value instanceof Timestamp) {
                value = "'" + value.toString() + "'";
            }
        } else if (SQLFunction.clreplaced.isreplacedignableFrom(aClreplaced)) {
            value = value.toString();
        } else if (aClreplaced == byte[].clreplaced) {
            value = "0x" + Hex.encodeHexString(((byte[]) value));
        } else {
            logger.error("Unhandled complex type: {}, value: {}", aClreplaced, value);
        }
        return value.toString();
    }

    public static String wrapCol(String col) {
        return '`' + col + '`';
    }

    private static String isAlter(String sql) {
        String[] words = { "alter ", "table " };
        String lower = sql.toLowerCase();
        int alterIndex = lower.indexOf(words[0]);
        int tableIndex = lower.indexOf(words[1], alterIndex + words[0].length());
        if (alterIndex != -1 && tableIndex != -1) {
            int afterTable = tableIndex + words[1].length();
            int addIndex = isAddAfter(lower, afterTable);
            if (addIndex != -1) {
                return sql.substring(afterTable, addIndex);
            }
            int dropIndex = isDrop(lower, afterTable);
            if (dropIndex != -1) {
                return sql.substring(afterTable, dropIndex);
            }
            int modifyIndex = isModify(lower, afterTable);
            if (modifyIndex != -1) {
                return sql.substring(afterTable, modifyIndex);
            }
        }
        return null;
    }

    private static int isAddAfter(String sql, int afterTable) {
        // alter table xx add yy after zz
        int i = sql.indexOf(" add ", afterTable);
        if (sql.indexOf(" after ", i) != -1) {
            return i;
        }
        return -1;
    }

    private static int isDrop(String sql, int afterTable) {
        // alter table xx drop column yy
        return sql.indexOf(" drop ", afterTable);
    }

    private static int isModify(String sql, int afterTable) {
        // alter table xx modify column yy after zz
        int i = sql.indexOf(" modify ", afterTable);
        if (sql.indexOf(" after ", i) != -1) {
            return i;
        }
        return -1;
    }

    public static AlterMeta alterMeta(String database, String sql) {
        String alterTarget = isAlter(sql);
        if (alterTarget != null) {
            alterTarget = alterTarget.replaceAll("`|\\s", "");
            if (!StringUtils.isEmpty(database)) {
                return new AlterMeta(database, alterTarget);
            }
            String[] split = alterTarget.split("\\.");
            replacedert split.length == 2;
            return new AlterMeta(split[0], split[1]);
        }
        return null;
    }
}

19 Source : FileUtil.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced FileUtil {

    private static Logger logger = LoggerFactory.getLogger(SyncerInputMeta.clreplaced);

    public static String readAll(String resourceName) throws IOException {
        return FileCopyUtils.copyToString(new InputStreamReader(getResource(resourceName).getInputStream()));
    }

    public static List<String> readLine(String resourceName) throws IOException {
        return Files.readAllLines(getResource(resourceName).getFile().toPath());
    }

    public static String readAll(InputStream inputStream) throws IOException {
        return FileCopyUtils.copyToString(new InputStreamReader(inputStream));
    }

    public static Resource getResource(String fileName) {
        Resource path = new FileSystemResource(fileName);
        if (!path.exists()) {
            path = new ClreplacedPathResource(fileName);
            if (!path.exists()) {
                throw new IllegalArgumentException("Config file [" + fileName + "] is not on clreplacedpath and it is not a absolute path file, fail to find it");
            }
        }
        return path;
    }

    public static void createDirIfNotExist(String fullPath) {
        Path metaDir = Paths.get(fullPath);
        if (!Files.exists(metaDir)) {
            logger.info("path[{}] not exists, creating a new one", fullPath);
            try {
                Files.createDirectories(metaDir);
            } catch (IOException e) {
                logger.error("Fail to create dir, aborting", e);
                throw new InvalidConfigException(e);
            }
        }
    }

    public static void createFile(Path path, Consumer<IOException> consumer) {
        if (!Files.exists(path)) {
            try {
                createDirIfNotExist(path.getParent().toString());
                Files.createFile(path);
            } catch (IOException e) {
                consumer.accept(e);
            }
        }
    }
}

19 Source : ArgUtil.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * @author zzt
 */
public clreplaced ArgUtil {

    private static final String DASH = "--";

    private static final Logger logger = LoggerFactory.getLogger(ArgUtil.clreplaced);

    public static HashMap<String, String> toMap(String[] args) {
        HashMap<String, String> argKV = new HashMap<>();
        for (String arg : args) {
            if (arg.startsWith(DASH)) {
                if (arg.contains("=")) {
                    String[] split = arg.split("=");
                    checkArgument(split.length == 2, "Invalid arg format: %s", arg);
                    String dash = split[0].substring(0, 2);
                    checkArgument(dash.equals(DASH) && split[0].length() > 2, "Invalid arg format: %s", arg);
                    argKV.put(split[0].substring(2), split[1]);
                } else {
                    String dash = arg.substring(0, 2);
                    checkArgument(dash.equals(DASH) && arg.length() > 2, "Invalid arg format: %s", arg);
                    argKV.put(arg.substring(2), "true");
                }
            } else {
                logger.error("Unsupported arg: {}", arg);
            }
        }
        return argKV;
    }
}

19 Source : SyncByQuery.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * update(set field)/delete by query
 * @see ExtraQuery
 */
public clreplaced SyncByQuery implements com.github.zzt93.syncer.data.SyncByQuery {

    private static final Logger logger = LoggerFactory.getLogger(SyncByQuery.clreplaced);

    private final HashMap<String, Object> syncBy = new HashMap<>();

    private final transient SyncData data;

    SyncByQuery(SyncData data) {
        this.data = data;
    }

    public SyncByQuery syncBy(String syncWithCol, Object value) {
        if (syncWithCol == null || value == null) {
            logger.warn("filter with {}={}", syncWithCol, value);
            return this;
        }
        data.syncByForeignKey();
        syncBy.put(syncWithCol, value);
        return this;
    }

    HashMap<String, Object> getSyncBy() {
        return syncBy;
    }

    @Override
    public String toString() {
        return "SyncByQuery{" + "syncBy=" + syncBy + '}';
    }
}

19 Source : ExtraQuery.java
with BSD 3-Clause "New" or "Revised" License
from zzt93

/**
 * ----------- index/insert by query ------------
 * @see SyncByQuery
 */
public clreplaced ExtraQuery implements com.github.zzt93.syncer.data.ExtraQuery {

    private static final Logger logger = LoggerFactory.getLogger(ExtraQuery.clreplaced);

    private final HashMap<String, Object> queryBy = new HashMap<>();

    private final transient SyncData data;

    private String queryId;

    private String indexName;

    private String typeName;

    private String[] select;

    private String[] as;

    private final HashMap<String, Object> queryResult = new HashMap<>();

    ExtraQuery(SyncData data) {
        this.data = data;
    }

    public String getTypeName() {
        return typeName;
    }

    public ExtraQuery setTypeName(String typeName) {
        this.typeName = typeName;
        return this;
    }

    public ExtraQuery filter(String field, Object value) {
        queryBy.put(field, value);
        return this;
    }

    public ExtraQuery select(String... field) {
        select = field;
        return this;
    }

    public ExtraQuery addField(String... cols) {
        if (cols.length != select.length) {
            throw new InvalidConfigException("Column length is not same as query select result");
        }
        this.as = cols;
        for (String col : cols) {
            data.getFields().put(col, this);
        }
        return this;
    }

    public String getIndexName() {
        return indexName;
    }

    public ExtraQuery setIndexName(String indexName) {
        this.indexName = indexName;
        return this;
    }

    public HashMap<String, Object> getQueryBy() {
        return queryBy;
    }

    public String[] getSelect() {
        return select;
    }

    public String getAs(int i) {
        return as != null ? as[i] : select[i];
    }

    public void addQueryResult(Map<String, Object> result) {
        queryResult.putAll(result);
    }

    public Object getQueryResult(String key) {
        Object o = queryResult.get(key);
        if (o == null) {
            logger.warn("Fail to query [{}] by {}", key, this);
        }
        return o;
    }

    public Object getField(String s) {
        return data.getField(s);
    }

    @Override
    public String toString() {
        return "ExtraQuery{select " + Arrays.toString(select) + " as " + Arrays.toString(as) + " from " + indexName + "." + typeName + " where " + queryBy + "}" + (!queryResult.isEmpty() ? queryResult : "");
    }
}

19 Source : KoalasAopUtil.java
with Apache License 2.0
from zyl19880411

/**
 * Copyright (C) 2018
 * All rights reserved
 * User: yulong.zhang
 * Date:2019年06月19日21:22:39
 */
public clreplaced KoalasAopUtil {

    private final static Logger logger = LoggerFactory.getLogger(KoalasAopUtil.clreplaced);

    /**
     * 获取被代理类的Object
     * @author Monkey
     */
    public static Object getTarget(Object proxy) {
        try {
            if (!AopUtils.isAopProxy(proxy)) {
                // 不是代理对象
                return proxy;
            }
            if (AopUtils.isJdkDynamicProxy(proxy)) {
                return getJdkDynamicProxyTargetObject(proxy);
            } else {
                // cglib
                return getCglibProxyTargetObject(proxy);
            }
        } catch (Exception e) {
            logger.error("get spring target bean error,proxy=" + proxy, e);
            return null;
        }
    }

    /**
     * CGLIB方式被代理类的获取
     * @author Monkey
     */
    private static Object getCglibProxyTargetObject(Object proxy) throws Exception {
        Field h = proxy.getClreplaced().getDeclaredField("CGLIB$CALLBACK_0");
        h.setAccessible(true);
        Object dynamicAdvisedInterceptor = h.get(proxy);
        Field advised = dynamicAdvisedInterceptor.getClreplaced().getDeclaredField("advised");
        advised.setAccessible(true);
        Object target = ((AdvisedSupport) advised.get(dynamicAdvisedInterceptor)).getTargetSource().getTarget();
        return target;
    }

    /**
     * JDK动态代理方式被代理类的获取
     * @author Monkey
     */
    private static Object getJdkDynamicProxyTargetObject(Object proxy) throws Exception {
        Field h = proxy.getClreplaced().getSuperclreplaced().getDeclaredField("h");
        h.setAccessible(true);
        AopProxy aopProxy = (AopProxy) h.get(proxy);
        Field advised = aopProxy.getClreplaced().getDeclaredField("advised");
        advised.setAccessible(true);
        Object target = ((AdvisedSupport) advised.get(aopProxy)).getTargetSource().getTarget();
        return target;
    }
}

19 Source : KoalasThreadedSelectorServer.java
with Apache License 2.0
from zyl19880411

/**
 * Copyright (C) 2018
 * All rights reserved
 * User: yulong.zhang
 * Date:2018年11月23日11:13:33
 */
public clreplaced KoalasThreadedSelectorServer extends KoalasAbstractNonblockingServer {

    private static final Logger LOGGER = LoggerFactory.getLogger(KoalasThreadedSelectorServer.clreplaced.getName());

    public static clreplaced Args extends AbstractNonblockingServerArgs<Args> {

        public int selectorThreads = 2;

        private int workerThreads = 5;

        private int stopTimeoutVal = 60;

        private TimeUnit stopTimeoutUnit = TimeUnit.SECONDS;

        private ExecutorService executorService = null;

        private int acceptQueueSizePerThread = 4;

        public static enum AcceptPolicy {

            FAIR_ACCEPT, FAST_ACCEPT
        }

        private AcceptPolicy acceptPolicy = AcceptPolicy.FAST_ACCEPT;

        public Args(TNonblockingServerTransport transport) {
            super(transport);
        }

        public Args selectorThreads(int i) {
            selectorThreads = i;
            return this;
        }

        public int getSelectorThreads() {
            return selectorThreads;
        }

        public Args workerThreads(int i) {
            workerThreads = i;
            return this;
        }

        public int getWorkerThreads() {
            return workerThreads;
        }

        public int getStopTimeoutVal() {
            return stopTimeoutVal;
        }

        public Args stopTimeoutVal(int stopTimeoutVal) {
            this.stopTimeoutVal = stopTimeoutVal;
            return this;
        }

        public TimeUnit getStopTimeoutUnit() {
            return stopTimeoutUnit;
        }

        public Args stopTimeoutUnit(TimeUnit stopTimeoutUnit) {
            this.stopTimeoutUnit = stopTimeoutUnit;
            return this;
        }

        public ExecutorService getExecutorService() {
            return executorService;
        }

        public Args executorService(ExecutorService executorService) {
            this.executorService = executorService;
            return this;
        }

        public int getAcceptQueueSizePerThread() {
            return acceptQueueSizePerThread;
        }

        public Args acceptQueueSizePerThread(int acceptQueueSizePerThread) {
            this.acceptQueueSizePerThread = acceptQueueSizePerThread;
            return this;
        }

        public AcceptPolicy getAcceptPolicy() {
            return acceptPolicy;
        }

        public Args acceptPolicy(AcceptPolicy acceptPolicy) {
            this.acceptPolicy = acceptPolicy;
            return this;
        }

        public void validate() {
            if (selectorThreads <= 0) {
                throw new IllegalArgumentException("selectorThreads must be positive.");
            }
            if (workerThreads < 0) {
                throw new IllegalArgumentException("workerThreads must be non-negative.");
            }
            if (acceptQueueSizePerThread <= 0) {
                throw new IllegalArgumentException("acceptQueueSizePerThread must be positive.");
            }
        }
    }

    private volatile boolean stopped_ = true;

    private AcceptThread acceptThread;

    private final Set<SelectorThread> selectorThreads = new HashSet<SelectorThread>();

    private final ExecutorService invoker;

    private final Args args;

    private String privateKey;

    private String publicKey;

    private String serviceName;

    private TProcessor tGenericProcessor;

    private boolean cat;

    public boolean isCat() {
        return cat;
    }

    public void setCat(boolean cat) {
        this.cat = cat;
    }

    public TProcessor gettGenericProcessor() {
        return tGenericProcessor;
    }

    public void settGenericProcessor(TProcessor tGenericProcessor) {
        this.tGenericProcessor = tGenericProcessor;
    }

    public String getServiceName() {
        return serviceName;
    }

    public void setServiceName(String serviceName) {
        this.serviceName = serviceName;
    }

    public String getPrivateKey() {
        return privateKey;
    }

    public void setPrivateKey(String privateKey) {
        this.privateKey = privateKey;
    }

    public String getPublicKey() {
        return publicKey;
    }

    public void setPublicKey(String publicKey) {
        this.publicKey = publicKey;
    }

    public KoalasThreadedSelectorServer(Args args) {
        super(args);
        args.validate();
        invoker = args.executorService == null ? createDefaultExecutor(args) : args.executorService;
        this.args = args;
    }

    @Override
    protected boolean startThreads() {
        try {
            for (int i = 0; i < args.selectorThreads; ++i) {
                selectorThreads.add(new SelectorThread(args.acceptQueueSizePerThread));
            }
            acceptThread = new AcceptThread((TNonblockingServerTransport) serverTransport_, createSelectorThreadLoadBalancer(selectorThreads));
            stopped_ = false;
            for (SelectorThread thread : selectorThreads) {
                thread.start();
            }
            acceptThread.start();
            return true;
        } catch (IOException e) {
            LOGGER.error("Failed to start threads!", e);
            return false;
        }
    }

    @Override
    protected void waitForShutdown() {
        try {
            joinThreads();
        } catch (InterruptedException e) {
            LOGGER.error("Interrupted while joining threads!", e);
        }
        gracefullyShutdownInvokerPool();
    }

    protected void joinThreads() throws InterruptedException {
        acceptThread.join();
        for (SelectorThread thread : selectorThreads) {
            thread.join();
        }
    }

    @Override
    public void stop() {
        stopped_ = true;
        stopListening();
        if (acceptThread != null) {
            acceptThread.wakeupSelector();
        }
        if (selectorThreads != null) {
            for (SelectorThread thread : selectorThreads) {
                if (thread != null)
                    thread.wakeupSelector();
            }
        }
    }

    protected void gracefullyShutdownInvokerPool() {
        invoker.shutdown();
        long timeoutMS = args.stopTimeoutUnit.toMillis(args.stopTimeoutVal);
        long now = System.currentTimeMillis();
        while (timeoutMS >= 0) {
            try {
                invoker.awaitTermination(timeoutMS, TimeUnit.MILLISECONDS);
                break;
            } catch (InterruptedException ix) {
                long newnow = System.currentTimeMillis();
                timeoutMS -= (newnow - now);
                now = newnow;
            }
        }
    }

    @Override
    protected boolean requestInvoke(FrameBuffer frameBuffer) {
        Runnable invocation = getRunnable(frameBuffer);
        if (invoker != null) {
            try {
                invoker.execute(invocation);
                return true;
            } catch (RejectedExecutionException rx) {
                LOGGER.warn("ExecutorService rejected execution!", rx);
                return false;
            }
        } else {
            invocation.run();
            return true;
        }
    }

    protected Runnable getRunnable(FrameBuffer frameBuffer) {
        return new Invocation(frameBuffer);
    }

    protected static ExecutorService createDefaultExecutor(Args options) {
        return (options.workerThreads > 0) ? Executors.newFixedThreadPool(options.workerThreads) : null;
    }

    private static BlockingQueue<TNonblockingTransport> createDefaultAcceptQueue(int queueSize) {
        if (queueSize == 0) {
            return new LinkedBlockingQueue<TNonblockingTransport>();
        }
        return new ArrayBlockingQueue<TNonblockingTransport>(queueSize);
    }

    protected clreplaced AcceptThread extends Thread {

        private final TNonblockingServerTransport serverTransport;

        private final Selector acceptSelector;

        private final SelectorThreadLoadBalancer threadChooser;

        public AcceptThread(TNonblockingServerTransport serverTransport, SelectorThreadLoadBalancer threadChooser) throws IOException {
            this.serverTransport = serverTransport;
            this.threadChooser = threadChooser;
            this.acceptSelector = SelectorProvider.provider().openSelector();
            this.serverTransport.registerSelector(acceptSelector);
        }

        public void run() {
            try {
                while (!stopped_) {
                    select();
                }
            } catch (Throwable t) {
                LOGGER.error("run() exiting due to uncaught error", t);
            } finally {
                KoalasThreadedSelectorServer.this.stop();
            }
        }

        public void wakeupSelector() {
            acceptSelector.wakeup();
        }

        private void select() {
            try {
                acceptSelector.select();
                Iterator<SelectionKey> selectedKeys = acceptSelector.selectedKeys().iterator();
                while (!stopped_ && selectedKeys.hasNext()) {
                    SelectionKey key = selectedKeys.next();
                    selectedKeys.remove();
                    if (!key.isValid()) {
                        continue;
                    }
                    if (key.isAcceptable()) {
                        handleAccept();
                    } else {
                        LOGGER.warn("Unexpected state in select! " + key.interestOps());
                    }
                }
            } catch (IOException e) {
                LOGGER.warn("Got an IOException while selecting!", e);
            }
        }

        private void handleAccept() {
            final TNonblockingTransport client = doAccept();
            if (client != null) {
                final SelectorThread targetThread = threadChooser.nextThread();
                if (args.acceptPolicy == Args.AcceptPolicy.FAST_ACCEPT || invoker == null) {
                    doAddAccept(targetThread, client);
                } else {
                    try {
                        invoker.submit(new Runnable() {

                            public void run() {
                                doAddAccept(targetThread, client);
                            }
                        });
                    } catch (RejectedExecutionException rx) {
                        LOGGER.warn("ExecutorService rejected accept registration!", rx);
                        client.close();
                    }
                }
            }
        }

        private TNonblockingTransport doAccept() {
            try {
                return (TNonblockingTransport) serverTransport.accept();
            } catch (TTransportException tte) {
                LOGGER.warn("Exception trying to accept!", tte);
                return null;
            }
        }

        private void doAddAccept(SelectorThread thread, TNonblockingTransport client) {
            if (!thread.addAcceptedConnection(client)) {
                client.close();
            }
        }
    }

    protected clreplaced SelectorThread extends AbstractSelectThread {

        private final BlockingQueue<TNonblockingTransport> acceptedQueue;

        public SelectorThread() throws IOException {
            this(new LinkedBlockingQueue<TNonblockingTransport>());
        }

        public SelectorThread(int maxPendingAccepts) throws IOException {
            this(createDefaultAcceptQueue(maxPendingAccepts));
        }

        public SelectorThread(BlockingQueue<TNonblockingTransport> acceptedQueue) throws IOException {
            this.acceptedQueue = acceptedQueue;
        }

        public boolean addAcceptedConnection(TNonblockingTransport accepted) {
            try {
                acceptedQueue.put(accepted);
            } catch (InterruptedException e) {
                LOGGER.warn("Interrupted while adding accepted connection!", e);
                return false;
            }
            selector.wakeup();
            return true;
        }

        public void run() {
            try {
                while (!stopped_) {
                    select();
                    processAcceptedConnections();
                    processInterestChanges();
                }
                for (SelectionKey selectionKey : selector.keys()) {
                    cleanupSelectionKey(selectionKey);
                }
            } catch (Throwable t) {
                LOGGER.error("run() exiting due to uncaught error", t);
            } finally {
                // This will wake up the accept thread and the other selector threads
                KoalasThreadedSelectorServer.this.stop();
            }
        }

        private void select() {
            try {
                selector.select();
                Iterator<SelectionKey> selectedKeys = selector.selectedKeys().iterator();
                while (!stopped_ && selectedKeys.hasNext()) {
                    SelectionKey key = selectedKeys.next();
                    selectedKeys.remove();
                    if (!key.isValid()) {
                        cleanupSelectionKey(key);
                        continue;
                    }
                    if (key.isReadable()) {
                        handleRead(key);
                    } else if (key.isWritable()) {
                        handleWrite(key);
                    } else {
                        LOGGER.warn("Unexpected state in select! " + key.interestOps());
                    }
                }
            } catch (IOException e) {
                LOGGER.warn("Got an IOException while selecting!", e);
            }
        }

        private void processAcceptedConnections() {
            // Register accepted connections
            while (!stopped_) {
                TNonblockingTransport accepted = acceptedQueue.poll();
                if (accepted == null) {
                    break;
                }
                registerAccepted(accepted);
            }
        }

        private void registerAccepted(TNonblockingTransport accepted) {
            SelectionKey clientKey = null;
            try {
                clientKey = accepted.registerSelector(selector, SelectionKey.OP_READ);
                FrameBuffer frameBuffer = new FrameBuffer(accepted, clientKey, SelectorThread.this, privateKey, publicKey, serviceName, tGenericProcessor, cat);
                clientKey.attach(frameBuffer);
            } catch (IOException e) {
                LOGGER.warn("Failed to register accepted connection to selector!", e);
                if (clientKey != null) {
                    cleanupSelectionKey(clientKey);
                }
                accepted.close();
            }
        }
    }

    protected SelectorThreadLoadBalancer createSelectorThreadLoadBalancer(Collection<? extends SelectorThread> threads) {
        return new SelectorThreadLoadBalancer(threads);
    }

    protected clreplaced SelectorThreadLoadBalancer {

        private final Collection<? extends SelectorThread> threads;

        private Iterator<? extends SelectorThread> nextThreadIterator;

        public <T extends SelectorThread> SelectorThreadLoadBalancer(Collection<T> threads) {
            if (threads.isEmpty()) {
                throw new IllegalArgumentException("At least one selector thread is required");
            }
            this.threads = Collections.unmodifiableList(new ArrayList<T>(threads));
            nextThreadIterator = this.threads.iterator();
        }

        public SelectorThread nextThread() {
            if (!nextThreadIterator.hasNext()) {
                nextThreadIterator = threads.iterator();
            }
            return nextThreadIterator.next();
        }
    }
}

19 Source : KoalasServerPublisher.java
with Apache License 2.0
from zyl19880411

/**
 * Copyright (C) 2018
 * All rights reserved
 * User: yulong.zhang
 * Date:2018年11月23日11:13:33
 */
public clreplaced KoalreplacederverPublisher extends AbstractKoalsServerPublisher implements FactoryBean<Object>, ApplicationContextAware, InitializingBean {

    private final static Logger logger = LoggerFactory.getLogger(KoalreplacederverPublisher.clreplaced);

    public static final String NETTY = "netty";

    public static final String THRIFT = "thrift";

    @Override
    public Object getObject() {
        return this;
    }

    @Override
    public Clreplaced<?> getObjectType() {
        return this.getClreplaced();
    }

    @Override
    public boolean isSingleton() {
        return true;
    }

    @Override
    public void afterPropertiesSet() {
        this.checkparam();
        if (NETTY.equals(this.serverType.toLowerCase().trim())) {
            ikoalreplacederver = new NettyServer(this);
        } else if (THRIFT.equals(this.serverType.toLowerCase().trim())) {
            ikoalreplacederver = new ThriftServer(this);
        } else {
            logger.error("other server is not support at since v1.0,clreplacedName:{}", getServiceInterface().getName());
            throw new IllegalArgumentException("other server is not support at since v1.0,clreplacedName=" + getServiceInterface().getName());
        }
        ikoalreplacederver.run();
    }

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        this.applicationContext = applicationContext;
    }

    public void destroy() {
        if (this.ikoalreplacederver != null) {
            this.ikoalreplacederver.stop();
        }
    }
}

19 Source : AbstractKoalsServerPublisher.java
with Apache License 2.0
from zyl19880411

/**
 * Copyright (C) 2018
 * All rights reserved
 * User: yulong.zhang
 * Date:2018年11月23日11:13:33
 */
public clreplaced AbstractKoalsServerPublisher {

    private final static Logger logger = LoggerFactory.getLogger(AbstractKoalsServerPublisher.clreplaced);

    public static final int DEFAULT_EVENT_LOOP_THREADS;

    public static final int DEFAULT_KOALAS_THREADS;

    public static final int DEFAULT_THRIFT_ACCETT_THREAD;

    static {
        DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", Runtime.getRuntime().availableProcessors() * 2));
        if (logger.isDebugEnabled()) {
            logger.debug("-Dio.netty.eventLoopThreads: {}", DEFAULT_EVENT_LOOP_THREADS);
        }
        DEFAULT_KOALAS_THREADS = 256;
        DEFAULT_THRIFT_ACCETT_THREAD = 5;
    }

    public static final String IFACE = "Iface";

    public static final String PROCESSOR = "Processor";

    public Object serviceImpl;

    public Clreplaced<?> serviceInterface;

    public int port;

    public String zkpath;

    public int bossThreadCount;

    public int workThreadCount;

    public int koalasThreadCount;

    public String env = "dev";

    public int weight = 10;

    public String serverType = "NETTY";

    public int workQueue;

    // RSA service
    public String privateKey;

    public String publicKey;

    public int maxLength = Integer.MAX_VALUE;

    public ApplicationContext applicationContext;

    public Ikoalreplacederver ikoalreplacederver;

    public int getMaxLength() {
        return maxLength;
    }

    public void setMaxLength(int maxLength) {
        this.maxLength = maxLength;
    }

    public boolean cat = false;

    public boolean isCat() {
        return cat;
    }

    public void setCat(boolean cat) {
        this.cat = cat;
    }

    public Object getServiceImpl() {
        return serviceImpl;
    }

    public void setServiceImpl(Object serviceImpl) {
        this.serviceImpl = serviceImpl;
    }

    public Clreplaced<?> getServiceInterface() {
        return serviceInterface;
    }

    public void setServiceInterface(Clreplaced<?> serviceInterface) {
        this.serviceInterface = serviceInterface;
    }

    public int getPort() {
        return port;
    }

    public void setPort(int port) {
        this.port = port;
    }

    public int getBossThreadCount() {
        return bossThreadCount;
    }

    public void setBossThreadCount(int bossThreadCount) {
        this.bossThreadCount = bossThreadCount;
    }

    public int getWorkThreadCount() {
        return workThreadCount;
    }

    public void setWorkThreadCount(int workThreadCount) {
        this.workThreadCount = workThreadCount;
    }

    public int getKoalasThreadCount() {
        return koalasThreadCount;
    }

    public void setKoalasThreadCount(int koalasThreadCount) {
        this.koalasThreadCount = koalasThreadCount;
    }

    public String getZkpath() {
        return zkpath;
    }

    public void setZkpath(String zkpath) {
        this.zkpath = zkpath;
    }

    public String getEnv() {
        return env;
    }

    public void setEnv(String env) {
        this.env = env;
    }

    public int getWeight() {
        return weight;
    }

    public void setWeight(int weight) {
        this.weight = weight;
    }

    public String getServerType() {
        return serverType;
    }

    public void setServerType(String serverType) {
        this.serverType = serverType;
    }

    public int getWorkQueue() {
        return workQueue;
    }

    public void setWorkQueue(int workQueue) {
        this.workQueue = workQueue;
    }

    public String getPrivateKey() {
        return privateKey;
    }

    public void setPrivateKey(String privateKey) {
        this.privateKey = privateKey;
    }

    public String getPublicKey() {
        return publicKey;
    }

    public void setPublicKey(String publicKey) {
        this.publicKey = publicKey;
    }

    public TProcessor getTProcessor() {
        Clreplaced iface = getSynIfaceInterface(serviceInterface);
        try {
            return getProcessorClreplaced(serviceInterface).getDeclaredConstructor(iface).newInstance(serviceImpl);
        } catch (NoSuchMethodException e) {
            logger.error("can't find the TProcessor Constructor with Iface", e);
        } catch (IllegalAccessException e) {
            logger.error("IllegalAccessException with Iface");
        } catch (InstantiationException e) {
            logger.error("IllegalInstantiationExceptionAccessException with Iface", e);
        } catch (InvocationTargetException e) {
            logger.error("InvocationTargetException with Iface", e);
        }
        return null;
    }

    public TProcessor getGenericTProcessor() {
        Clreplaced iface = getSynIfaceInterface(GenericService.clreplaced);
        try {
            return getProcessorClreplaced(GenericService.clreplaced).getDeclaredConstructor(iface).newInstance(new GenericServiceImpl(serviceImpl));
        } catch (NoSuchMethodException e) {
            logger.error("can't find the GenericTProcessor Constructor with Iface", e);
        } catch (IllegalAccessException e) {
            logger.error("IllegalAccessException the GenericTProcessor with Iface");
        } catch (InstantiationException e) {
            logger.error("IllegalInstantiationExceptionAccessException the GenericTProcessor with Iface", e);
        } catch (InvocationTargetException e) {
            logger.error("InvocationTargetException the GenericTProcessor with Iface", e);
        }
        return null;
    }

    private Clreplaced<?> getSynIfaceInterface(Clreplaced<?> serviceInterface) {
        Clreplaced<?>[] clreplacedes = serviceInterface.getClreplacedes();
        for (Clreplaced c : clreplacedes) if (c.isMemberClreplaced() && c.isInterface() && c.getSimpleName().equals(IFACE)) {
            return c;
        }
        throw new IllegalArgumentException("serviceInterface must contain Sub Interface of Iface");
    }

    private Clreplaced<TProcessor> getProcessorClreplaced(Clreplaced<?> serviceInterface) {
        Clreplaced<?>[] clreplacedes = serviceInterface.getClreplacedes();
        for (Clreplaced c : clreplacedes) if (c.isMemberClreplaced() && !c.isInterface() && c.getSimpleName().equals(PROCESSOR)) {
            return c;
        }
        throw new IllegalArgumentException("serviceInterface must contain Sub Interface of Processor");
    }

    protected void checkparam() {
        if (serviceImpl == null) {
            throw new IllegalArgumentException("the serviceImpl can't be null");
        }
        if (serviceInterface == null) {
            throw new IllegalArgumentException("the serviceInterface can't be null");
        }
        if (port == 0) {
            throw new IllegalArgumentException("set the right port");
        }
    }

    @Override
    public String toString() {
        return "KoalreplacederverPublisher{" + "serviceImpl=" + serviceImpl + ", serviceInterface=" + serviceInterface + ", port=" + port + ", zkpath='" + zkpath + '\'' + ", env='" + env + '\'' + ", serverType='" + serverType + '\'' + '}';
    }
}

19 Source : ZookeeperClient.java
with Apache License 2.0
from zyl19880411

/**
 * Copyright (C) 2018
 * All rights reserved
 * User: yulong.zhang
 * Date:2018年11月23日11:13:33
 */
public clreplaced ZookeeperClient {

    private static final Logger LOG = LoggerFactory.getLogger(ZookeeperClient.clreplaced);

    public static final int RETRY_TIMES = 2;

    public static final int SESSION_TIMEOUT = 3000;

    public static final Watcher NULL = null;

    public static final String UTF_8 = "UTF-8";

    public static final int TIMEOUT = 3000;

    public static final byte HEARTBEAT = (byte) 2;

    private String env;

    private String path;

    private String serviceName;

    private ZooKeeper zookeeper = null;

    private ZookeeperClusterImpl zookeeperClister;

    // private Map<String, Watcher> serviceWatcher = new ConcurrentHashMap<> ();
    private CountDownLatch firstInitChildren = new CountDownLatch(1);

    // private boolean firstInitChildren = true;
    // 当前服务列表
    private List<RemoteServer> serverList = new CopyOnWriteArrayList<>();

    // 心跳服务列表
    private Map<String, HeartbeatService.Client> serverHeartbeatMap = new ConcurrentHashMap<>();

    private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);

    public List<RemoteServer> getServerList() {
        return serverList;
    }

    public ZookeeperClient(String env, String path, String serviceName, ZookeeperClusterImpl zookeeperClister) {
        if (env == null) {
            throw new RuntimeException("env can't be null");
        }
        if (serviceName == null) {
            throw new RuntimeException("serviceName can't be null");
        }
        if (path == null) {
            throw new RuntimeException("zk ip and port can't be null");
        }
        this.env = env;
        this.path = path;
        this.serviceName = serviceName;
        this.zookeeperClister = zookeeperClister;
    }

    public void initZooKeeper() {
        CountDownLatch c = new CountDownLatch(1);
        if (zookeeper == null) {
            try {
                zookeeper = new ZooKeeper(path, SESSION_TIMEOUT, new ClientInireplacedcher(c));
            } catch (IOException e) {
                LOG.error("zk server faild service:" + env + "-" + serviceName, e);
            }
        }
        try {
            // 网络抖动重试3次
            int retry = 0;
            boolean connected = false;
            while (retry++ < RETRY_TIMES) {
                if (c.await(5, TimeUnit.SECONDS)) {
                    connected = true;
                    break;
                }
            }
            if (!connected) {
                LOG.error("zk client connected fail! :" + env + "-" + serviceName);
                throw new IllegalArgumentException("zk client connected fail!");
            }
        } catch (InterruptedException e) {
            LOG.error(e.getMessage(), e);
        }
        try {
            String envPath = env.startsWith("/") ? env : "/".concat(env);
            if (zookeeper.exists(envPath, null) == null) {
                zookeeper.create(envPath, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            }
            String servicePath = serviceName.startsWith("/") ? serviceName : "/".concat(serviceName);
            if (zookeeper.exists(envPath + servicePath, null) == null) {
                zookeeper.create(envPath + servicePath, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
            }
            Watcher w = new koalasWatcher();
            // /env/com.test.service
            getChildren(envPath + servicePath, w);
            watchChildDateChange(envPath + servicePath, w);
            initScheduled();
        } catch (KeeperException e) {
            LOG.error(e.getMessage(), e);
        } catch (InterruptedException e) {
            LOG.error(e.getMessage(), e);
        }
    }

    private void initScheduled() {
        Calendar now = Calendar.getInstance();
        Calendar after = Calendar.getInstance();
        after.set(Calendar.MINUTE, now.get(Calendar.MINUTE) + 1);
        after.set(Calendar.SECOND, 0);
        after.set(Calendar.MILLISECOND, 0);
        if ((after.getTimeInMillis() - now.getTimeInMillis()) >= 10 * 1000) {
            executor.scheduleWithFixedDelay(new HeartbeatRun(), (after.getTimeInMillis() - now.getTimeInMillis()) / 1000, 60, TimeUnit.SECONDS);
        } else {
            executor.scheduleWithFixedDelay(new HeartbeatRun(), (after.getTimeInMillis() - now.getTimeInMillis()) / 1000 + 60, 60, TimeUnit.SECONDS);
        }
    }

    private void watchChildDateChange(String path, Watcher w) {
        // path /env/com.test.service
        try {
            // 192.168.3.1:6666 192.168.3.1:9990 192.168.3.1:9999
            List<String> childpaths = this.zookeeper.getChildren(path, null);
            for (String _childPath : childpaths) {
                // /env/com.test.service/192.168.3.2:8080
                String fullPath = path.concat("/").concat(_childPath);
                this.zookeeper.getData(fullPath, w, new Stat());
            }
        } catch (KeeperException e) {
            LOG.error(e.getMessage(), e);
        } catch (InterruptedException e) {
            LOG.error(e.getMessage(), e);
        } finally {
            firstInitChildren.countDown();
        }
    }

    private void getChildren(String path, Watcher w) {
        // path /env/com.test.service
        try {
            // 192.168.3.1:6666 192.168.3.1:9990 192.168.3.1:9999
            List<String> childpaths = this.zookeeper.getChildren(path, w);
            updateServerList(childpaths, path);
        } catch (KeeperException e) {
            LOG.error(e.getMessage(), e);
        } catch (InterruptedException e) {
            LOG.error(e.getMessage(), e);
        }
    }

    private clreplaced koalasWatcher implements Watcher {

        @Override
        public void process(WatchedEvent event) {
            if (event.getState() == Event.KeeperState.SyncConnected) {
                // when the new server or shutdown
                if (event.getType() == Event.EventType.NodeChildrenChanged) {
                    // /env/com.test.service
                    String parentPath = event.getPath();
                    LOG.info("the service {} is changed ! ", serviceName);
                    try {
                        // wait the init childChanged
                        firstInitChildren.await();
                        List<String> childpaths = ZookeeperClient.this.zookeeper.getChildren(parentPath, this);
                        ZookeeperClient.this.updateServerList(childpaths, parentPath);
                        LOG.info("the serviceList: {} ! ", childpaths);
                        for (String _childpaths : childpaths) {
                            String fullpath = parentPath.concat("/").concat(_childpaths);
                            // 192.168.3.1
                            ZookeeperClient.this.zookeeper.getData(fullpath, this, new Stat());
                        }
                    } catch (KeeperException e) {
                        LOG.error(e.getMessage(), e);
                    } catch (InterruptedException e) {
                        LOG.error(e.getMessage(), e);
                    }
                }
                if (event.getType() == Event.EventType.NodeDataChanged) {
                    // /env/com.test.service/192.168.3.2:6666
                    String fullPath = event.getPath();
                    LOG.info("the service 【{}】 data {} is changed ! full mess is 【{}】 ", serviceName, fullPath);
                    try {
                        // wait the init childDataChanged
                        firstInitChildren.await();
                        String data = new String(ZookeeperClient.this.zookeeper.getData(fullPath, this, new Stat()), UTF_8);
                        JSONObject json = JSONObject.parseObject(data);
                        // 192.168.3.2:6666
                        String _childPath = fullPath.substring(fullPath.lastIndexOf("/") + 1);
                        // 远程ip
                        String ip = _childPath.split(":")[0];
                        // 服务端口
                        String port = _childPath.split(":")[1];
                        // 权重
                        String weight = json.getString("weight");
                        // 是否可用
                        String enable = json.getString("enable");
                        // 注冊类型
                        String server = json.getString("server");
                        RemoteServer remoteServer = new RemoteServer(ip, port, Integer.valueOf(weight), "1".equals(enable), server);
                        ZookeeperClient.this.updateServer(remoteServer);
                    } catch (KeeperException e) {
                        e.printStackTrace();
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    } catch (UnsupportedEncodingException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    private void updateServer(RemoteServer remoteServer) {
        try {
            zookeeperClister.writeLock.lock();
            if (serverList != null)
                for (int i = 0; i < serverList.size(); i++) {
                    RemoteServer tempServer_ = serverList.get(i);
                    if (tempServer_.getIp().equals(remoteServer.getIp()) && tempServer_.getPort().equals(remoteServer.getPort())) {
                        serverList.set(i, remoteServer);
                        // help gc
                        tempServer_ = null;
                    }
                }
        } finally {
            zookeeperClister.writeLock.unlock();
        }
    }

    private void updateServerList(List<String> childpaths, String parentPath) {
        try {
            zookeeperClister.writeLock.lock();
            if (serverList.size() != 0) {
                serverList.clear();
            }
            if (serverHeartbeatMap.size() != 0) {
                serverHeartbeatMap.clear();
            }
            if (zookeeperClister.serverPollMap != null && zookeeperClister.serverPollMap.size() > 0) {
                for (String string : zookeeperClister.serverPollMap.keySet()) {
                    GenericObjectPool p = zookeeperClister.serverPollMap.get(string);
                    if (p != null)
                        p.close();
                    zookeeperClister.serverPollMap.remove(string);
                }
            }
            for (String _childPath : childpaths) {
                // /env/com.test.service/192.168.1.10:6666
                String currPath = parentPath.concat("/").concat(_childPath);
                // 
                try {
                    byte[] bytes = zookeeper.getData(currPath, null, new Stat());
                    try {
                        String data = new String(bytes, "UTF-8");
                        JSONObject json = JSONObject.parseObject(data);
                        // 远程ip
                        String ip = _childPath.split(":")[0];
                        // 服务端口
                        String port = _childPath.split(":")[1];
                        // 权重
                        String weight = json.getString("weight");
                        // 是否可用
                        String enable = json.getString("enable");
                        // 注冊类型
                        String server = json.getString("server");
                        RemoteServer remoteServer = new RemoteServer(ip, port, Integer.valueOf(weight), "1".equals(enable), server);
                        serverList.add(remoteServer);
                        // Heartbeat
                        TSocket t = new TSocket(remoteServer.getIp(), Integer.parseInt(remoteServer.getPort()), TIMEOUT);
                        TTransport transport = new TKoalasFramedTransport(t);
                        ((TKoalasFramedTransport) transport).setHeartbeat(HEARTBEAT);
                        TProtocol protocol = new KoalasBinaryProtocol(transport);
                        HeartbeatService.Client client = new HeartbeatService.Client(protocol);
                        transport.open();
                        serverHeartbeatMap.put(zookeeperClister.createMapKey(remoteServer), client);
                    } catch (UnsupportedEncodingException e) {
                        LOG.error(e.getMessage() + " UTF-8 is not allow!", e);
                    } catch (TTransportException e) {
                        LOG.error(e.getMessage(), e);
                    }
                } catch (KeeperException e) {
                    LOG.error(e.getMessage() + "currPath is not exists!", e);
                } catch (InterruptedException e) {
                    LOG.error(e.getMessage() + "the current thread is Interrupted", e);
                }
            }
        } finally {
            zookeeperClister.writeLock.unlock();
        }
    }

    private clreplaced ClientInireplacedcher implements Watcher {

        private CountDownLatch countDownLatch;

        public ClientInireplacedcher(CountDownLatch countDownLatch) {
            this.countDownLatch = countDownLatch;
        }

        @Override
        public void process(WatchedEvent event) {
            if (Event.KeeperState.SyncConnected == event.getState()) {
                LOG.info("the service {}-{}-{} is SyncConnected!", IPUtil.getIpV4(), ZookeeperClient.this.env, ZookeeperClient.this.serviceName);
                countDownLatch.countDown();
            }
            if (Event.KeeperState.Expired == event.getState()) {
                LOG.warn("the service {}-{}-{} is expired!", IPUtil.getIpV4(), ZookeeperClient.this.env, ZookeeperClient.this.serviceName);
                reConnected();
            }
            if (Event.KeeperState.Disconnected == event.getState()) {
                LOG.warn("the service {}-{}-{} is Disconnected!", IPUtil.getIpV4(), ZookeeperClient.this.env, ZookeeperClient.this.serviceName);
            }
        }

        private void reConnected() {
            try {
                zookeeperClister.writeLock.lock();
                ZookeeperClient.this.destroy();
                firstInitChildren = new CountDownLatch(1);
                serverList = new CopyOnWriteArrayList<>();
                // 心跳服务列表
                serverHeartbeatMap = new ConcurrentHashMap<>();
                executor = Executors.newScheduledThreadPool(1);
                ZookeeperClient.this.initZooKeeper();
            } finally {
                zookeeperClister.writeLock.unlock();
            }
        }
    }

    public void destroy() {
        // help gc
        serverList = null;
        if (!executor.isShutdown()) {
            executor.shutdownNow();
            executor = null;
        }
        serverHeartbeatMap = null;
        if (zookeeper != null) {
            try {
                zookeeper.close();
                zookeeper = null;
            } catch (InterruptedException e) {
                LOG.error("the service 【{}】zk close faild", env.concat(serviceName));
            }
        }
    }

    private clreplaced HeartbeatRun implements Runnable {

        @Override
        public void run() {
            try {
                zookeeperClister.writeLock.lock();
                if (serverHeartbeatMap != null && serverHeartbeatMap.size() > 0) {
                    Iterator<String> key = serverHeartbeatMap.keySet().iterator();
                    in: while (key.hasNext()) {
                        String str = key.next();
                        String ip = str.split("-")[0];
                        String port = str.split("-")[1];
                        if (serverList != null) {
                            for (int i = serverList.size() - 1; i >= 0; i--) {
                                RemoteServer remoteServer = serverList.get(i);
                                if (remoteServer.getIp().equals(ip) && remoteServer.getPort().equals(port)) {
                                    if (!remoteServer.isEnable()) {
                                        continue in;
                                    }
                                }
                            }
                        }
                        HeartbeatService.Client client = serverHeartbeatMap.get(str);
                        HeartBeat heartBeat = new HeartBeat();
                        heartBeat.setIp(IPUtil.getIpV4());
                        heartBeat.setServiceName(ZookeeperClient.this.serviceName);
                        heartBeat.setDate(new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(new Date()));
                        int retry = 3;
                        while (retry-- > 0) {
                            try {
                                HeartBeat respone = client.getHeartBeat(heartBeat);
                                LOG.info("HeartBeat info:ip:{},serviceName:{}", respone.getIp(), serviceName);
                                break;
                            } catch (Exception e) {
                                if (retry == 0) {
                                    LOG.warn("HeartBeat error:{}", heartBeat);
                                    if (serverList != null) {
                                        for (int i = serverList.size() - 1; i >= 0; i--) {
                                            RemoteServer remoteServer = serverList.get(i);
                                            if (remoteServer.getIp().equals(ip) && remoteServer.getPort().equals(port)) {
                                                try {
                                                    serverList.remove(i);
                                                    key.remove();
                                                    if (zookeeperClister.serverPollMap.containsKey(str)) {
                                                        GenericObjectPool<TTransport> transport = zookeeperClister.serverPollMap.get(str);
                                                        zookeeperClister.serverPollMap.remove(str);
                                                        transport.close();
                                                    }
                                                    continue in;
                                                } catch (Exception e1) {
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            } finally {
                zookeeperClister.writeLock.unlock();
            }
        }
    }
}

See More Examples