bokeh.io.export_png

def save_plot(plot, name: str):
    base = Path('test-outputs')
    path = base / Path(name)
    path.parent.mkdir(exist_ok=True, parents=True)
    suf = path.suffix
    if suf == '.html':
        from bokeh.io import output_file, save
        output_file(str(path), title='hello', mode='inline', root_dir=None)
        save(plot)
    elif suf == '.png':
        # todo sigh.. seems that png export is way too slow
        from bokeh.io import export_png
        export_png(plot, filename=str(path))
    else:
        raise RuntimeError(name, suf)


def make_test(plot_factory):

    def save_picture(self):
        # export the graph
        export_png(self.graphs_layout, filename="graph.png")

def plot_all(limit=None):
    from bokeh.io import export_png, export_svgs, save
    # todo add min/max date?
    # todo multiple threads?

    # todo extract this mock data into HPI
    # use some python library to generate it
    real = True
    if real:
        df = locations_dataframe()
    else:
        # todo move to hpi or something
        idf = pd.DataFrame([{
            'dt': datetime.strptime('20200101', '%Y%m%d') + timedelta(minutes=30 * x),
            'lat': max((0.01 * x) % 90, 0.1),
            'lon': max((0.01 * x) % 90, 0.1),
        } for x in range(1, 1000)])
        df = idf.set_index('dt')

    # todo make defensive, collect errors
    def process(day_and_grp):
        day, grp = day_and_grp
        # todo uhoh. chromedriver might die?
        days = day.strftime('%Y%m%d')

        fname = f'output/{days}.png'

        # TODO shit. 20170402 -- float division by zero..
        p = plot(day=days, df=grp)
        if days   <  = '20170403':
            print(f'skipping {fname}')
            return

        print(f'saving {fname}')
        if True:
            export_png (p, filename=fname)
        else:
            # hmm, this doesn't produce the background (map). but faster to dump?
            p.output_backend = 'svg'
            export_svgs(p, filename=fname)

    inputs = [(day, grp) for day, grp in df.groupby(lambda x: x.date())]

    # todo ugh. pretty sure it's resulting in race conditions... (probably because of shared chromedriver?) need to test it properly
    from concurrent.futures import ThreadPoolExecutor as Pool
    # todo and process pool executor just gets stuck?


    parallel = False

    if parallel:
        with Pool() as pool:
            for _ in pool.map(process, inputs):
                pass
    else:
        for _ in map(process, inputs):
            pass

    # [(key, grp) for key, grp in df[:10].set_index('dt')
    # for day in [
    #         '20171116',
    #         '20171117',
    #         '20171118',
    #         '20171119',
    # ]:
    #     plot(day)
    # date_ = datetime.strptime(day, '%Y%m%d').date()
    # points = [l for l in locs if l.dt.date() == date_]


# pip3 install --user geoviews
# also see https://scitools.org.uk/cartopy/docs/latest/installing.html#requirements
# in my case had to
# - apt install proj-bin proj-data.
#   Possibly enough to install libproj15 and libproj-dev?
# - apt install libgeos-dev

    def combine(self, runs: List[Dict], img_save_path: str):
        """
        Combine runs from several files.

        :param paths: The paths to the runs to combine.
        """
        output_file('combined_plots.html')
        plot = figure(title="Balances & Accuracy on Hidden Test Set", )
        plot.width = 800
        plot.height = 800

        plot.xaxis.axis_label = "Time (days)"
        plot.yaxis.axis_label = "Percent"
        plot.title.text_font_size = '20pt'
        plot.xaxis.major_label_text_font_size = '16pt'
        plot.xaxis.axis_label_text_font_size = '16pt'
        plot.yaxis.major_label_text_font_size = '16pt'
        plot.yaxis.axis_label_text_font_size = '16pt'

        plot.xaxis[0].ticker = AdaptiveTicker(base=5 * 24 * 60 * 60)
        plot.xgrid[0].ticker = AdaptiveTicker(base=24 * 60 * 60)

        # JavaScript code.
        plot.xaxis[0].formatter = FuncTickFormatter(code="""
                return (tick / 86400).toFixed(0);
                """)
        plot.yaxis[0].formatter = PrintfTickFormatter(format="%0.1f%%")

        # TODO Make plot wider (or maybe it's ok for the paper).

        good_colors = cycle([
            colors.named.green,
            colors.named.lawngreen,
            colors.named.darkgreen,
            colors.named.limegreen,
        ])
        bad_colors = cycle([
            colors.named.red,
            colors.named.darkred,
            colors.named.orangered,
            colors.named.indianred,
        ])
        accuracy_colors = cycle([
            colors.named.blue,
            colors.named.cadetblue,
            colors.named.cornflowerblue,
            colors.named.darkblue,
        ])
        baseline_accuracy_colors = cycle([
            colors.named.black,
            colors.named.darkgrey,
            colors.named.slategrey,
            colors.named.darkslategrey,
        ])
        line_dashes = cycle([
            'solid',
            'dashed',
            'dotted',
            'dotdash',
            'dashdot',
        ])

        legend = []

        for run in runs:
            name = run['name']
            path = run['path']
            line_dash = next(line_dashes)
            self._logger.info("Opening \"%s\".", path)
            with open(path) as f:
                data = json.load(f)
                baseline_accuracy = data['baselineAccuracy']
                if baseline_accuracy is not None:
                    self._logger.debug("Baseline accuracy: %s", baseline_accuracy)
                    r = plot.ray(x=[0], y=[baseline_accuracy * 100], length=0, angle=0, line_width=2,
                                 line_dash=line_dash,
                                 color=next(baseline_accuracy_colors))
                    legend.append((f"{name} accuracy when trained with all data: {baseline_accuracy * 100:0.1f}%", [r]))
                agents: Dict[str, Agent] = dict()
                for agent in data['agents']:
                    agent = Agent(**agent)
                    agents[agent.address] = agent
                l = plot.line(x=[d['t'] for d in data['accuracies']],
                              y=[d['accuracy'] * 100 for d in data['accuracies']],
                              line_dash=line_dash,

                              line_width=2,
                              color=next(accuracy_colors),
                              )
                legend.append((f"{name} Accuracy", [l]))
                agent_balance_data = defaultdict(list)
                for balance_data in data['balances']:
                    agent = balance_data['a']
                    agent_balance_data[agent].append(
                        (balance_data['t'], balance_data['b'] * 100 / agents[agent].start_balance))
                for agent_id, balance_data in sorted(agent_balance_data.items(), key=itemgetter(0)):
                    agent = agents[agent_id]
                    if agent.good:
                        color = next(good_colors)
                    else:
                        color = next(bad_colors)
                    l = plot.line(x=list(map(itemgetter(0), balance_data)),
                                  y=list(map(itemgetter(1), balance_data)),
                                  line_dash=line_dash,
                                  line_width=2,
                                  color=color,
                                  )
                    legend.append((f"{name} {agent.address} Agent Balance", [l]))
        self._logger.info("Done going through runs.")

        legend = Legend(items=legend, location='center_left')
        plot.add_layout(legend, 'above')
        plot.legend.label_text_font_size = '12pt'

        self._logger.info("Saving image to: %s", img_save_path)
        export_png(plot, img_save_path)


if __name__ == '__main__':

    def save_plot_image(self, plot, plot_save_path):
        try:
            export_png(plot, filename=plot_save_path)
        except Exception as e:
            if self._warned_about_saving_plot:
                return
            show_error_details = True
            message = "Could not save picture of the plot."
            try:
                # Check if in WSL.
                show_error_details = not ('microsoft' in uname().release.lower())
            except:
                pass
            if show_error_details:
                self._logger.exception(message, exc_info=e)
            else:
                self._logger.warning(f"{message} %s", e)
            self._warned_about_saving_plot = True

    def simulate(self,

def export_plots(p: bokeh.plotting.figure,
                 filename: str,
                 title: str,
                 width: int = WIDTH,
                 height: int = HEIGHT,
                 box: bool = False,
                 show_title: bool = False,
                 y_range_start: Optional[float] = None,
                 y_range_end: Optional[float] = None) -> None:
    # HTML
    if not show_title:
        p.title = None
    bokeh.plotting.save(p, title=title, filename=filename + ".html", resources=bokeh.resources.CDN)

    # PNG
    if y_range_start:
        p.y_range.start = y_range_start
    if y_range_end:
        p.y_range.end = y_range_end

    set_font_size(p)
    p.sizing_mode = "fixed"
    p.width = width
    if box:
        p.height = width
    else:
        p.height = height
    p.toolbar_location = None
    bokeh.io.export_png(p, filename=filename + ".png", height=HEIGHT, width=WIDTH)

    # SVG:
    # p.output_backend = "svg"
    # bokeh.io.export_svgs(p, filename=filename + ".svg")
    #
    # os.system(f"inkscape --without-gui --export-pdf={filename}.pdf {filename}.svg")


def box_whiskers_plot(df: pd.DataFrame, out_folder: str, statistic: str = "ll", subtitle: str = "") -> None:

def export_plots(p: bokeh.plotting.figure,
                 filename: str,
                 title: str,
                 width: int = WIDTH,
                 height: int = HEIGHT,
                 box: bool = False,
                 show_title: bool = False,
                 y_range_start: Optional[float] = None,
                 y_range_end: Optional[float] = None,
                 x_range_start: Optional[float] = None,
                 x_range_end: Optional[float] = None) -> None:
    # HTML
    if not show_title:
        p.title = None
    bokeh.plotting.save(p, title=title, filename=filename + ".html", resources=bokeh.resources.CDN)

    # PNG
    if y_range_start:
        p.y_range.start = y_range_start
    if y_range_end:
        p.y_range.end = y_range_end
    if x_range_start:
        p.x_range.start = x_range_start
    if x_range_end:
        p.x_range.end = x_range_end

    set_font_size(p)
    p.sizing_mode = "fixed"
    p.width = width
    p.height = height
    if box:
        p.width = height
    p.toolbar_location = None
    bokeh.io.export_png(p, filename=filename + ".png", height=HEIGHT, width=WIDTH)

    # SVG:
    # p.output_backend = "svg"
    # bokeh.io.export_svgs(p, filename=filename + ".svg")
    #
    # os.system(f"inkscape --without-gui --export-pdf={filename}.pdf {filename}.svg")

def visualize_sentences(vecs, sentences, palette="Viridis256", filename="/notebooks/embedding/sentences.png",
                        use_notebook=False):
    tsne = TSNE(n_components=2)
    tsne_results = tsne.fit_transform(vecs)
    df = pd.DataFrame(columns=['x', 'y', 'sentence'])
    df['x'], df['y'], df['sentence'] = tsne_results[:, 0], tsne_results[:, 1], sentences
    source = ColumnDataSource(ColumnDataSource.from_df(df))
    labels = LabelSet(x="x", y="y", text="sentence", y_offset=8,
                      text_font_size="12pt", text_color="#555555",
                      source=source, text_align='center')
    color_mapper = LinearColorMapper(palette=palette, low=min(tsne_results[:, 1]), high=max(tsne_results[:, 1]))
    plot = figure(plot_width=900, plot_height=900)
    plot.scatter("x", "y", size=12, source=source, color={'field': 'y', 'transform': color_mapper}, line_color=None, fill_alpha=0.8)
    plot.add_layout(labels)
    if use_notebook:
        output_notebook()
        show(plot)
    else:
        export_png(plot, filename)
        print("save @ " + filename)


"""

def visualize_homonym(homonym, tokenized_sentences, vecs, model_name, palette="Viridis256",
                      filename="/notebooks/embedding/homonym.png", use_notebook=False):
    # process sentences
    token_list, processed_sentences = [], []
    for tokens in tokenized_sentences:
        token_list.extend(tokens)
        sentence = []
        for token in tokens:
            if model_name == "bert":
                processed_token = token.replace("##", "")
            else:
                processed_token = token
            if token == homonym:
                processed_token = "\"" + processed_token + "\""
            sentence.append(processed_token)
        processed_sentences.append(' '.join(sentence))
    # dimension reduction
    tsne = TSNE(n_components=2)
    tsne_results = tsne.fit_transform(vecs[1:])
    # only plot the word representation of interest
    interest_vecs, idx = np.zeros((len(tokenized_sentences), 2)), 0
    for word, vec in zip(token_list, tsne_results):
        if word == homonym:
            interest_vecs[idx] = vec
            idx += 1
    df = pd.DataFrame(columns=['x', 'y', 'annotation'])
    df['x'], df['y'], df['annotation'] = interest_vecs[:, 0], interest_vecs[:, 1], processed_sentences
    source = ColumnDataSource(ColumnDataSource.from_df(df))
    labels = LabelSet(x="x", y="y", text="annotation", y_offset=8,
                      text_font_size="12pt", text_color="#555555",
                      source=source, text_align='center')
    color_mapper = LinearColorMapper(palette=palette, low=min(tsne_results[:, 1]), high=max(tsne_results[:, 1]))
    plot = figure(plot_width=900, plot_height=900)
    plot.scatter("x", "y", size=12, source=source, color={'field': 'y', 'transform': color_mapper},
                 line_color=None,
                 fill_alpha=0.8)
    plot.add_layout(labels)
    if use_notebook:
        output_notebook()
        show(plot)
    else:
        export_png(plot, filename)
        print("save @ " + filename)


def visualize_between_sentences(sentences, vec_list, palette="Viridis256",

def visualize_between_sentences(sentences, vec_list, palette="Viridis256",
                                filename="/notebooks/embedding/between-sentences.png",
                                use_notebook=False):
    df_list, score_list = [], []
    for sent1_idx, sentence1 in enumerate(sentences):
        for sent2_idx, sentence2 in enumerate(sentences):
            vec1, vec2 = vec_list[sent1_idx], vec_list[sent2_idx]
            if np.any(vec1) and np.any(vec2):
                score = cosine_similarity(X=[vec1], Y=[vec2])
                df_list.append({'x': sentence1, 'y': sentence2, 'similarity': score[0][0]})
                score_list.append(score[0][0])
    df = pd.DataFrame(df_list)
    color_mapper = LinearColorMapper(palette=palette, low=np.max(score_list), high=np.min(score_list))
    TOOLS = "hover,save,pan,box_zoom,reset,wheel_zoom"
    p = figure(x_range=sentences, y_range=list(reversed(sentences)),
                x_axis_location="above", plot_width=900, plot_height=900,
                toolbar_location='below', tools=TOOLS,
                tooltips=[('sentences', '@x @y'), ('similarity', '@similarity')])
    p.grid.grid_line_color = None
    p.axis.axis_line_color = None
    p.axis.major_tick_line_color = None
    p.axis.major_label_standoff = 0
    p.xaxis.major_label_orientation = 3.14 / 3
    p.rect(x="x", y="y", width=1, height=1,
            source=df,
            fill_color={'field': 'similarity', 'transform': color_mapper},
            line_color=None)
    color_bar = ColorBar(ticker=BasicTicker(desired_num_ticks=5),
                        color_mapper=color_mapper, major_label_text_font_size="7pt",
                        label_standoff=6, border_line_color=None, location=(0, 0))
    p.add_layout(color_bar, 'right')
    if use_notebook:
        output_notebook()
        show(p)
    else:
        export_png(p, filename)
        print("save @ " + filename)


def visualize_self_attention_scores(tokens, scores, filename="/notebooks/embedding/self-attention.png",

def visualize_self_attention_scores(tokens, scores, filename="/notebooks/embedding/self-attention.png",
                                    use_notebook=False):
    mean_prob = np.mean(scores)
    weighted_edges = []
    for idx_1, token_prob_dist_1 in enumerate(scores):
        for idx_2, el in enumerate(token_prob_dist_1):
            if idx_1 == idx_2 or el   <   mean_prob:
                weighted_edges.append((tokens[idx_1], tokens[idx_2], 0))
            else:
                weighted_edges.append((tokens[idx_1], tokens[idx_2], el))
    max_prob = np.max([el[2] for el in weighted_edges])
    weighted_edges = [(el[0], el[1], (el[2] - mean_prob) / (max_prob - mean_prob)) for el in weighted_edges]

    G = nx.Graph()
    G.add_nodes_from([el for el in tokens])
    G.add_weighted_edges_from(weighted_edges)

    plot = Plot(plot_width=500, plot_height=500,
                x_range=Range1d(-1.1, 1.1), y_range=Range1d(-1.1, 1.1))
    plot.add_tools(HoverTool(tooltips=None), TapTool(), BoxSelectTool())

    graph_renderer = from_networkx(G, nx.circular_layout, scale=1, center=(0, 0))

    graph_renderer.node_renderer.data_source.data['colors'] = Spectral8[:len(tokens)]
    graph_renderer.node_renderer.glyph = Circle(size=15, line_color=None, fill_color="colors")
    graph_renderer.node_renderer.selection_glyph = Circle(size=15, fill_color="colors")
    graph_renderer.node_renderer.hover_glyph = Circle(size=15, fill_color="grey")

    graph_renderer.edge_renderer.data_source.data["line_width"] = [G.get_edge_data(a, b)['weight'] * 3 for a, b in
                                                                   G.edges()]
    graph_renderer.edge_renderer.glyph = MultiLine(line_color="#CCCCCC", line_width={'field': 'line_width'})
    graph_renderer.edge_renderer.selection_glyph = MultiLine(line_color="grey", line_width=5)
    graph_renderer.edge_renderer.hover_glyph = MultiLine(line_color="grey", line_width=5)

    graph_renderer.selection_policy = NodesAndLinkedEdges()
    graph_renderer.inspection_policy = EdgesAndLinkedNodes()

    plot.renderers.append(graph_renderer)

    x, y = zip(*graph_renderer.layout_provider.graph_layout.values())
    data = {'x': list(x), 'y': list(y), 'connectionNames': tokens}
    source = ColumnDataSource(data)
    labels = LabelSet(x='x', y='y', text='connectionNames', source=source, text_align='center')
    plot.renderers.append(labels)
    plot.add_tools(SaveTool())
    if use_notebook:
        output_notebook()
        show(plot)
    else:
        export_png(plot, filename)
        print("save @ " + filename)


def visualize_words(words, vecs, palette="Viridis256", filename="/notebooks/embedding/words.png",

def visualize_words(words, vecs, palette="Viridis256", filename="/notebooks/embedding/words.png",
                    use_notebook=False):
    tsne = TSNE(n_components=2)
    tsne_results = tsne.fit_transform(vecs)
    df = pd.DataFrame(columns=['x', 'y', 'word'])
    df['x'], df['y'], df['word'] = tsne_results[:, 0], tsne_results[:, 1], list(words)
    source = ColumnDataSource(ColumnDataSource.from_df(df))
    labels = LabelSet(x="x", y="y", text="word", y_offset=8,
                      text_font_size="15pt", text_color="#555555",
                      source=source, text_align='center')
    color_mapper = LinearColorMapper(palette=palette, low=min(tsne_results[:, 1]), high=max(tsne_results[:, 1]))
    plot = figure(plot_width=900, plot_height=900)
    plot.scatter("x", "y", size=12, source=source, color={'field': 'y', 'transform': color_mapper}, line_color=None,
                 fill_alpha=0.8)
    plot.add_layout(labels)
    if use_notebook:
        output_notebook()
        show(plot)
    else:
        export_png(plot, filename)
        print("save @ " + filename)


def visualize_between_words(words, vecs, palette="Viridis256", filename="/notebooks/embedding/between-words.png",

def visualize_between_words(words, vecs, palette="Viridis256", filename="/notebooks/embedding/between-words.png",
                            use_notebook=False):
    df_list = []
    for word1_idx, word1 in enumerate(words):
        for word2_idx, word2 in enumerate(words):
            vec1 = vecs[word1_idx]
            vec2 = vecs[word2_idx]
            if np.any(vec1) and np.any(vec2):
                score = cosine_similarity(X=[vec1], Y=[vec2])
                df_list.append({'x': word1, 'y': word2, 'similarity': score[0][0]})
    df = pd.DataFrame(df_list)
    color_mapper = LinearColorMapper(palette=palette, low=1, high=0)
    TOOLS = "hover,save,pan,box_zoom,reset,wheel_zoom"
    p = figure(x_range=list(words), y_range=list(reversed(list(words))),
               x_axis_location="above", plot_width=900, plot_height=900,
               toolbar_location='below', tools=TOOLS,
               tooltips=[('words', '@x @y'), ('similarity', '@similarity')])
    p.grid.grid_line_color = None
    p.axis.axis_line_color = None
    p.axis.major_tick_line_color = None
    p.axis.major_label_standoff = 0
    p.xaxis.major_label_orientation = 3.14 / 3
    p.rect(x="x", y="y", width=1, height=1,
           source=df,
           fill_color={'field': 'similarity', 'transform': color_mapper},
           line_color=None)
    color_bar = ColorBar(ticker=BasicTicker(desired_num_ticks=5),
                         color_mapper=color_mapper, major_label_text_font_size="7pt",
                         label_standoff=6, border_line_color=None, location=(0, 0))
    p.add_layout(color_bar, 'right')
    if use_notebook:
        output_notebook()
        show(p)
    else:
        export_png(p, filename)
        print("save @ " + filename)