db = DuckDBClient.of({
  summary: FileAttachment("results/summary.parquet"),
  stats: FileAttachment("results/stats.parquet"),
  basics: {file: FileAttachment("dataset_basics.csv"), header: true},
  algorithm_basics: {file: FileAttachment("algorithms_basics.csv"), header: true}
});
k_values = db.sql`select distinct k from summary;`
dataset_catalog_rows = db.sql`
  select distinct dataset, type
  from basics natural join (select distinct dataset from summary)
`;
dataset_catalog = Array.from(dataset_catalog_rows)
  .sort((a, b) => {
    const aAdditional = a.type.endsWith("-additional");
    const bAdditional = b.type.endsWith("-additional");
    return d3.ascending(aAdditional, bAdditional) || d3.ascending(a.dataset, b.dataset);
  });
default_datasets = dataset_catalog
  .filter((d) => !d.type.endsWith("-additional"))
  .map((d) => d.dataset);
dataset_types1 = dataset_catalog.reduce((map, d) => {
  map[d.dataset] = d.type.replace(/-additional$/, "");
  return map;
}, {});

// Colour encoding shared by the dataset selector and the radar plots.
type_color = (t) => (t == "in-distribution") ? "#1f77b4" : "#ff7f0e";
swatch = (t) => html`<span style="
    display: inline-block;
    width: 1.4ex;
    height: 1.4ex;
    flex: none;
    border-radius: 2px;
    background: ${type_color(t)};
  "></span>`;
swatch_label = (t, text) => html`<span style="
    display: inline-flex; align-items: center; gap: 0.6ex;
  ">${swatch(t)}${text}</span>`;

viewof recall_threshold = Inputs.range([0,1], {step: 0.01, value: 0.95, label: "minimum recall"});
viewof k_value = Inputs.select(k_values.map(d => d.k), {value: 10, label: "value of k"});
viewof selected_datasets = {
  const input = Inputs.checkbox(dataset_catalog.map((d) => d.dataset), {
    label: html`<span style="display: flex; flex-wrap: wrap; align-items: baseline; column-gap: 1.2ex; row-gap: 0.2rem;">datasets<span style="
        display: flex; flex-wrap: wrap; column-gap: 1.2ex; row-gap: 0.2rem;
        font-weight: 400; font-size: 0.85em;
      ">${swatch_label("in-distribution", "in-distribution")}${swatch_label("out-of-distribution", "out-of-distribution")}</span></span>`,
    format: (x) => swatch_label(dataset_types1[x], x),
    value: default_datasets
  });
  input.classList.add("dataset-selector");
  return input;
}
selected_dataset_lookup = `|${selected_datasets.join("|")}|`;

Overview of the results

These radar charts present a bird’s eye view of the results, at a fixed minimum recall level. For each dataset, we pick the fastest configuration of each algorithm reaching a recall higher than the selected minimum. Each circle in the plot below depicts the performance of a single algorithm across all selected datasets, where each dataset corresponds to a radial spine. Along a spine, the performance of a given algorithm is reported as the percentage of its queries-per-second (qps) with respect to the fastest algorithm on that dataset. Therefore, the farther from the center the better the performance.

As such, an algorithm whose shaded area covers a large portion of the circle has an overall good performance. The algorithms are then arranged, in row major order, by the average rank they achieve across all selected datasets.

computing…

fastdata = db.sql`
  with
    normalized_names as (
      select k, dataset, algorithm, params, qps, recall from summary
    ),
    selected_datasets as (
      select dataset, regexp_replace(type, '-additional$', '') as type
      from basics
      where contains(${selected_dataset_lookup}, '|' || dataset || '|')
    ),
    filtered_summary as (
      select * from normalized_names natural join selected_datasets natural left join algorithm_basics
      where not is_gpu 
    ),
    all_datasets as ( select distinct dataset, type from filtered_summary ),
    all_algorithms as ( select distinct algorithm from filtered_summary ),
    expected_combinations as (
      select *
      from all_datasets cross join all_algorithms
    ),
    ranked as (
        select *, row_number() over (partition by algorithm, dataset order by qps desc) as rank
        from filtered_summary
        where recall >= ${recall_threshold} and k = ${k_value}
    ),
    scaled as (
      select
        *,
        ifnull(qps, 0) / max(qps) over (partition by dataset, type) as scaled_qps
      from ranked
        natural right join expected_combinations
    ),
    best_performing as (
      select algorithm, dataset, max(scaled_qps) as best_performance
      from scaled
      group by all
    ),
    algorithms as (
      select algorithm, avg(best_performance) as mean_perf
      from best_performing
      group by all
    ),
    algorithms_ranks as (
      select
        algorithm, mean_perf,
        row_number() over (order by mean_perf desc) as algorithm_rank
      from algorithms
    ),
    datasets as (
      select dataset, type, avg(rc100) as difficulty
      from stats natural join selected_datasets
      group by all
    ),
    dataset_ranks as (
      select
        dataset, difficulty,
        row_number() over (order by type, contains(dataset, '-ip'), difficulty desc) as dataset_rank
      from datasets 
    )
  select
    algorithm,
    regexp_replace(dataset, '-[0-9]+-(cosine|normalized|euclidean|ip)', '') as dataset,
    type as dataset_type,
    dataset_rank,
    k,
    scaled_qps,
    qps,
    params,
    -- cast to INTEGER so that it reaches Observable as a number, not a BigInt
    cast(algorithm_rank as integer) as algorithm_rank
  from scaled natural join algorithms_ranks natural join dataset_ranks
  where rank = 1 or rank is null
  order by dataset_rank, algorithm, qps;
`

facet_keys = Array.from(d3.union(fastdata.map((d) => d.algorithm)));

dataset_ranks = fastdata.reduce((map, d) => {
  map[d.dataset] = d.dataset_rank;
  return map;
}, {});
dataset_types = fastdata.reduce((map, d) => {
  map[d.dataset] = d.dataset_type;
  return map;
}, {});

longitude_domain = Object.keys(dataset_ranks).sort((a, b) => dataset_ranks[a] - dataset_ranks[b]);

// Scales
longitude = d3.scalePoint(
  longitude_domain,
  [180, -180]
).padding(0.5).align(1);
fmt_qps = d3.format(".2s");
radar_width = Math.min(width, document.querySelector(".overview-radars").clientWidth);

// Grid geometry, recomputed whenever the pane is resized: fit as many radars
// per row as possible while keeping each cell wide enough to stay readable.
// Cell 0 is the legend (the empty facet carrying the full axis labels), so the
// grid has to hold one cell more than there are algorithms.
radar_cell_min_width = 200;
radar_cell_aspect = 1.1;
radar_cells = facet_keys.length + 1;
radar_cols = Math.max(2, Math.min(radar_cells - 1, Math.round(radar_width / radar_cell_min_width)));
radar_rows = Math.ceil(radar_cells / radar_cols);
radar_height = radar_rows * (radar_width / radar_cols) * radar_cell_aspect;

// Busy indicator: the pane is flagged as soon as one of the controls changes,
// and the flag is cleared again by the plot cell, once the recomputed data has
// made it all the way to a new figure.
radar_pane = document.querySelector(".overview-radars");
set_radar_busy = function(busy) {
  if (radar_pane) radar_pane.classList.toggle("is-computing", busy);
  return busy;
}
radar_busy = {
  // Depend on the controls (rather than on the query result) so that this runs
  // synchronously, while the DuckDB query behind `fastdata` is still pending.
  recall_threshold;
  k_value;
  selected_dataset_lookup;
  return set_radar_busy(true);
}

{
  radar_busy; // keep the "computing" overlay up until this figure is ready
  const plot = Plot.plot({
    width: radar_width,
    height: radar_height,
    marginTop: 20,
    marginBottom: 20,
    marginLeft: 20,
    marginRight: 20,
    projection: {
      type: "azimuthal-equidistant",
      rotate: [0, -90],
      // Note: 1.22° corresponds to max. percentage (1.0), plus some room for the labels
      domain: d3.geoCircle().center([0, 90]).radius(1.22)()
    },
    facet: {
      data: fastdata,
      y: (d) => Math.floor(d.algorithm_rank / radar_cols),
      x: (d) => d.algorithm_rank % radar_cols,
      axis: null
    },
    fx: {padding: .15},
    fy: {padding: .15},
    marks: [
      // Facet name
      Plot.text(fastdata,
        Plot.selectFirst({
            text: "algorithm",
            frameAnchor: "top",
            fontWeight: "700",
            fontSize: 18,
            dy: -15
        })
      ),

      // grey discs
      Plot.geo([1.0, 0.8, 0.6, 0.4, 0.2], {
        geometry: (r) => d3.geoCircle().center([0, 90]).radius(r)(),
        stroke: "black",
        fill: "black",
        strokeOpacity: 0.2,
        fillOpacity: 0.03,
        strokeWidth: 0.5
      }),

      // colored axes
      Plot.link(longitude.domain(), {
        x1: longitude,
        y1: 90 - 1,
        x2: 0,
        y2: 90,
        stroke: (d) => type_color(dataset_types[d]),
        strokeOpacity: 0.5,
        strokeWidth: 2.5
      }),

      // tick labels
      Plot.text([0.4, 0.6, 0.8], {
        fx: 0, fy: 0,
        x: 180,
        y: (d) => 90 - d,
        dx: 2,
        textAnchor: "start",
        text: (d) => ( d == 0.8 ? `${100 * d}%` : `${100 * d}%`),
        fill: "currentColor",
        stroke: "white",
        fontSize: 14
      }),

      // axes labels
      Plot.text(longitude.domain(), {
        fx: 0, fy: 0,
        x: longitude,
        y: 90 - 1.07,
        text: (d) => d.replaceAll("-", "\n"),
        lineWidth: 12,
        lineHeight: 1.05,
        fontSize: 12,
        fontWeight: 600
      }),

      // axes labels, initials
      Plot.text(longitude.domain(), {
        fx: 0, fy: 0, facet: "exclude",
        x: longitude,
        y: 90 - 1.15,
        text: d => d.slice(0,2),
        lineWidth: 5,
        lineHeight: 1.05,
        fontSize: 14,
        fontWeight: 600
      }),

      // areas
      Plot.area(fastdata, {
        x1: ({ dataset }) => longitude(dataset),
        y1: ({ scaled_qps }) => 90 - scaled_qps,
        x2: 0,
        y2: 90,
        fill: "gray",
        fillOpacity: 0.25,
        stroke: "gray",
        curve: "cardinal-closed"
      }),

      // data values
      Plot.dot(fastdata, {
        x: (d) => longitude(d.dataset),
        y: (d) => 90 - d.scaled_qps,
        fill: (d) => type_color(d.dataset_type),
        stroke: "white"
      }),

      // interactive labels
      Plot.text(
        fastdata,
        Plot.pointer({
          x: ({ dataset }) => longitude(dataset),
          y: ({ scaled_qps }) => 90 - scaled_qps,
          text: (d) => `${fmt_qps(d.qps)} qps\n(${Math.round(100 * d.scaled_qps)}%)\n${d.params}`,
          textAnchor: "start",
          dx: 4,
          fill: "currentColor",
          stroke: "white",
          maxRadius: 10,
          fontSize: 18
        })
      )
      
    ]
  });
  set_radar_busy(false);
  return plot;
}