Introduce the concept of seed truth for enrolling users about ajl.ai

[boazsender]

@gnarf suggested having one truth table called known with a column for seed versus consensus:

CREATE TABLE known (
  id SERIAL PRIMARY KEY,
  image_id INTEGER NOT NULL REFERENCES image(id),
  annotation_type_id INTEGER NOT NULL REFERENCES annotation_type(id),
  data JSONB NOT NULL,
  is_seed BOOLEAN NOT NULL DEFAULT false,
  UNIQUE(image_id, annotation_type_id)
);

[gnarf]

We can use this crazy sql (that tyler could probably optimize) to get the images for the workload.

-- First, select the total count of annotaiton types
WITH types as (
  SELECT
    COUNT(*) as count
  FROM
    annotation_type
),
-- How many known truths are present for each image
known_count as (
  SELECT
    image_id,
    COUNT(*) as count
  FROM
    known
  GROUP BY image_id
),
-- Each annotator should never see the same image twice
annotator as (
  SELECT 
    image_id,
    count(*) as count
  FROM image_annotation
  WHERE annotater_id = ${annotatorId}
  GROUP BY image_id
),
-- Look for images, calculate if we can use them as knowns
-- restrict the images we've annotated, and randomize order
truth_table as (
  SELECT
    image.id,
    image.url,
    image.width,
    image.height,
    COALESCE(known.count = types.count, false) as is_known
  FROM
    image
    LEFT JOIN known_count known on image.id = known.image_id
    LEFT JOIN annotator on image.id = annotator.image_id,
    types
  WHERE 
    annotator.count is null
  ORDER BY RANDOM()
),
-- Select the first "8" (2/3's of the limit) known images
truths as (
  SELECT * FROM truth_table WHERE is_known LIMIT ${numTruths}
),
-- Select a whole "limit" bucket of unknown images
news as (
  SELECT * FROM truth_table WHERE NOT(is_truth) LIMIT ${limit}
)
-- Select the union of the known images, and unknown images, limiting to
-- the total limit (so because we "overfilled" the buffer for the unknowns)
-- it will always give us 12 images, with up to `numTruths` truths if we 
-- had them...
SELECT * FROM truths
UNION ALL
SELECT * FROM news
LIMIT ${limit};

[boazsender]

Creating some seed truths based on my own annotations:

  {"workloadId":275,"images":[{"id":690,"annotations":[{"name":"Perceived Age","option":"young adult"},{"name":"Perceived Gender","option":"female"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":3060,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"female"},{"name":"Perceived Ethnicity","option":"black"}]},{"id":1310,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"white"}]}]}

  {"workloadId":276,"images":[{"id":2668,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"female"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":3368,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"female"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":4070,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"black"}]}]}

  {"workloadId":277,"images":[{"id":2434,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"female"},{"name":"Perceived Ethnicity","option":"asian"}]},{"id":1863,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"female"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":1979,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"white"}]}]}

  {"workloadId":278,"images":[{"id":3116,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":2827,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"female"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":3528,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"not listed"}]}]}

  {"workloadId":279,"images":[{"id":826,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":1515,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"white"}]},{"id":3178,"annotations":[{"name":"Perceived Age","option":"adult"},{"name":"Perceived Gender","option":"male"},{"name":"Perceived Ethnicity","option":"white"}]}]}

[boazsender]

resolved by #78