• Read Computer Vision for Artists and Designers.
• Experiment with the provided code examples using ml5.js, TensorFlow.js, or Transformers.js. Try applying the models to different images and videos. Can you think of a creative way to integrate image segmentation into an interactive project?
• Document your experiments and share it on the Assignment 3 Wiki.

Body Segmentation + Image Segmentation:

So I finally got a simple idea of an application to do with ml body segmentation tools. I made a simple “red light green light” hand controller. Combining the bodySegmentation-mask-background and the bodySegmentation-select-body-parts scripts, I was able to use my hands to control the color of my background.

red-light-green-light.mov

/*
 * 👋 Hello! This is an ml5.js example made and shared with ❤️.
 * Learn more about the ml5.js project: <https://ml5js.org/>
 * ml5.js license and Code of Conduct: <https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md>
 *
 * This example demonstrates segmenting a person by body parts with ml5.bodySegmentation.
 */

let bodySegmentation;
let video;
let segmentation;
let segmentation2;
let rVal=255;
let gVal=255;
let bVal=255;

let options = {
  maskType: "parts",
  maskType: "background"
};

function preload() {
  bodySegmentation = ml5.bodySegmentation("BodyPix", options);
  bodySegmentation2 = ml5.bodySegmentation("SelfieSegmentation", options);
}

function setup() {
  createCanvas(640, 480);
  // Create the video
  video = createCapture(VIDEO);
  video.size(640, 480);
  video.hide();

  bodySegmentation.detectStart(video, gotResults);
  bodySegmentation2.detectStart(video, gotResults2);
}

function draw() {
  background(rVal,gVal,bVal);
  if (segmentation2) {
    video.mask(segmentation2.mask);
  }
  image(video, 0, 0);
  if (segmentation) {
    let parts = bodySegmentation.getPartsId();
    let gridSize = 10;
    for (let x = 0; x < video.width; x += gridSize) {
      for (let y = 0; y < video.height; y += gridSize) {
        if (segmentation.data[y * video.width + x] == parts.LEFT_HAND) {
          rVal=255;
          gVal=0;
          bVal=0;
          // fill(255, 0, 0);
          // noStroke();
          // circle(x, y, gridSize);
        }
        else if(segmentation.data[y * video.width + x] == parts.RIGHT_HAND){
          rVal=0;
          gVal=255;
          bVal=0;
          // fill(255, 0, 0);
          // noStroke();
          // circle(x, y, gridSize);
        } 
      }
    }
  }
}

// callback function for body segmentation
function gotResults(result) {
  segmentation = result;
}

// callback function for body segmentation
function gotResults2(result2) {
  segmentation2 = result2;
}