Webcam Piano with Processing v0.1

This is the beginnings of a Processing / Java port of the webcam-to-osc/midi app I originally did in Quartz Composer. The source code for the processing version is below, and you can watch (or download) the Quartz Composer version here).

Its quite early days yet and doesn't have all the features I want (scales, realtime sizing of grid etc.), but I'm posting posting it because:
a.) it does work on a basic level,
b.) It was requested on the processing forums and I thought it might be useful...

It doesn't transmit midi, but does transmit OSC, and I'm using OSCulator to forward the OSC messages to midi. I prefer doing it this way because I can have another computer on wifi receive the OSC messages and map to midi (and send to Logic), keeping the CPU on both machines lighter... (or just keep the oscTargetIP as 127.0.0.1 to send the OSC to the same machine and have everything running on one machine. Flexibility is always sweet).

For some reason the applet doesn't work when published on an html page, probably something to do with video input, when I figure it out will post the applet. In the meantime the source code is below, and the original PDE and OSCD (OSCulator document) are attached.

Apologies for lack of comments etc, its early days. Any questions just ask!

/***********************************************************************
-----------------------------------
 
Copyright (c) 2008, Memo Akten, www.memo.tv
 
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
 
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
 
You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
***********************************************************************/	
 
import oscP5.*;
import netP5.*;
import processing.video.*;
import controlP5.*;
 
/**************************** CONSTANTS & PARAMS **********************************/
int oscTargetPort = 8000;
String oscTargetIP = "127.0.0.1";
 
int numGridX = 15;
int numGridY = 4;
 
float gridSpacing = 0.1;      // spacing of squares 
 
float vidMult = 0.5;        // ratio of camera resolution to output res
 
int fps = 30;
 
float triggerThreshold = 0.05;
float velocityMult = 5;
 
 
/**************************** VARS ***********************************/
OscP5 oscP5;
NetAddress address;
ControlP5 controlP5;
 
int numPixels;
int[] prevGrey;
Capture video;
 
float totalMovement;
 
int gridSizeX;
int gridSizeY;
 
float gridMult = 1 / vidMult;
 
float[][] gridInfo;
 
float maxTimeDiff = 5;  // trigger once every 5 seconds 
 
PImage img;
 
 
/**************************** SETUP ***********************************/
 
void setup() {
  size(640, 480); 
  video = new Capture(this, (int) (width * vidMult), (int) (height * vidMult), fps);
  numPixels = video.width * video.height;
  gridSizeX = video.width / numGridX;
  gridSizeY = video.height / numGridY;
 
  prevGrey = new int[numPixels];
  gridInfo = new float[numGridY][numGridX];
 
  img = createImage(video.width, video.height, RGB);
 
  oscP5 = new OscP5(this, oscTargetPort);
  address = new NetAddress(oscTargetIP, oscTargetPort);
 
  initApp();
 
  frameRate(fps);
}
 
void initApp() {
  int sliderWidth = (int) (width * 0.4);
  controlP5 = new ControlP5(this);
  controlP5.addSlider("triggerThreshold", 0, 1, triggerThreshold, 20, 20, sliderWidth, 15);
  controlP5.addSlider("velocityMult", 1, 20, velocityMult, 20, 40, sliderWidth, 15);
}
 
 
/**************************** UPDATE ***********************************/
void draw() {
  if (video.available()) {
    initGridInfo();
 
    video.read(); // Read the new frame from the camera
    video.loadPixels();
    img.loadPixels();
 
    totalMovement = 0;
 
    image(video, 0, 0, width, height);
    for (int i=0; i<numPixels; i++) {
      int posX = i % video.width;
      int posY = floor(i / video.width);
      int gridX = floor(posX / gridSizeX);
      int gridY = floor(posY / gridSizeY);
      if(gridX >= numGridX) gridX = numGridX - 1;
      if(gridY >= numGridY) gridY = numGridY - 1;
      int gridNo  = gridY * numGridX + gridX;
 
      color curColor = video.pixels[i];
      int curR = (curColor >> 16) & 0xFF;
      int curG = (curColor >> 8) & 0xFF;
      int curB = curColor & 0xFF;
      // average RGB components (there are better ways of calculating intensity from RGB, but this will suffice for these purposes)
      int curGrey = (curR + curG + curB) / 3; 
      int diff = abs(curGrey - prevGrey[i]) ;
      //img.pixels[i] = 0xff000000 | (diff << 16) | (diff << 8) | diff;
 
      gridInfo[gridY][gridX] += diff;
      totalMovement += diff;
 
      prevGrey[i] = curGrey;
    }
 
    drawGrid();
    totalMovement /= numPixels * 256;
    OscMessage oscMessage = new OscMessage("/cam/movement");
    oscMessage.add(totalMovement);
    //if(totalMovement>triggerThreshold * 100) 
    oscP5.send(oscMessage, address); 
  }
}
 
void drawGrid() {
  noStroke();
  for(int y=0; y<numGridY; y++) {
    for(int x=0; x<numGridX; x++) {
      float gridMovement = gridInfo[y][x] / (gridSizeX * gridSizeY * 256); 
      if(gridMovement > triggerThreshold)  {
        fill(255, gridMovement * 250 + 50);
        OscMessage oscMessage = new OscMessage("/cam/note");
        oscMessage.add(true);
        oscMessage.add((y * numGridX + x) / (float)(numGridX * numGridY));
        oscMessage.add(gridMovement * velocityMult);
        oscP5.send(oscMessage, address); 
      }  
      else {
        fill(255, 20);
        OscMessage oscMessage = new OscMessage("/cam/note");
        oscMessage.add(false);
        oscP5.send(oscMessage, address); 
      }
 
      rect((x * gridSizeX + gridSizeX * gridSpacing/2) * gridMult, (y * gridSizeY + gridSizeY * gridSpacing/2) * gridMult, 
      gridSizeX * (1 - gridSpacing) * gridMult, gridSizeY * (1 - gridSpacing) * gridMult
        ); 
 
 
    }
  }
}
 
 
void initGridInfo() {
  for(int y=0; y<numGridY; y++) {
    for(int x=0; x<numGridX; x++) {
      gridInfo[y][x] = 0;
    }
  }
}

AttachmentSize
webcam_piano.pde4.93 KB
webcam_piano.oscd3.51 KB