Processing sketch for interactive artwork "Ascending/Descending", located at the US check-in area in the Halifax International Airport.

The sketch plays the video of an elavator door that remains closed until someone approaches the video and stands on a custom-made sensor.

The sketch listens to the serial port for a signal from an Arduino. The signal is sent when someone stands on a custom-made normally-open

switch that is concealed under a mat in front of the projection screen.

Considerations are being made to substitute the Arduino with a direct connection from the normally open switch to a key on a keyboard,

thus changing the video a keyPressed function.  

This sketch is possible because of the existing resources

http://processing.org/learning/topics/sequential.html: for playing the image of the elevator door opening as a sequence of images.

  This was a necessary alternative to loading the elevator door as a video. Loading it as a video required too much processing power

  and resulted on a vert slow performance of the sketch.

http://code.compartmental.net/tools/minim/: for 


// import the minim library for playing sounds

// and define objects for playing sounds

import ddf.minim.*;

AudioSnippet openingSound;

AudioSnippet closingSound;

AudioSnippet openSound;

AudioSnippet closedSound;

Minim minim;

//the following libraries and variables control position of the mouse 

import java.awt.Robot;

import java.awt.AWTException;

int xx = 10, yy = 10;

Robot robby;

//Import serial library to communicate with Arduino

import processing.serial.*;

// The following variables are for uploading the series of .jpgs

// needed to play the videos.

int numFrames_opening = 187;  // The number of frames in the animation

int numFrames_closed = 188;  // The number of frames in the animation

int numFrames_closing = 169;  // The number of frames in the animation

int numFrames_open = 47;  // The number of frames in the animation

int frame_opening = 0;

int frame_closed = 0;

int frame_closing = 0;

int frame_open = 0;

PImage[] images_opening = new PImage[numFrames_opening];

PImage[] images_closed = new PImage[numFrames_closed];

PImage[] images_closing = new PImage[numFrames_closing];

PImage[] images_open = new PImage[numFrames_open];

int fr = 30;// set framerate for reading the images

// width and height should be set here

int xWidth = 1280;

int yHeight = 800;

// these variables are for the serial port connection object

boolean output = true;

Serial port;

// String portname = "/dev/tty.usbserial-00002006"; This is not necessary anymore note later how the name of the usb is assigned

int baudrate = 9600;  //  set baudrate here

int value;  // variables used to store value from serial port

String buf=""; // String buffer to store serial values

int value1;  // value1 is the read value

// the serial event function takes the value of the event

// and store it in the corresponding variable

void serialEvent(int serial) {

  if(serial!=10) {        

    buf += char(serial);


  else {

    //extract the value from the string 'buf'

    //  buf = buf.substring(1,buf.length());

    //cast the value to an integer

    value1 = int(buf);




// this is the threshold for activating the animations; not needed

// here because this value was only useful for distance sensors

//int threshold = 800;

// The role of these booleans will be explained later

boolean openingSwitch = true;

boolean closingSwitch = true;

void setup() {

  //setup size and framerate

  size(xWidth, yHeight);


  //Initialize robot that contros the mouse



    robby = new Robot();


  catch (AWTException e)


    println("Robot class not supported by your system!");



// Begin communication with USB port

  println("Available serial ports:") // Begin by showing all available ports

  println(Serial.list()); // See which USB port is used by Arduino

  port = new Serial(this, Serial.list()[0], baudrate); //Communicate with that port



// Initialize Minim Library and load sounds as Snippets

  minim = new Minim(this);

  openingSound = minim.loadSnippet("opening.aif");

  openSound = minim.loadSnippet("open.aif");

  closingSound = minim.loadSnippet("closing.aif");

  closedSound = minim.loadSnippet("closed.aif");

// The following series of 'for' statements load all of the .jpgs used for 

// the videos. Not sure how everything works. Code appropriated from Processing

// Reference library.

// the nomenclature of the files is (title of action) + (number) + .jpg ... For example Elevator_opening003.jpg

  for(int i=0; i<numFrames_opening; i++) {

    String imageName_opening = "Elevator_opening" + nf(i, 3) + ".jpg"; // nf(i,3) determines somehow the amount of numbers in the image number on the file

                                                                       // for example, all "Elevator_opening" files have 3 numbers that go from 000 to 187

                                                                       // therefore the number "3"

    images_opening[i] = loadImage(imageName_opening);


  for(int i=0; i<numFrames_closed; i++) {

    String imageName_closed = "Elevator_closed" + nf(i, 3) + ".jpg";

    images_closed[i] = loadImage(imageName_closed);


  for(int i=0; i<numFrames_closing; i++) {

    String imageName_closing = "Elevator_closing" + nf(i, 3) + ".jpg";

    images_closing[i] = loadImage(imageName_closing);


  for(int i=0; i<numFrames_open; i++) {

    String imageName_open = "Elevator_open" + nf(i, 2) + ".jpg";      // all "Elevator_open" files only use 2 numbers, therefore the number in nf(i,2) is "2"

    images_open[i] = loadImage(imageName_open);



void draw() {

  // Use the robot to snap the mouse on location 

  robby.mouseMove(xx, yy);


  // draw listens to serial port, and if there is no response from an Arduino

  // it remains trapped in a while loop

  while(port.available() > 0) {

    value = port.read();



// Display the values received from the Arduino as stored in the variable value1

  if(output) println(value1);

//The following 'if' statements pay attention to value1 to decide which videos to play/////////

  if (value1==1) { // if someone enters...

    closingSwitch = true; // Prepare this boolean for the closing part.

                          // This boolean as well as "openingSwitch" allows for the 

                          // closing and opening animations to occur.

                          // By stablishing it as "true" you're preparing it when someone exits or enter the sensor.

                          // The program will look for this boolean when someone enters after it checks 

                          // whether the animation is halfway closing or halfway opening.

                          // After it finishes the incomplete animation (if it has to) it begins playing the closing

                          // or opening action - depending on whether the person is off or on the mat.

                          // The boolean functions as a switch that turns to false when the animation finishes closing

                          // or opening. This then triggers the following action: while someone remains either on top or off

                          // the mat, the false boolean indicates that all animations are finished and that the loop of

                          // the door closed or open should come into effect.


    if (frame_closing>0) { // If the string of images if above 0, this means that the animation was halfway closing when someone entered the mat

      frame_closing = (frame_closing+1);  // continue loading string of images

      image(images_closing[frame_closing], 0,0,1280,800); // place the image

      if (frame_closing>167) { // when the animation reaches its last frame...

        closingSound.pause();//pause sound...

        closingSound.rewind(); //...rewind sound

        frame_closing = 0;  //... reset to 0 and make this if statement invalid



   // Begin opening sequence

    else if (frame_closing == 0 && openingSwitch == true) { //notice how it make sures that the closing sequence is at 0 and that the boolean is TRUE

      closedSound.pause(); //shuts down this sound

      openingSound.play(); // plays sound of opening door

      frame_opening = (frame_opening+1); // starts adding number to the string of images that make up this animation

      image(images_opening[frame_opening], 0,0,1280,800);// places the jpg

      if (frame_opening > 185) {// if it reaches this number of images, it means the animation is coming to an end.

        openingSound.pause(); // stop the sount

        openingSound.rewind(); // rewind it

        frame_opening = 0; // rewind the opening frame back to 0

        openingSwitch = false; // declare this boolean False to make if statement invalid



      // if the person remains within range


    else if (frame_opening == 0 && openingSwitch == false) { // note how it's looking for the boolean to be False

      if (openSound.isPlaying() == false)


        openSound.loop();// loop sound


      frame_open = (frame_open+1) % numFrames_open; // this line of code loops the images that compose the loop the door open, use % to cycle

      image(images_open[frame_open], 0,0,1280,800); // place the jpgs that make up this animation



//do the same for when value from the switch reads 0 (=no one is activating the sensor)

  else if (value1==0) {

    openingSwitch = true;

    if (frame_opening>0) {// check if the door is in the middle of opening, and finish doing that if necessary

      frame_opening = (frame_opening+1); 

      image(images_opening[frame_opening], 0,0,1280,800);

      if (frame_opening > 185) {



        frame_opening = 0;



    else if (frame_opening == 0 && closingSwitch == true) { //if door is open, start closing the door



      frame_closing = (frame_closing+1);  

      image(images_closing[frame_closing], 0,0,1280,800);

      if (frame_closing>167) {// if door closes: stop, rewind, and go to loop



        frame_closing = 0;

        closingSwitch = false;



    else if (frame_closing == 0 && closingSwitch == false) {// no one is within range, so play closed door loop.

      if (closedSound.isPlaying() == false)




      frame_closed = (frame_closed+1) % numFrames_closed;  // Use % to cycle through frames

      image(images_closed[frame_closed], 0,0,1280,800);




void stop()


  // always close Minim audio classes when you are done with them