3 Ways of Worldmaking: Mobile Android AR/VR



René Magritte, La Condition Humaine, 1935

Filippo Brunelleschi (1377-1446) with his mirror device 15th century


1. Augmented Reality

Beispiel Augmented Reality: Reactable



While the virtual elements in Reactable are limited to being displayed on a steady surface and can be seen as an enhanced user interface, it has recently become popular to relate augmented reality dynamically to three-dimensional space.

Beispiel Augmented Reality: Pool Live Aid



More than one reality?

Mixed Reality Scale after Milgram et. al., 1994


2 Mobile Augmented Reality

>

Beispiel Mobile Augmented Reality: Pokémon Go



Beispiel Mobile Augmented Reality: exTouch

Beispiel Mobile Augmented Reality: A/B

AR Overkill: Link



2. Mobile AR with Processing

>

Setup android mode

  • Processing: By using Processing`s contribution manager the Android Mode and Android SDK has to be installed
  • Android-Device: "Einstellungen/System/Info zum Telefon": tap on built-number 7 times to enter developer mode.
  • activate USB-Debugging: Link

Test the android mode


        void setup() {
          fullScreen();
          background(10,40,10);
          stroke(255);
        }
        
        void draw(){
          //version 1:
          line(pmouseX, pmouseY, mouseX, mouseY);
          
          //version 2:
          /*float heavyness = 30-((abs(mouseX-pmouseX))+(abs(mouseY-pmouseY)));
          heavyness = constrain(heavyness, 6, 30);
          strokeWeight(heavyness);
          stroke(150,mouseY,mouseX, 100);
          line(pmouseX, pmouseY, mouseX, mouseY);*/
        }
        

Use the phone`s camera


  import ketai.camera.*;

  KetaiCamera cam;

  void setup() {
  orientation(LANDSCAPE);
  imageMode(CENTER);
  cam = new KetaiCamera(this, 1280, 720, 24);
  }

  void draw() {
  background(255);
  image(cam, width/2, height/2);
  }

  void onCameraPreviewEvent()
  {
  cam.read();
  }

  // start/stop camera preview by tapping the screen
  void mousePressed()
  {
  if (cam.isStarted())
  {
  cam.stop();
  }
  else
  cam.start();
  }
  

- Import the ketai library in the library manager
- save your sketch
- in the menu "Android/Sketch Permissions" activate CAMERA!




    import ketai.camera.*;

    KetaiCamera cam;

    float resolutionX = 30;
    float resolutionY = 30;

    void setup() {
      orientation(LANDSCAPE);
      imageMode(CENTER);
      cam = new KetaiCamera(this, 1280, 720, 24);
    }

    void draw() {
      background(255);
      //image(cam, width/2, height/2);
      if(cam.isStarted()){
        image(cam, width/2, height/2);
        for(int y=0; y < height; y += resolutionY){ 
        for(int x=0; x < width; x += resolutionX){
          int pos = x + y * width;
          color c = cam.get(x,y);
          fill(c);
          noStroke();
          ellipse(x,y,resolutionX, resolutionY);
        }
      }
      }
    }

    void onCameraPreviewEvent()
    {
      cam.read();
    }

    // start/stop camera preview by tapping the screen
    void mousePressed()
    {
      if (cam.isStarted())
      {
        cam.stop();
      }
      else
        cam.start();
    }
  

Assignment 1
Implement your camera effect from the last exercise to the phone.



Find planes in (real) space


    import processing.ar.*;
    
    ARTracker tracker;
    
    void setup() {
      fullScreen(AR);
      tracker = new ARTracker(this);
      tracker.start();
      noStroke();
    }
    
    void draw() {
      lights();
      for (int i = 0; i < tracker.count(); i++) {
        ARTrackable t = tracker.get(i);
        pushMatrix();
        t.transform();
        float lx = t.lengthX();
        float lz = t.lengthZ();    
        fill(255, 100);
        beginShape(QUADS);
        vertex(-lx/2, 0, -lz/2);
        vertex(-lx/2, 0, +lz/2);
        vertex(+lx/2, 0, +lz/2);
        vertex(+lx/2, 0, -lz/2);
        endShape();
        popMatrix();  
      }  
    }
  


How to place an object on a plane


    import processing.ar.*;
    
    ARTracker tracker;
    ARAnchor anchor;
    
    float angle;
    
    void setup(){
      fullScreen(AR);
      
      tracker = new ARTracker(this);
      tracker.start();
      fill(255);
    
    }
    
    void draw(){
      lights();
      
      if (mousePressed == true){
        if (anchor != null){
        anchor.dispose();
      }
      
      ARTrackable hit = tracker.get(mouseX, mouseY);
      
      if (hit != null){
        anchor = new ARAnchor (hit);
      }
        else anchor = null;
      }
      
      if (anchor != null){
        anchor.attach();
        rotateY(radians(angle));
        box(0.1);
        anchor.detach();
      }
      
      angle = angle + 1;
    
    }

Textured AR-Objects

import processing.ar.*;
  
  ARTracker tracker;
  ARAnchor anchor;
  PImage map;
  PShape mapboard;
  
  void setup() {
    fullScreen(AR);
    tracker = new ARTracker(this);
    tracker.start();
    map = loadImage("Austriacycleroutes.jpeg");
    mapboard = createShape(BOX, -0.48*2, -0.26*2, 0.01);
    mapboard.setTexture(map);
    noStroke();
  }
  
  void draw() {
    lights();
    
    if (mousePressed){
      //create new anchor at the current touch point
      if (anchor != null) anchor.dispose();
      ARTrackable hit = tracker.get(mouseX, mouseY);
      if (hit != null) anchor = new ARAnchor (hit);
        else anchor = null;
    }
    
    if (anchor != null){
      anchor.attach();
      shape(mapboard);
      anchor.detach();
    }
  }
    


4. AR with a laptop and a camera

Installing nyARToolkit



NyID Markers from ID 0 to 10. Made with Marker Generator

Marker Generator für NyARToolkit-Library.

Using the nyARtoolkit with markers:

import processing.video.*;
  import jp.nyatla.nyar4psg.*;		// import nyARToolkit library
  
  Capture cam;				// declare object for cam
  MultiMarker nya;			// declare object for nya-marker
  
  void setup() {
    size(640,480,P3D);
    cam=new Capture(this,640,480);	// initialize cam
              // set Multimarker
    nya=new MultiMarker(this,width,height,"data/camera_para.dat",NyAR4PsgConfig.CONFIG_PSG);
    
    // Data for marker from image-file or generated marker-data
    nya.addARMarker(loadImage("data/hiro.png"),16,25,80);
    //nya.addARMarker("data/patt.hiro",80);
    cam.start();				// start cam feed
    }
    
    void draw(){
    if (cam.available() !=true) {	// if cam hasn`t been started yet, go to start
        return;
    }
    cam.read();				// read image from cam 
    nya.detect(cam);			// analyze cam image for markers
    }
    


Mixed Reality with markers

import processing.video.*;
    import jp.nyatla.nyar4psg.*;
    
    Capture cam;
    MultiMarker nya;
    int amount = 10;
    color[] colors = new color[amount];              // declare Array for colors
    float[] rotation = new float[amount];            // declare Array for rotation
    PShape figure;
    
    void setup() {
      size(640,360,P3D);
      colorMode(HSB, 100);
      figure = loadShape("dummy_obj.obj");
      shapeMode(CENTER);
      //cam=new Capture(this,640,480);          // Internal Cam
      cam=new Capture(this,640,360, "Live! Cam Sync HD VF0770");
      nya=new MultiMarker(this,width,height,"data/camera_para.dat",NyAR4PsgConfig.CONFIG_PSG);
      
      // Add several NyID Markers by this for-loop --------------------------
      for (int i=0; i < amount; i++){
        nya.addNyIdMarker(i,80);                      // add Marker
        colors[i] = color(i*2+75, 100,100, 40);      // add color for linked object
        rotation[i] = random(TWO_PI);              // add rotation for linked object
      }
      cam.start();
      lights();
    }
    
    void draw()
    {
      if (cam.available() !=true) {
          return;
      }
      cam.read();
      nya.detect(cam);
      background(0);
      nya.drawBackground(cam);
      
      // Add transformations for all the markers in this for-loop --------------------------
      for (int i=0; i < amount; i++){
        if((nya.isExist(i))){            // test, if a marker is detected
          nya.beginTransform(i);          // start drawing in 3D of the detected marker
            figure.setFill(colors[i]);    // set color of object
            rotateX(PI/2);
            rotateY(rotation[i]);        // set rotation of object
            translate(100,80);          // center it on marker
            shape(figure, 0, 0);        // draw object
          nya.endTransform();          // end drawing in 3D of the marker
        }
      }  
    }
    
    


AR with generative objects



import processing.video.*;
    import jp.nyatla.nyar4psg.*;
    
    Capture cam;
    MultiMarker nya;
    
    int amount = 11;
    
    Blossom[] blossoms = new Blossom[amount];
    
    float theta = 0.0;
    int totalShapes = 8;
    int totalPoints = 50;
    float scaling = random(20, 40);
    
    float[] pointX = new float[totalPoints];
    float[] pointY = new float[totalPoints];
    float[] pointZ = new float[totalPoints];
    
    void setup() {
      size(640,360,P3D);
      colorMode(HSB);
      //cam=new Capture(this,640,480);          // Internal Cam
      cam=new Capture(this,640,360, "Live! Cam Sync HD VF0770");
      nya=new MultiMarker(this,width,height,"data/camera_para.dat",NyAR4PsgConfig.CONFIG_PSG);
      for (int i=0; i < amount; i++){
        nya.addNyIdMarker(i,80);
        blossoms[i] = new Blossom();
        blossoms[i].setPoints();
      }
      cam.start();
      lights();
    }
    
    void draw()
    {
      if (cam.available() !=true) {
          return;
      }
      cam.read();
      nya.detect(cam);
      nya.drawBackground(cam);
    
      for (int i=0; i < amount; i++){
        if((nya.isExist(i))){
          nya.beginTransform(i);
            stroke(0,100);
            strokeWeight(6);
            line(0,0,0,0,0,blossoms[i].getStemHeight());
            translate(0, 0, blossoms[i].getStemHeight());
            blossoms[i].rotateObject();
            
            rotateX(blossoms[i].getRotation());
            rotateY(blossoms[i].getRotation());
            blossoms[i].drawObject();
          nya.endTransform();
        }
      }
    }
    
    
    
    


class Blossom {
      float theta = 0.0;
      float rotationSpeed = random(0.001, 0.02);
      float stemHeight = random(100,300);
      int totalPoints = 40;
      float scaling = random(20, 40);
      
      float[] pointX = new float[totalPoints];
      float[] pointY = new float[totalPoints];
      float[] pointZ = new float[totalPoints];
      
     Blossom(){
       
     }
      
     void setPoints(){
      for(int i = 0; i < totalPoints ; i ++){
        pointX[i] = randomGaussian()*scaling;
        pointY[i] = randomGaussian()*scaling;
        pointZ[i] = randomGaussian()*scaling;
      } 
     }
      
     void drawObject(){
      beginShape(TRIANGLES);
      for(int i = 0; i < totalPoints-2 ; i ++){
        fill(205+i, 200, 200, 150);
        strokeWeight(2);
        stroke(255,80);
        vertex(pointX[i], pointY[i], pointZ[i]);
        vertex(pointX[i+1], pointY[i+1], pointZ[i+1]);
        vertex(pointX[i+2], pointY[i+2], pointZ[i+2]);
      }
      endShape(); 
     }
     
     void rotateObject(){
        theta += rotationSpeed;
     }
     
     float getStemHeight(){
        return stemHeight;
     }
     
     float getRotation(){
        return theta;
     }  
    }
    
    

Assignment 2 "Virtual Exhibition"
Use the nyARtoolkit library and markers to make an virtual exhibition. Topic of the exhibition could be your own paintings, your favourite photos, 3D Obj-Files of your product designs or generative virtual objects.