processing创建虚拟现实

Meta（元）现在很热门，虚拟与现实显得无法分隔。processing是一个很好的展示软件，这里我们用它与其他一些硬件软件结合来接通虚拟与现实，用两个案例展示一下。
1. Wii手柄+Arduino+processing创建虚拟现实：
1.1 任天堂的Wii游戏机的Wii Nunchuck手柄：
Wii Nunchuck手柄集成了1个3轴加速度传感器，其提供的I2C接口能方便的获取手柄上各个传感器的数据，包括2轴摇杆，2个按钮按钮，3轴加速度传感器等。使用Wii Nunchuck手柄和Arduino能做出非常酷的互动作品。WiiChuck转接板能将Wii手柄的I2C端口引出,从而使Wii Nunchuck手柄方便与Arduino连接而不需任何焊接和连线，不过这个WiiChuck转接板DF停产下架了，淘宝上应该还可以买到。这里我们用arduino使用sparrow主板获取加速度的值。WiiChuck转接板和Wii手柄连接的参考资料见：https://wiki.dfrobot.com.cn/_SKU_DFR0062_WiiChuck%E8%BD%AC%E6%8E%A5%E5%99%A8


1.2 手柄代码：
#include <WiiChuck.h>


#include <math.h>
#include <stdlib.h>
#include "Wire.h"
#include "WiiChuck.h"

WiiChuck wii = WiiChuck();

void setup()
{
wii.initWithPower();
Serial.begin(19200);
}

void loop()
{
  if (true == wii.read()) {

    Serial.print(wii.getAccelAxisX(), DEC);
    Serial.print(",");
    Serial.print(wii.getAccelAxisY(), DEC);
    Serial.print(",");
    Serial.print(wii.getAccelAxisZ(), DEC);
    Serial.print(" \n");


  }
  delay(100);

}

1.3 processing软件：
processing是一个方便的软件，通过写代码来画出图案和2D、3D动画，并且可以通过串口和其他硬件进行交互通讯。软件下载地址：https://processing.org/网站里面也有很多案例参考学习。这里我们接收wii手柄传过来的加速度值对创建的3d图形进行同步控制。注意com口和波特率与arduino里面保持一致。mac系统的com口跟windows不一样，按它在arduino中的显示去写。

1.4 processing软件代码：
/**
* Mixture
* by Simon Greenwold.
*
* Display a box with three different kinds of lights.
*/
import processing.serial.*;
int x = 0;
int y = 0;
int z = 0;
Serial myPort;
void setup() {
  size(640, 360, P3D);
  noStroke();
  myPort = new Serial(this,"/dev/cu.usbmodem1421",19200);
  myPort.bufferUntil('\n');
}
void serialEvent(Serial p){
  String inString =p.readString();
  String[] list =split(inString,',');
  x = int(list[0]);
  y = int(list[1]);
}



void draw() {
  background(0);
  translate(width / 2, height / 2);

  // Orange point light on the right
  pointLight(150, 100, 0, // Color
             200, -150, 0); // Position

  // Blue directional light from the left
  directionalLight(0, 102, 255, // Color
                   1, 0, 0); // The x-, y-, z-axis direction

  // Yellow spotlight from the front
  spotLight(255, 255, 109, // Color
            0, 40, 200, // Position
            0, -0.5, -0.5, // Direction
            PI / 2, 2); // Angle, concentration

  rotateY(map(x, 0, width, 0, PI));
  rotateX(map(y, 0, height, 0, PI));
  box(150);
}

1.5 演示：
随着手柄的动作3d立方体也跟着动作了。




2.视觉识别手的姿势然后在processing中3d模拟显示
2.1 用python视觉识别手的姿态：
import cv2
import mediapipe as mp
import time
import math


import  pty
import  os
import serial
import serial.tools.list_ports

def  mkpty ( ) :
    # 打开虚拟终端
    master1,  slave =  pty. openpty( )
    slaveName1 =  os. ttyname( slave)
    print( '虚拟设备名称: ' ,  slaveName1)
    #return  master1
    return master1,slaveName1

class handDetctor():
    def __init__(self, mode=False, maxHands=2, detectionCon=0.5, trackCon=0.5):
        self.mode = mode
        self.maxHands = maxHands
        self.detectionCon = detectionCon
        self.trackCon = trackCon
        self.mpHands = mp.solutions.hands
        #self.hands = self.mpHands.Hands(self.mode, self.maxHands,self.detectionCon, self.trackCon)
        self.hands = self.mpHands.Hands(min_detection_confidence=0.8, min_tracking_confidence=0.5)
        self.mpDraw = mp.solutions.drawing_utils

    def findHands(self, img, draw=True, ):
        imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)    #转换为rgb
        self.results = self.hands.process(imgRGB)
        # print(results.multi_hand_landmarks)
        if self.results.multi_hand_landmarks:
            for handLms in self.results.multi_hand_landmarks:
                if draw:
                    self.mpDraw.draw_landmarks(img, handLms, self.mpHands.HAND_CONNECTIONS)

        return img

    def findPosition(self, img, handNo=0, draw=True):
        lmList = []
        if self.results.multi_hand_landmarks:
            myHand = self.results.multi_hand_landmarks[handNo]
            for id, lm in enumerate(myHand.landmark):
                # print(id, lm)
                # 获取手指关节点
                h, w, c = img.shape
                cx, cy = int(lm.x*w), int(lm.y*h)
                lmList.append([id, cx, cy])
                if draw:
                    cv2.putText(img, str(int(id)), (cx+10, cy+10), cv2.FONT_HERSHEY_PLAIN,
                                1, (0, 0, 255), 2)

        return lmList

    # 返回列表 包含每个手指的开合状态
    def fingerStatus(self, lmList):
        fingerList = []
        id, originx, originy = lmList[0]
        keypoint_list = [[2, 4], [6, 8], [10, 12], [14, 16], [18, 20]]
        for point in keypoint_list:
            id, x1, y1 = lmList[point[0]]
            id, x2, y2 = lmList[point[1]]
            if math.hypot(x2-originx, y2-originy) > math.hypot(x1-originx, y1-originy):
                fingerList.append(True)
            else:
                fingerList.append(False)
        return fingerList

def main():

    baunRate = 115200
    master,slave=mkpty( )
    ser=serial.Serial(slave,baunRate,timeout=5)

    #cap = cv2.VideoCapture(0, cv2.CAP_DSHOW)
    cap = cv2.VideoCapture(0)
    # 帧率统计
    pTime = 0
    cTime = 0
    detector = handDetctor()
    while True:
        success, img = cap.read()
        img = detector.findHands(img)
        lmList = detector.findPosition(img, draw=False)
        if len(lmList) != 0:
            # print(lmList)
            #print(detector.fingerStatus(lmList))
            finger=detector.fingerStatus(lmList)
            print(finger)
            data=str(finger[0])+','+str(finger[1])+','+str(finger[2])+','+str(finger[3])+','+str(finger[4])+',\n'
            print(data)
            os.write(master,data.encode())
            time.sleep(0.1)
        # 统计屏幕帧率
        cTime = time.time()
        fps = 1 / (cTime - pTime)
        pTime = cTime
        cv2.putText(img, str(int(fps)), (10, 70), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 255), 3)

        cv2.imshow("image", img)
        if cv2.waitKey(2) & 0xFF == 27:
            break

    cap.release()


if __name__ == '__main__':
    main()

2.2 用processing根据手的姿态控制3d手模型的姿态：
因为手的模型比较复杂，这里不是用processing的语句去画手，而是调用一个手的3d模型，由于3d建模还不熟悉，网上问朋友要了个手的3d模型。
这个资源消耗比较大，要内存比较大的电脑去运行，mac电脑比较老，换了个windows电脑运行。processing需要安装toxiclibs库。用手识别控制这个模型或者用上面的Wii Nunchuck手柄来控制这个模型也是可以的。

2.3 processing代码：
PShape A, B, C, D, E, F, G, H, I, J, K, L, N, M, O, P, Q, R, S, T, U, V, W, X, Y, Z, a, b, c, d, e, f;

import processing.serial.*;
import processing.opengl.*;
import toxi.geom.*;
import toxi.processing.*;

boolean leftIsPressed = false;
boolean shiftIsPressed = false;
boolean rightIsPressed = false;
boolean upIsPressed = false;
boolean downIsPressed = false;
boolean controlIsPressed = false;
int x = 0;
//int ax;
//int ay;
//int az;
String hand_a = "";
String hand_b = "";
String hand_c = "";
String hand_d = "";
String hand_e = "";
// NOTE: requires ToxicLibs to be installed in order to run properly.
// 1. Download from http://toxiclibs.org/downloads
// 2. Extract into [userdir]/Processing/libraries
//    (location may be different on Mac/Linux)
// 3. Run and bask in awesomeness

ToxiclibsSupport gfx;

Serial myPort;
char[] teapotPacket = new char[14];  // InvenSense Teapot packet
int serialCount = 0;                 // current packet byte position
int synced = 0;
int interval = 0;

float[] q = new float[4];
Quaternion quat = new Quaternion(1, 0, 0, 0);

float[] gravity = new float[3];
float[] euler = new float[3];
float[] ypr = new float[3];

void setup() {
    // 300px square viewport using OpenGL rendering
    size(800,800, P3D);
    gfx = new ToxiclibsSupport(this);
    A = loadShape("A.obj");
    B = loadShape("B.obj");
    C = loadShape("C.obj");
    D = loadShape("2.obj");
    E = loadShape("4.obj");
    F = loadShape("5.obj");
    G = loadShape("6.obj");
    H = loadShape("7.obj");
    I = loadShape("8.obj");
    J = loadShape("9.obj");
    K = loadShape("10.obj");
    L = loadShape("11.obj");
    N = loadShape("12.obj");
    M = loadShape("13.obj");
    O = loadShape("14.obj");
    P = loadShape("15.obj");
    Q = loadShape("18.obj");
    R = loadShape("19.obj");
    S = loadShape("20.obj");
    T = loadShape("21.obj");
    U = loadShape("22.obj");
    V = loadShape("23.obj");
    W = loadShape("24.obj");
    X = loadShape("25.obj");
    Y = loadShape("26.obj");
    Z = loadShape("27.obj");
    a = loadShape("28.obj");
    b = loadShape("29.obj");
    c = loadShape("30.obj");
    d = loadShape("31.obj");
    e = loadShape("32.obj");
    f = loadShape("33.obj");

    // setup lights and antialiasing


    // display serial port list for debugging/clarity
    //println(Serial.list());

    // get the first available port (use EITHER this OR the specific port code below)
    //String portName = Serial.list()[0];

    // get a specific serial port (use EITHER this OR the first-available code above)
    String portName = "/dev/pts/5";
    // open the serial port
        myPort = new Serial(this, portName, 115200);
        myPort.bufferUntil('\n');

}

void draw() {

    if (millis() - interval > 1000) {
        // resend single character to trigger DMP init/start
        // in case the MPU is halted/reset while applet is running
        interval = millis();
    }

    // black background
    background(0);

    translate(width / 2, height / 2);

    // 3-step rotation from yaw/pitch/roll angles (gimbal lock!)
    // ...and other weirdness I haven't figured out yet
    //rotateY(-ypr[0]);
    //rotateZ(-ypr[1]);
    //rotateX(-ypr[2]);
    //rotateY(ay/2);
    //rotateZ(az/2);
    //rotateX(ax/2);

    // toxiclibs direct angle/axis rotation from quaternion (NO gimbal lock!)
    // (axis order [1, 3, 2] and inversion [-1, +1, +1] is a consequence of
    // different coordinate system orientation assumptions between Processing
    // and InvenSense DMP)
    //float[] axis = quat.toAxisAngle();
    //rotate(axis[0], -axis[1], axis[3], axis[2]);
    //rotate(ax, 99, az, ay);
    rotate(50,29, 10, 20);



  x = 0;
  /*
  if(shiftIsPressed){
    x = x +1;
  }
  if(upIsPressed){
    x = x +4;
  }
  if(downIsPressed){
    x = x +18;
  }
  if(leftIsPressed){
    x = x +2;
  }
  if(rightIsPressed){
    x = x +8;
  }
  */

//--------------------------------
  if(hand_a.equals("False")){
    x = x +1;
  }
  if(hand_b.equals("False")){
    x = x +2;
  }
  if(hand_c.equals("False")){
    x = x +4;
  }
  if(hand_d.equals("False")){
    x = x +8;
  }
  if(hand_e.equals("False")){
    x = x +18;
  }
//--------------------------------

   if(x==0){
       shape(A);
   }
   if(x==1){
     shape(B);}
     if(x==2){
     shape(D);}
     if(x==3){
       shape(C);}
       if(x==4){
     shape(E);}
     if(x==5){
     shape(F);}
     if(x==6){
       shape(G);}
       if(x==7){
     shape(H);}
     if(x==8){
     shape(I);}
     if(x==9){
       shape(J);}
       if(x==10){
       shape(K);}
       if(x==11){
       shape(L);}
       if(x==12){
       shape(N);}
       if(x==13){
       shape(M);}
       if(x==14){
       shape(O);}
       if(x==15){
       shape(P);}
       if(x==18){
       shape(Q);}
       if(x==19){
       shape(R);}
       if(x==20){
       shape(S);}
       if(x==21){
       shape(T);}
       if(x==22){
       shape(U);}
       if(x==23){
       shape(V);}
       if(x==24){
       shape(W);}
       if(x==25){
       shape(X);}
       if(x==26){
       shape(Y);}
       if(x==27){
       shape(Z);}
       if(x==28){
       shape(a);}
       if(x==29){
       shape(b);}
       if(x==30){
       shape(c);}
       if(x==31){
       shape(d);}
       if(x==32){
       shape(e);}
       if(x==33){
       shape(f);}
}
void keyPressed(){
  if (key==CODED){
    if (keyCode == LEFT) leftIsPressed = true;
    if (keyCode == SHIFT) shiftIsPressed = true;
    if (keyCode == RIGHT) rightIsPressed = true;
    if (keyCode == UP) upIsPressed = true;
    if (keyCode == DOWN) downIsPressed = true;
    if (keyCode == CONTROL) controlIsPressed = true;
  }
}

void keyReleased(){
  if (key==CODED){
    if (keyCode == LEFT) leftIsPressed = false;
    if (keyCode == SHIFT) shiftIsPressed = false;
    if (keyCode == RIGHT) rightIsPressed = false;
    if (keyCode == UP) upIsPressed = false;
    if (keyCode == DOWN) downIsPressed = false;
    if (keyCode == CONTROL) controlIsPressed = false;
  }
}
void serialEvent(Serial p) {
  String inString =p.readString();
  String[] list =split(inString,',');
  hand_a=list[0];
  hand_b=list[1];
  hand_c=list[2];
  hand_d=list[3];
  hand_e=list[4];
  System.out.print(hand_e);
  //ax = (int(list[0])-100)*1;
  //ay = (int(list[1])-100)*1;
  //az = (int(list[2])-100)*1;
}

void drawCylinder(float topRadius, float bottomRadius, float tall, int sides) {
    float angle = 0;
    float angleIncrement = TWO_PI / sides;
    beginShape(QUAD_STRIP);
    for (int i = 0; i < sides + 1; ++i) {
        vertex(topRadius*cos(angle), 0, topRadius*sin(angle));
        vertex(bottomRadius*cos(angle), tall, bottomRadius*sin(angle));
        angle += angleIncrement;
    }
    endShape();

    // If it is not a cone, draw the circular top cap
    if (topRadius != 0) {
        angle = 0;
        beginShape(TRIANGLE_FAN);

        // Center point
        vertex(0, 0, 0);
        for (int i = 0; i < sides + 1; i++) {
            vertex(topRadius * cos(angle), 0, topRadius * sin(angle));
            angle += angleIncrement;
        }
        endShape();
    }

    // If it is not a cone, draw the circular bottom cap
    if (bottomRadius != 0) {
        angle = 0;
        beginShape(TRIANGLE_FAN);

        // Center point
        vertex(0, tall, 0);
        for (int i = 0; i < sides + 1; i++) {
            vertex(bottomRadius * cos(angle), tall, bottomRadius * sin(angle));
            angle += angleIncrement;
        }
        endShape();
    }
}


2.4 演示：



3.0 以上代码都参考了网上的一些案例资料https://www.basemu.com/modeling-the-values-from-the-arduino-mpu-6050-in-3d-using-processing.html，另最后面附上3d手的模型文件。


4.0 总结：通过这两个案例展示了processing的强大功能，以及它与其他硬件、软件结合在虚拟现实应用中的使用场景。后续是否可以建立两个差异图形，然后戴上3d眼镜看到立体的场景应用呢？运行平台也可以选择LattePanda 拿铁熊猫或者树莓派这种小型化平台来运行processing，方便整个设备的小型化。我想应该是可以的，大家可以进一步尝试。

很棒的项目，赞一个

6666666

谢谢兄弟们点赞