DentaLab.

Team Lead & Programmer | VR Multiplayer Dental Simulation | Unity & OVR
5 developers | 5 months | December 2021
Published in2022 IEEE Conference on Virtual Reality and 3D User Interfaces Abstracts and Workshops (VRW)


Description.

DentaLab is an innovative educational platform tailored for dentistry students, offering immersive and remote learning experiences. Developed in collaboration between the University of Florida's Digital Worlds Institute and the University of Illinois Chicago's College of Dentistry, I played a pivotal role in introducing multiplayer functionality and a dynamic lecture room environment enriched with interactive features. Explore further details in the publication here.


Contributions.

As Team Lead & Programmer, I coordinated development efforts within my communication team, delivering the requested features for professors at the University of Illinois Chicago and the University of Florida. I also played a significant role in co-authoring the published journal.


  • PDF and PowerPoint viewers [Jump].
  • Color-changing marker for whiteboard [Jump].
  • Customizable exam assessment [Jump].
  • Multiplayer functionality for entire application utilizing PUN 2.
A virtual, dental exam room with a patient lying on the exam table.

PDF & PowerPoint Viewers.

Description.

Each virtual reality environment in DentaLab replicates its real-world counterpart with augmented reality. The PowerPoint display enables customizable presentations, and the PDF viewer allows users to read notes or research papers. Files can be added, edited, and removed without requiring Unity knowledge.

Challenges.

One major goal was to provide flexibility in managing stored files, necessitating automatic file conversion to .png images for user-friendliness. Although the conversion works flawlessly in development, Window's System.Drawing caused build conflicts in Unity. Fortunately, the data structure representing the PDF and PowerPoint files supports regular .png use, enabling users to change files easily (See PDF.cs under Code). Controls switch between file selection and scrolling. Once a file is chosen, file selection controls disappear, allowing navigation within the selected file. Transitioning from a raycast selector to a touchscreen press caused unintended control activations due to overlapping areas. Implementing a simple Coroutine introduced a short grace period to prevent these accidental presses (See PdfInteractable.cs under Code).

Pdf.cs.

using System.IO;
using UnityEngine;
using UnityEngine.UI;
using Image = UnityEngine.UI.Image;
using System.Collections.Generic;
public classPDF :MonoBehaviour
{
[Header("Browsing List")]
[Tooltip("The parent GameObject holding all list objects")]
public GameObjectlist;
[Tooltip("The parent GameObjects of the 3 list displays")]
public GameObjecttopManager,middleManager,bottomManager;
Image topImg, middleImg, bottomImg;// Preview images in each of the list displays
Text topTxt, middleTxt, bottomTxt;// Titles in each of the list displays
int listIndex= 0;// Tracks browsing index
[Header("Scrolling Display")]
[Tooltip("The parent GameObject holding all display objects")]
public GameObjectdisplay;
[Tooltip("The Image that holds the current page being displayed")]
public Imageimage;
[Tooltip("The Text for the current document title")]
public Texttitle;
[Tooltip("The position of the forward direction of the owning player")]
public TransformplayerForwardDir;
[Tooltip("The position of the owning player")]
public Transformplayer;
int currentPage;// The index of the current document
string selectedPdf;// The selected document title
Dictionary<string,int>maxPages;// The dictionary containing max page numbers for all documents
List<string>pdfs;// The list of document names
void Awake()
{
// Gets list of PDF files in Assets/PDF
string[] pdfsFile= Directory.GetFiles(Directory.GetCurrentDirectory() +"/Assets/Assets/PDF/","*.pdf");
pdfs =new List<string>();
maxPages =new Dictionary<string, int>();
foreach (string pdf in pdfsFile)
{
string inputPdfPath= Path.Combine(Directory.GetCurrentDirectory() +"/Assets/Assets/PDF/", pdf);
string pdfName= pdf.Substring(0, pdf.Length - 4);
string outputPath= Path.Combine(Directory.GetCurrentDirectory() +"/Assets/Assets/PDF/", pdfName);
pdfs.Add(pdf.Substring(0, pdf.Length - 4));
maxPages.Add(pdfName, Directory.GetFiles(outputPath,"*").Length);
}
// Gets preview images
topImg= topManager.transform.GetChild(1).GetComponent<Image>();
middleImg= middleManager.transform.GetChild(1).GetComponent<Image>();
bottomImg= bottomManager.transform.GetChild(1).GetComponent<Image>();
// Gets preview text
topTxt= topManager.transform.GetChild(0).GetComponent<Text>();
middleTxt= middleManager.transform.GetChild(0).GetComponent<Text>();
bottomTxt= bottomManager.transform.GetChild(0).GetComponent<Text>();
LoadList();
}
private voidOnEnable()
{
// Puts PDF 1m in front of player
transform.position = playerForwardDir.position + playerForwardDir.forward;
QuaternionnewRotation= Quaternion.LookRotation(player.position - transform.position);
Vector3eulerRotation= newRotation.eulerAngles;
eulerRotation.x =0f;
eulerRotation.y +=180f;
transform.eulerAngles = eulerRotation;
}
Sprite GetPage(string pdf, intpage)
{
// Sets title
string[] split= pdf.Split('/');
title.text = split[split.Length - 1];
// Create sprite texture
Texture2D pdfTexture= new Texture2D(1,1);
// Loads image from path
string path= pdf+"/"+ page.ToString() +".png";
pdfTexture.LoadImage(System.IO.File.ReadAllBytes(path));
// Returns image as sprite
return Sprite.Create(pdfTexture,new Rect(0,0, pdfTexture.width, pdfTexture.height),new Vector2(0.5f,0.5f));
}
public voidOnLeftClick()
{
if (currentPage >0)
currentPage--;
image.GetComponent<Image>().sprite = GetPage(selectedPdf, currentPage);
}
public voidOnRightClick()
{
if (currentPage< maxPages[selectedPdf]-1)
currentPage++;
image.GetComponent<Image>().sprite = GetPage(selectedPdf, currentPage);
}
void LoadPage()
{
// Hides List
list.SetActive(false);
// Unhides Display
display.SetActive(true);
// Replaces sprite with first PDF
currentPage =0;
image.sprite = GetPage(selectedPdf, 0);
}
void LoadList()
{
/*
* Loads the browsing list
*/
string top= pdfs[listIndex];
topImg.sprite = GetPage(top,0);
string[] topSplit= top.Split('/');
topTxt.text = topSplit[topSplit.Length -1];
string middle="";
string bottom="";
if (pdfs.Count > 1+ listIndex) {
middleManager.SetActive(true);
middle = pdfs[listIndex +1];
middleImg.sprite = GetPage(middle,0);
string[] middleSplit= middle.Split('/');
middleTxt.text = middleSplit[middleSplit.Length - 1];
} else
{
middleManager.SetActive(false);
middleImg.sprite =null;
middleTxt.text ="";
}
if (pdfs.Count > 2+ listIndex)
{
bottomManager.SetActive(true);
bottom = pdfs[listIndex +2];
bottomImg.sprite = GetPage(bottom,0);
string[] bottomSplit= bottom.Split('/');
bottomTxt.text = bottomSplit[bottomSplit.Length - 1];
} else
{
bottomManager.SetActive(false);
bottomImg.sprite =null;
bottomTxt.text ="";
}
}
public voidTopPage()
{
selectedPdf= pdfs[listIndex];
LoadPage();
listIndex +=3;
}
public voidMiddlePage()
{
selectedPdf= pdfs[listIndex +1];
LoadPage();
listIndex +=3;
}
public voidBottomPage()
{
selectedPdf= pdfs[listIndex +2];
LoadPage();
listIndex +=3;
}
public voidScrollUp()
{
if (listIndex >= 3)
{
listIndex -=3;
LoadList();
}
}
public voidScrollDown()
{
if (listIndex <= pdfs.Count)
{
listIndex +=3;
LoadList();
}
}
public voidBack()
{
// Unhides List
list.SetActive(true);
// Hides Display
display.SetActive(false);
listIndex =0;
LoadList();
}
}
view raw PDF.cs hosted with ❤ by GitHub

PdfInteractable.cs.

using Photon.Pun;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public enumPdfButtonName {UpButton,DownButton,Top,Middle,Bottom,LeftButton,RightButton,BackButton }
public classPdfInteractable :MonoBehaviourPun,IPunObservable
{
[Header("Button")]
[Tooltip("The button this script is representing")]
public PdfButtonNamebuttonType;
[Tooltip("The parent GameObject that holds the Up and Down buttons")]
public GameObjectlistNavigation;
[Tooltip("The parent GameObject that holds the Left, Right, and Back buttons")]
public GameObjectpdfNavigation;
[Header("PDF")]
[Tooltip("The interactable PDF itself")]
public GameObjectpdf;
[Header("Haptics")]
[Tooltip("The audio to be played when a button is pressed")]
public AudioCliphapticAudio;
PDF pdfScript;// The PDF.cs script attached to pdf
bool isPressed;// Whether the button has been pressed or not
void Start()
{
pdfScript = pdf.GetComponentInChildren<PDF>();
}
private voidOnTriggerEnter(Collider other)
{
if (other.gameObject.tag =="IndexR"&& !isPressed)
{
isPressed =true;
OVRHapticsCliphapticsClip =new OVRHapticsClip(hapticAudio);
OVRHaptics.RightChannel.Preempt(hapticsClip);
ButtonOnClick();
}
else if(other.gameObject.tag =="IndexR"&& isPressed)
{
isPressed =false;
OVRHapticsCliphapticsClip =new OVRHapticsClip(hapticAudio);
OVRHaptics.RightChannel.Preempt(hapticsClip);
ButtonOnClick();
}
}
void ButtonOnClick()
{
switch (buttonType)
{
case PdfButtonName.UpButton:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfScrollUp", RpcTarget.All);
}
break;
case PdfButtonName.DownButton:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfScrollDown", RpcTarget.All);
}
break;
case PdfButtonName.Top:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfTop", RpcTarget.All);
}
break;
case PdfButtonName.Middle:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfMiddle", RpcTarget.All);
}
break;
case PdfButtonName.Bottom:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfBottom", RpcTarget.All);
}
break;
case PdfButtonName.LeftButton:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfLeft", RpcTarget.All);
}
break;
case PdfButtonName.RightButton:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfRight", RpcTarget.All);
}
break;
case PdfButtonName.BackButton:
if (photonView.IsMine)
{
photonView.RPC("RPC_PdfBack", RpcTarget.All);
}
break;
}
}
[PunRPC]
void RPC_PdfScrollUp(PhotonMessageInfoinfo)
{
pdfScript.ScrollUp();
}
[PunRPC]
void RPC_PdfScrollDown(PhotonMessageInfoinfo)
{
pdfScript.ScrollDown();
}
[PunRPC]
void RPC_PdfTop(PhotonMessageInfoinfo)
{
pdfScript.TopPage();
StartCoroutine(ToggleNavigation());
}
[PunRPC]
void RPC_PdfMiddle(PhotonMessageInfoinfo)
{
pdfScript.MiddlePage();
StartCoroutine(ToggleNavigation());
}
[PunRPC]
void RPC_PdfBottom(PhotonMessageInfoinfo)
{
pdfScript.BottomPage();
StartCoroutine(ToggleNavigation());
}
[PunRPC]
void RPC_PdfLeft(PhotonMessageInfoinfo)
{
pdfScript.OnLeftClick();
}
[PunRPC]
void RPC_PdfRight(PhotonMessageInfoinfo)
{
pdfScript.OnRightClick();
}
[PunRPC]
void RPC_PdfBack(PhotonMessageInfoinfo)
{
pdfScript.Back();
StartCoroutine(ToggleNavigation());
}
IEnumerator ToggleNavigation()
{
yield returnnew WaitForSeconds(0.5f);
isPressed =false;
listNavigation.SetActive(!listNavigation.activeSelf);
pdfNavigation.SetActive(!pdfNavigation.activeSelf);
}
...
}

Whiteboard.

Description.

Each user has a wrist user-interface, providing them with a personal whiteboard. While only the whiteboard's owner can move it around the room, other users can collaborate by writing or drawing on it.

Challenges.

Multiplayer implementation was the primary challenge of DentaLab. Transmitting whiteboard data required careful thought. Using Photon Unity Networking (PUN), I implemented remote procedure calls to notify other users of any whiteboard changes. However, sending/receiving a whiteboard's image upon every change caused severe frame rate drops and latency. To provide a lag-free, real-time experience, PUN only tracks the positional and rotational data of the whiteboard and its marker. This way, each player runs the drawing functions locally, eliminating the need for image data transmission (See Whiteboard.cs). Meanwhile, remote procedure calls notify all users if a marker's color changes (See PenColorSelector.cs).

A user pointing to 3 different colored buttons next to a virtual whiteboard

Whiteboard.cs.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System.Linq;
using Photon.Pun;
public classWhiteboard :MonoBehaviourPun,IPunObservable
{
...
public voidOnPhotonSerializeView(PhotonStream stream,PhotonMessageInfo info)
{
if (stream.IsWriting)
{
stream.SendNext(transform.position);
stream.SendNext(transform.rotation);
} elseif (stream.IsReading)
{
transform.position =(Vector3)stream.ReceiveNext();
transform.rotation =(Quaternion)stream.ReceiveNext();
}
}
}

PenColorSelector.cs.

using System.Collections;
using System;
using System.Collections.Generic;
using UnityEngine;
using Photon.Pun;
public enumPenColor {Blue,Red,Black }
public classPenColorSelector :MonoBehaviourPun,IPunObservable
{
[Header("Button")]
[Tooltip("The button this script is representing")]
public PenColorbuttonType;
[Header("Haptic")]
[Tooltip("The audio to be played with haptic feedback")]
public AudioCliphapticAudio;
[Header("Marker")]
public GameObjectmarker;
...
void ButtonOnClick()
{
switch (buttonType)
{
case PenColor.Blue:
photonView.RPC("RPC_ChangePenColor", RpcTarget.All, 0, 0, 255);
break;
case PenColor.Red:
photonView.RPC("RPC_ChangePenColor", RpcTarget.All, 255, 0, 0);
break;
case PenColor.Black:
photonView.RPC("RPC_ChangePenColor", RpcTarget.All, 0, 0, 0);
break;
}
}
[PunRPC]
voidRPC_ChangePenColor(int r, intg,int b,PhotonMessageInfo info)
{
WhiteboardPen pen= marker.GetComponent<WhiteboardPen>();
pen.penTip.GetComponent<Renderer>().material.color = new Color(r, g, b);
pen.ColorPicker =new Color(r, g, b);
}
...
}

Exam Assessments.

Description.

Assessments grant students the opportunity to recall what they have learned in DentaLab. Given the automatic grading and review feature, students can see their educational progress without leaving virtual reality. By simply editing a text document, professors can easily adjust the exam to reflect the current curriculum as well.

Challenges.

Reading and storing the exam itself was the main task. One solution would have been implementing a class "Question," containing the question itself, its possible answers, and the correct answer. However, I was using this exam feature to teach my new-to-coding peer and decided not to delve into object-oriented programming just yet. Instead, I opted for a solution with the same effect: a List of Tuples containing the same data (See ExamManager.cs).
Going from the lecture room to the test-taking room should feel as natural as possible, increasing player immersion and comfort. In the real world, we don't use an interface to open a door. Luckily, the environment has a door in the back. So, the solution is for the players to enter the Exam Room by grabbing the doorknob (See ToExamRoom.cs).

A virtual environment with a classroom table and a gray door labeled 'Exam Room'

ExamManager.cs.

using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using UnityEngine;
using UnityEngine.UI;
public classExamManager :MonoBehaviour
{
[Tooltip("The Text for the question display")]
public Textquestion;
[Tooltip("The Text for the automatic evaluation display")]
public Textevaluation;
[Tooltip("The parents of the GameObjects containing each choice")]
public GameObjecta,b,c,d;
[Tooltip("The Text for each choice")]
public TextaText,bText,cText,dText;
[Tooltip("The navigation Buttons")]
public Buttonback,next;
[Tooltip("The Sprites with backgrounds corresponding to selection and choice correctness")]
public SpritedefaultImg,selectedImg,wrongImg,rightImg;
[Tooltip("The parent GameObjects holding the Next Button and Submit Button respectively")]
public GameObjectnextObj,submit;
bool submitted;// Tracks if the user has submnitted the exam
string examFileName="Exam1";// The name of the .txt file
int index;// Current question index
string title; // The exams title
// Question data: Question, A-D multiple choice questions, and correct answer
List<Tuple<string,string,string,string,string,string>> questions;
List<string>answers;// The students answers
Button aBtn, bBtn, cBtn, dBtn;// Buttons A, B, C, D
private voidStart()
{
// Gets buttons
aBtn = a.GetComponentInChildren<Button>();
bBtn = b.GetComponentInChildren<Button>();
cBtn = c.GetComponentInChildren<Button>();
dBtn = d.GetComponentInChildren<Button>();
// Adds button listeners
back.onClick.AddListener(OnBack);
next.onClick.AddListener(OnNext);
aBtn.onClick.AddListener(OnSelectA);
bBtn.onClick.AddListener(OnSelectB);
cBtn.onClick.AddListener(OnSelectC);
dBtn.onClick.AddListener(OnSelectD);
submit.GetComponent<Button>().onClick.AddListener(OnSubmit);
LoadExam(examFileName);
}
void LoadExam(stringexamFileName)
{
/*
* Loads given exam from Assets/Assets/Exams directory
*/
questions =new List<Tuple<string, string, string, string, string, string>>();
index =0;
string contents= Directory.GetCurrentDirectory() +"/Assets/Assets/Exams/"+ examFileName+".txt";
List<string>lines = File.ReadAllLines(contents).ToList();
title = lines[0]; // Stores title
// Stores question, its choices, and answer
for (int i= 2; i< lines.Count;i +=8)
{
Tuple<string,string,string,string,string,string>question =new Tuple<string, string, string, string, string, string>(
lines[i]+" "+ lines[i+1],
lines[i+2],
lines[i+3],
lines[i+4],
lines[i+5],
lines[i+6]
);
questions.Add(question);
}
// Fills current answers as blank
answers =new List<string>(questions.Count);
for (int i= 0; i< questions.Count; i++)
answers.Add("");
LoadQuestion();
}
void LoadQuestion()
{
/*
* Loads question onto Canvas
*/
// Displays question of current index
Tuple<string,string,string,string,string,string>q = questions[index];
question.text = q.Item1;
aText.text = q.Item2.Substring(3, q.Item2.Count()-3);
bText.text = q.Item3.Substring(3, q.Item3.Count()-3);
cText.text = q.Item4.Substring(3, q.Item4.Count()-3);
dText.text = q.Item5.Substring(3, q.Item5.Count()-3);
ResetButtonColors();
// Sets button to selected color if applicable
if (answers[index].Equals("A"))
SetAnswerButtonColors(a);
else if(answers[index].Equals("B"))
SetAnswerButtonColors(b);
else if(answers[index].Equals("C"))
SetAnswerButtonColors(c);
else if(answers[index].Equals("D"))
SetAnswerButtonColors(d);
// Displays next or submit button
if (index== questions.Count - 1)
{
nextObj.SetActive(false);
submit.SetActive(true);
} else
{
nextObj.SetActive(true);
submit.SetActive(false);
}
}
void ResetButtonColors()
{
// Resets button colors back to default
a.GetComponentInChildren<Image>().sprite = defaultImg;
b.GetComponentInChildren<Image>().sprite = defaultImg;
c.GetComponentInChildren<Image>().sprite = defaultImg;
d.GetComponentInChildren<Image>().sprite = defaultImg;
}
voidSetAnswerButtonColors(GameObject multi)
{
// Sets button color to selected
multi.GetComponentInChildren<Image>().sprite = selectedImg;
}
void OnBack()
{
// Goes to previous question
if (index >0)
index--;
if (submitted)
LoadAnswer();
else
LoadQuestion();
}
void OnNext()
{
// Goes to next question
if (index< questions.Count - 1)
index++;
if (submitted)
LoadAnswer();
else
LoadQuestion();
}
void OnSelectA()
{
// Stores A as answer to question
if (!submitted)
{
answers[index]="A";
ResetButtonColors();
SetAnswerButtonColors(a);
}
}
void OnSelectB()
{
// Stores B as answer to question
if (!submitted)
{
answers[index]="B";
ResetButtonColors();
SetAnswerButtonColors(b);
}
}
void OnSelectC()
{
// Stores C as answer to question
if (!submitted)
{
answers[index]="C";
ResetButtonColors();
SetAnswerButtonColors(c);
}
}
void OnSelectD()
{
// Stores D as answer to question
if (!submitted)
{
answers[index]="D";
ResetButtonColors();
SetAnswerButtonColors(d);
}
}
void SetRightAnswer(GameObjectmulti)
{
// Sets color of button to correct (e.g. green)
multi.GetComponentInChildren<Image>().sprite = rightImg;
}
void SetWrongAnswer(GameObjectmulti)
{
// Sets color of button to incorrect (e.g. red)
multi.GetComponentInChildren<Image>().sprite = wrongImg;
}
void LoadAnswer()
{
/*
* Loads answer onto Canvas
*/
// Displays question and gets correct answer
Tuple<string,string,string,string,string,string>q = questions[index];
question.text = q.Item1;
aText.text = q.Item2;
bText.text = q.Item3;
cText.text = q.Item4;
dText.text = q.Item5;
string correctChoice= q.Item6;
ResetButtonColors();
if (answers[index] ==correctChoice)// Correct Answer Chosen
{
if (answers[index].Equals("A"))
SetRightAnswer(a);
else if(answers[index].Equals("B"))
SetRightAnswer(b);
else if(answers[index].Equals("C"))
SetRightAnswer(c);
else if(answers[index].Equals("D"))
SetRightAnswer(d);
} else// Incorrect Answer Chosen
{
// Display their answer
if (answers[index].Equals("A"))
SetWrongAnswer(a);
else if(answers[index].Equals("B"))
SetWrongAnswer(b);
else if(answers[index].Equals("C"))
SetWrongAnswer(c);
else if(answers[index].Equals("D"))
SetWrongAnswer(d);
// Display correct answer
if (correctChoice.Equals("A"))
SetRightAnswer(a);
else if(correctChoice.Equals("B"))
SetRightAnswer(b);
else if(correctChoice.Equals("C"))
SetRightAnswer(c);
else if(correctChoice.Equals("D"))
SetRightAnswer(d);
}
}
void OnSubmit()
{
submitted =true;
// Next button is always available (not submit)
nextObj.SetActive(true);
submit.SetActive(false);
// Counts correctly answered questions
int correct= 0;
for (int i= 0; i< questions.Count; i++)
{
if (answers[i].Equals(questions[i].Item6))
correct++;
}
// Gives final evaluation and shows answers
evaluation.text = correct.ToString() +"/"+ questions.Count.ToString();
index =0;
LoadAnswer();
}
}

ToExamRoom.cs.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.SceneManagement;
public classToExamRoom :MonoBehaviour
{
[Header("Scene")]
[Tooltip("The scene to navigate to")]
public stringscene;
[Header("Haptics")]
[Tooltip("The haptics to be played the doorknob is grabbed")]
public AudioCliphapticAudio;
private voidOnTriggerEnter(Collider other)
{
if (other.gameObject.tag =="IndexRig")
{
OVRHapticsCliphapticsClip =new OVRHapticsClip(hapticAudio);
OVRHaptics.RightChannel.Preempt(hapticsClip);
OnKnobGrab();
}
}
void OnKnobGrab()
{
SceneManager.LoadScene(scene);
}
}