The looming AU material deadline has finally forced me to work out how to use Kinect gestures to navigate within an AutoCAD model. It’s far from perfect, but the fundamentals are all there: we have a loop – outside of a jig, this time, as we don’t need to display a point cloud or generate geometry in-place – that takes skeleton data provided by the Kinect and uses it to adjust the current view.
Like most people, my head gets a bit twisted when dealing with DCS and WCS, cameras, targets, views, etc., but thankfully I stumbled across an old piece of code from Jan Liska (a former member of DevTech and a current member of Autodesk Consulting) that greatly simplifies the task. It wraps the properties up in a Camera class that can be used to apply them to the current view.
I’m currently using it in a very crude way, though: rather than performing a true orbit or zoom, I’m applying a basic offset from the camera in screen coordinates (DCS), which then gets translated into a change in world coordinates. It’s inelegant but it at least results in the view changing in a semi-predictable way (and at this stage I’m mostly focused on proving the concept – cleaning up the actual mechanism can be done over the days leading up to AU).
Here’s the C# code:
using Autodesk.AutoCAD.ApplicationServices;
using Autodesk.AutoCAD.DatabaseServices;
using Autodesk.AutoCAD.EditorInput;
using Autodesk.AutoCAD.Geometry;
using Autodesk.AutoCAD.Runtime;
using Microsoft.Research.Kinect.Nui;
using System.Runtime.InteropServices;
using System;
namespace KinectNavigation
{
public class Camera : IDisposable
{
// Members
private Point3d _location;
private Point3d _target;
private Vector2d _locOffset;
private Vector2d _trgOffset;
private double _lensLength = 1.8;
private double _zoom = 1.0;
private double _forward = 0.0;
private Transaction _tr = null;
private Document _doc = null;
// Properties
public double LensLength
{
get { return _lensLength; }
set { _lensLength = value; }
}
public Point3d Location
{
get { return _location; }
set { _location = value; }
}
public Vector2d LocOffset
{
get { return _locOffset; }
set { _locOffset = value; }
}
public Vector2d TargOffset
{
get { return _trgOffset; }
set { _trgOffset = value; }
}
public Point3d Target
{
get { return _target; }
set { _target = value; }
}
public double Zoom
{
get { return _zoom; }
set { _zoom = value; }
}
public double Forward
{
get { return _forward; }
set { _forward = value; }
}
// Start a transaction on construction
public Camera(Document doc)
{
_doc = doc;
_tr = _doc.TransactionManager.StartTransaction();
}
// Commit and dispose of the transaction on Dispose
public void Dispose()
{
if (_tr != null)
{
_tr.Commit();
_tr.Dispose();
_tr = null;
}
}
// Generate the transformation matrix to go from
// WCS to DCS units in a view/viewport
public Matrix3d Wcs2Dcs(AbstractViewTableRecord vtr)
{
Matrix3d mat =
Matrix3d.PlaneToWorld(vtr.ViewDirection);
mat =
Matrix3d.Displacement(vtr.Target - Point3d.Origin) * mat;
mat =
Matrix3d.Rotation(
-vtr.ViewTwist,
vtr.ViewDirection,
vtr.Target) * mat;
return mat.Inverse();
}
// Apply our existing settings to the current view
public void ApplyToCurrentView()
{
Editor ed = _doc.Editor;
ViewportTableRecord vptr =
(ViewportTableRecord)_tr.GetObject(
ed.ActiveViewportId, OpenMode.ForRead
);
Point3d trg = _target, loc = _location;
// Adjust the target
if (Commands.NonZero(_trgOffset))
trg =
AddDcsOffsetToWcsValue(ref _target, _trgOffset, vptr);
// Adjust the camera location
if (Commands.NonZero(_locOffset))
loc =
AddDcsOffsetToWcsValue(ref _location, _locOffset, vptr);
// Set up a view for the current settings
ViewTableRecord vtr = new ViewTableRecord();
vtr.CenterPoint = Point2d.Origin;
vtr.ViewTwist = 0.0;
vtr.PerspectiveEnabled = true;
vtr.IsPaperspaceView = false;
vtr.Height = vptr.Height;
vtr.Width = vptr.Width;
vtr.ViewDirection =
trg.GetVectorTo(loc).MultiplyBy(Zoom);
vtr.Target = trg;
vtr.LensLength = LensLength;
// Set it as the current view
ed.SetCurrentView(vtr);
}
private Point3d AddDcsOffsetToWcsValue(
ref Point3d value, Vector2d offset,
AbstractViewTableRecord vtr
)
{
// Get our transformation matrices
Matrix3d wcs2dcs = Wcs2Dcs(vtr);
Matrix3d dcs2wcs = wcs2dcs.Inverse();
// Transform the WCS value passed in to DCS
// and add the offset vector
Point3d dcsVal =
value.TransformBy(wcs2dcs) +
new Vector3d (offset.X, offset.Y, 0);
// Before converting back to WCS and setting
// and returning it
value = dcsVal.TransformBy(dcs2wcs);
return value;
}
}
public class Commands
{
// Flags for navigation modes
bool _finished = false;
bool _reset = false;
bool _navigating = false;
bool _orbiting = false;
// The direction we're navigating in
Vector3d _direction;
double _zoomDist = 0.0;
void OnSkeletonFrameReady(
object sender, SkeletonFrameReadyEventArgs e
)
{
SkeletonFrame s = e.SkeletonFrame;
foreach (SkeletonData data in s.Skeletons)
{
if (SkeletonTrackingState.Tracked == data.TrackingState)
{
// Get the positions of joints we care about
Point3d leftShoulder =
PointFromVector(
data.Joints[JointID.ShoulderLeft].Position
);
Point3d rightShoulder =
PointFromVector(
data.Joints[JointID.ShoulderRight].Position
);
Point3d leftHand =
PointFromVector(
data.Joints[JointID.HandLeft].Position
);
Point3d rightHand =
PointFromVector(
data.Joints[JointID.HandRight].Position
);
Point3d centerShoulder =
PointFromVector(
data.Joints[JointID.ShoulderCenter].Position
);
// Make sure our hands are non-zero
if (NonZero(leftHand) && NonZero(rightHand))
{
// We're finished if our hands are close together
// and near the centre of our shoulders
_finished =
CloseTo(leftHand, rightHand, 0.1) &&
CloseTo(leftHand, centerShoulder, 0.4);
// Reset the view if our hands are at about the
// same level but further apart
_reset =
leftHand.DistanceTo(rightHand) > 0.5 &&
Math.Abs(leftHand.Y - rightHand.Y) < 0.1 &&
Math.Abs(leftHand.Z - rightHand.Z) < 0.1;
// If neither of these modes is set...
if (!_finished && !_reset)
{
// .. we may still be navigating or orbiting
_navigating = false;
_orbiting = false;
if (CloseTo(leftHand, rightHand, 0.05))
{
// Hands are close together, but not near
// the chest which means we're navigating
_navigating = true;
_direction =
GetDirection(
leftHand + ((rightHand - leftHand) / 2),
centerShoulder
);
// Normalize to unit length
_direction = _direction / _direction.Length;
}
else if (
(
CloseTo(leftHand, leftShoulder, 0.3) &&
rightHand.DistanceTo(rightShoulder) > 0.4
) ||
(
CloseTo(rightHand, rightShoulder, 0.3) &&
leftHand.DistanceTo(leftShoulder) > 0.4
)
)
{
// One hand is near its shoulder, and the other is
// pointed outwards, which means we're orbiting
_orbiting = true;
_direction =
(CloseTo(leftHand, leftShoulder) ?
GetDirection(rightHand, rightShoulder) :
GetDirection(leftHand, leftShoulder)
);
// Normalize to unit length
_direction = _direction / _direction.Length;
}
}
}
break;
}
}
}
// Is the Point3d non-zero?
internal static bool NonZero(Point3d pt)
{
return
!CloseTo(
pt, Point3d.Origin, Tolerance.Global.EqualPoint
);
}
// Is the Vector2d non-zero?
internal static bool NonZero(Vector2d vec)
{
return
!CloseTo(
new Point3d(vec.X, vec.Y, 0.0),
Point3d.Origin, Tolerance.Global.EqualPoint
);
}
// Are two points within a certain distance?
private static bool CloseTo(
Point3d first, Point3d second, double dist = 0.1
)
{
return first.DistanceTo(second) < dist;
}
// Get a Point3d from a Kinect Vector
private static Point3d PointFromVector(Vector v)
{
return new Point3d(v.X, v.Y, v.Z);
}
// Get the vector from the shoulder to the hand
private static Vector3d GetDirection(
Point3d hand, Point3d shoulder
)
{
return shoulder - hand;
}
[CommandMethod("ADNPLUGINS", "KINNAV", CommandFlags.Modal)]
public void NavigateWithKinect()
{
Document doc =
Autodesk.AutoCAD.ApplicationServices.
Application.DocumentManager.MdiActiveDocument;
Database db = doc.Database;
Editor ed = doc.Editor;
// As the user to select the camera and target locations
PromptPointResult camRes =
ed.GetPoint("\nPick camera location");
if (camRes.Status != PromptStatus.OK)
return;
PromptPointResult tgtRes =
ed.GetPoint("\nPick target location");
if (tgtRes.Status != PromptStatus.OK)
return;
// And also the height from ground level
PromptDoubleOptions pdo =
new PromptDoubleOptions("\nEnter height from ground");
pdo.UseDefaultValue = true;
// Default height of 6' or 2m
pdo.DefaultValue = (db.Lunits > 2 ? 6 : 2);
PromptDoubleResult hgtRes = ed.GetDouble(pdo);
if (hgtRes.Status != PromptStatus.OK)
return;
_zoomDist = hgtRes.Value / 10.0;
// We need a Kinect object
Runtime kinect = null;
// Make sure we dispose of our Camera (which will
// commit the transaction)
using (Camera camera = new Camera(doc))
{
// Set the initial values of our view
Vector3d height = new Vector3d(0, 0, hgtRes.Value);
camera.Location = camRes.Value + height;
camera.Target = tgtRes.Value + height;
camera.ApplyToCurrentView();
// Unset the loop termination flag
_finished = false;
try
{
// We need our Kinect sensor
kinect = Runtime.Kinects[0];
// We only care about skeleton information
kinect.SkeletonFrameReady +=
new EventHandler<SkeletonFrameReadyEventArgs>(
OnSkeletonFrameReady
);
kinect.Initialize(
RuntimeOptions.UseSkeletalTracking
);
}
catch (System.Exception ex)
{
ed.WriteMessage(
"\nUnable to start Kinect sensor: " + ex.Message
);
return;
}
// Loop until user terminates or cancels
while (
!_finished &&
!HostApplicationServices.Current.UserBreak()
)
{
// Direction from Kinect is:
//
// 0, 0, 1 - arm pointing directly at the center
// 1, 0, 0 - arm pointing directly left
// -1, 0, 0 - arm pointing directly right
// 0, 1, 0 - arm pointing directly down
// 0, -1, 0 - arm pointing directly up
// We'll get the vertical view size in WCS units, and
// use a fraction of this for our navigation increases
double viewSize =
(double)Application.GetSystemVariable("VIEWSIZE");
double fac = viewSize / 10;
if (_reset)
{
// Reset to the initial view parameters
camera.Location = camRes.Value + height;
camera.Target = tgtRes.Value + height;
camera.ApplyToCurrentView();
}
else if (_orbiting || _navigating)
{
// We will offset based on the provided direction
Vector2d offset =
new Vector2d(
-(_direction.X) * fac,
-(_direction.Y) * fac
);
// Offset the camera location
camera.LocOffset = offset;
if (_navigating)
{
// If navigating/zooming, offset the target, too
camera.TargOffset = offset;
// And move the camera forward
camera.Forward = _direction.Z * _zoomDist;
}
camera.ApplyToCurrentView();
}
// Reset the offset and movement values
camera.LocOffset = new Vector2d();
camera.TargOffset = new Vector2d();
camera.Forward = 0;
System.Windows.Forms.Application.DoEvents();
}
}
kinect.Uninitialize();
kinect.SkeletonFrameReady -=
new EventHandler<SkeletonFrameReadyEventArgs>(
OnSkeletonFrameReady
);
}
}
}
When you run the KINNAV command, you’ll be prompted to select a location for both the camera and the target, along with the height each should be from the ground. The idea is that this makes it easier to “walk through” a model. The reality – as mentioned earlier – is that it doesn’t yet work all that well, but hopefully I’ll iron out the wrinkles, in time.
A few different gestures are supported:
- To orbit (kinda), you hold one arm in, next to your chest, and the other in the direction you want to move in
- The zoom (again, kinda) you hold both arms out in front, once again in the direction you want to move
- As it’s easy to get lost, if you hold your arms out with your hands apart, the view gets reset to the initial one
- If you put your hands together, close to your chest, the command should terminate
In case you hadn’t realised, the first two gestures were strongly inspired by Superman (I’m really aiming for the sensation of flying through a model – we’ll see if I get there).
Although this is the latest of a long line of tests with the Kinect, I still believe that navigation is really the “killer app” for this technology when it comes to the design space. I don’t see anyone wanting to spend 40 hours a week modelling using full body gestures (ouch), but I could very easily imagine a quick, collaborative 3D review and markup session being made with a very low barrier of entry for participants from a UI perspective. After all, if you can pretend to be Superman you can navigate a 3D model using this kind of implementation. ;-)