Computer vision is a very trendy topic nowadays and it's likely to become even more present in the near future. I wanted to have a first experience with it: it quickly appeared that the reference library on the topic is OpenCV, an SDK initially created by IBM and later backed up by the community.
I could run a couple of OpenCV samples but without a project in mind it would become boring very quickly, so I decided to work on an integration with Forge. After reading that article OpenCV tutorial: Computer vision with Node.js, it made sense to use the concepts exposed there with a Forge model: my first demo will use the Node.js bindings node-opencv to determine an oriented optimized bounding of a loaded Forge Viewer model.
Here is the workflow:
- Create a web page that loads a Forge model
- Implement some server logic that loads the same model on the server. This can be easily achieved using Puppeteer (the headless Chrome API for Node). I was already working on the topic in a previous blog article: Running Forge Viewer headless in Chrome with Puppeteer
- Keep the session active to allow interaction between client and server models
- Using Puppeteer API, generate a screenshot of the server model and pass it to OpenCV API to compute the oriented bounding box
- Communicate the result back to client using a set of custom endpoints
Below are the two major highlights of the implementation of the project:
In order to keep the server available to requests from multiple client, the loading of the model on the server happens in a worker process. Here is the code of that worker. This is where you can find the implementation of the OpenCV logic:
import puppeteer from 'puppeteer'
import pathUtils from 'path'
import cv from 'opencv'
import fs from 'fs'
/////////////////////////////////////////////////////////////////
// Loads an image with node OpenCV SDK
//
/////////////////////////////////////////////////////////////////
const loadImage = (filename) => {
return new Promise((resolve, reject) => {
cv.readImage(filename, (err, img) => {
try {
if (err) {
return reject(err)
}
if (img.height() < 1 || img.width() < 1) {
return reject('Image has no size')
}
resolve(img)
} catch(ex) {
return reject(ex)
}
})
})
}
/////////////////////////////////////////////////////////////////
// Gets Oriented Bounding Box with node OpenCV SDK
//
/////////////////////////////////////////////////////////////////
const getOBB = (img) => {
const highThresh = 150
const iterations = 2
const lowThresh = 0
img.convertGrayscale()
img.gaussianBlur([3, 3])
img.canny(lowThresh, highThresh)
img.dilate(iterations)
const contours = img.findContours()
const clr = [0, 0, 255]
let largestAreaIndex = 0
let largestArea = 0
for (let i = 0; i < contours.size(); ++i) {
if (contours.area(i) > largestArea) {
largestArea = contours.area(i)
largestAreaIndex = i
}
}
return contours.minAreaRect(largestAreaIndex)
}
/////////////////////////////////////////////////////////////////
// Helper method for puppeteer
//
/////////////////////////////////////////////////////////////////
const setState = (page, state) => {
return page.evaluate((state) => {
window.setState(state)
}, state)
}
/////////////////////////////////////////////////////////////////
// Helper method for puppeteer
//
/////////////////////////////////////////////////////////////////
const clientToWorld = (page, {x, y}) => {
return new Promise(async(resolve, reject) => {
const onMessage = (msg) => {
page.removeListener('console', onMessage)
resolve (JSON.parse(msg._text))
}
page.on('console', onMessage)
page.evaluate((x, y) => {
console.log(window.clientToWorld(x,y))
}, x, y)
})
}
/////////////////////////////////////////////////////////////////
// Generates random GUID
//
/////////////////////////////////////////////////////////////////
const guid = (format='xxxxxxxxxxxx') => {
var d = new Date().getTime();
var guid = format.replace(
/[xy]/g,
function (c) {
var r = (d + Math.random() * 16) % 16 | 0;
d = Math.floor(d / 16);
return (c == 'x' ? r : (r & 0x7 | 0x8)).toString(16);
});
return guid
}
/////////////////////////////////////////////////////////
// Worker implementation
//
/////////////////////////////////////////////////////////
export default class Worker {
/////////////////////////////////////////////////////////
//
//
/////////////////////////////////////////////////////////
constructor (config) {
this.sendMessage = this.sendMessage.bind(this)
this.pid = process.pid
}
/////////////////////////////////////////////////////////
// Sends message to master process
//
/////////////////////////////////////////////////////////
sendMessage (msg) {
process.send(msg)
}
/////////////////////////////////////////////////////////
// Terminates worker
//
/////////////////////////////////////////////////////////
terminate () {
if (this.browser) {
this.browser.close()
}
}
/////////////////////////////////////////////////////////
// Fires an instance of puppeteer
// and loads Forge model from URN
//
/////////////////////////////////////////////////////////
async load (accessToken, urn) {
const browser = await puppeteer.launch({
headless: false,
args: [
'--hide-scrollbars',
'--mute-audio',
'--no-sandbox',
'--headless'
]
})
try {
const filename = pathUtils.resolve(
__dirname, '../..',
'./resources/viewer/viewer.html')
const url = `file://${filename}?accessToken=${accessToken}&urn=${urn}`
const page = await browser.newPage()
await page.goto(url)
await page.mainFrame().waitForSelector(
'.geometry-loaded', {
timeout: 300000
})
this.browser = browser
this.page = page
this.sendMessage({
data: 'loaded',
status: 200,
id: 'load'
})
} catch (ex) {
browser.close()
this.sendMessage({
status: 500,
id: 'load',
data: ex
})
}
}
/////////////////////////////////////////////////////////
// Gets Oriented Bounding Box
//
/////////////////////////////////////////////////////////
async getOBB (state, size) {
try {
await setState(this.page, state)
const path = pathUtils.resolve(
__dirname, '../..',
`./TMP/${guid()}.jpg`)
const clip = {
height: size.height,
width: size.width,
x: 0,
y: 0,
}
await this.page.setViewport(size)
await this.page.screenshot({
path,
clip
})
const img = await loadImage(path)
const obb = getOBB (img)
const p1 = await clientToWorld(this.page, obb.points[0])
const p2 = await clientToWorld(this.page, obb.points[1])
const p3 = await clientToWorld(this.page, obb.points[2])
const p4 = await clientToWorld(this.page, obb.points[3])
fs.unlink(path, (error) => {})
this.sendMessage({
data: [p1, p2, p3, p4],
status: 200,
id: 'obb'
})
} catch (ex) {
this.sendMessage({
status: 500,
id: 'obb',
data: ex
})
}
}
}The worker is being instantiated and controlled by server logic exposed through custom endpoints, here is their implementation:
import ServiceManager from '../services/SvcManager'
import compression from 'compression'
import cp from 'child_process'
import express from 'express'
import path from 'path'
/////////////////////////////////////////////////////////
// Tiny util class to manage workers
//
/////////////////////////////////////////////////////////
class WorkersMap {
constructor () {
this._workersMap = {}
}
addWorker (id, worker) {
if (this._workersMap[id]) {
this._workersMap[id].kill()
delete this._workersMap[id]
this._workersMap[id] = null
}
this._workersMap[id] = worker
}
removeWorker (id) {
if (this._workersMap[id]) {
this._workersMap.kill()
delete this._workersMap[id]
this._workersMap[id] = null
}
}
getWorker(id) {
return this._workersMap[id]
}
}
/////////////////////////////////////////////////////////
// API routes
//
/////////////////////////////////////////////////////////
module.exports = () => {
const socketSvc = ServiceManager.getService('SocketSvc')
/////////////////////////////////////////////////////////
//
//
/////////////////////////////////////////////////////////
const workersMap = new WorkersMap()
const router = express.Router()
const shouldCompress = (req, res) => {
return true
}
router.use(compression({
filter: shouldCompress
}))
socketSvc.on('disconnect', (id) => {
workersMap.removeWorker(id)
})
/////////////////////////////////////////////////////////
// Instanciate worker and loads Forge model
//
/////////////////////////////////////////////////////////
router.post('/worker/load', async (req, res) => {
try {
const socketSvc = ServiceManager.getService('SocketSvc')
res.json('loading')
const {urn, socketId} = req.body
const workerPath = path.resolve(
__dirname, '../../../../bin/worker')
const worker = cp.fork(workerPath, {
execArgv: [
'--max-old-space-size=4096',
'--gc_interval=100'
]
})
const handler = (msg) => {
switch (msg.id) {
case 'load':
worker.removeListener('message', handler)
if (msg.status === 200) {
workersMap.addWorker(
socketId, worker)
socketSvc.broadcast (
'opencv.loaded',
msg.data,
socketId)
} else {
worker.kill()
socketSvc.broadcast (
'opencv.error',
msg.data,
socketId)
}
}
}
worker.on('message', handler)
worker.on('error', (err) => {
console.log(err)
workersMap.removeWorker(socketId)
})
const forgeSvc = ServiceManager.getService('ForgeSvc')
const {access_token} = await forgeSvc.get2LeggedToken()
worker.send({
access_token,
id: 'load',
urn
})
} catch (ex) {
res.status(ex.status || 500)
res.json(ex)
}
})
/////////////////////////////////////////////////////////
// Request OBB from worker
//
/////////////////////////////////////////////////////////
router.post('/worker/obb/:socketId', async (req, res) => {
try {
const worker = workersMap.getWorker(
req.params.socketId)
if (!worker) {
res.status(404)
return res.json('Invalid socketId')
}
const {state, size} = req.body
const handler = (msg) => {
switch (msg.id) {
case 'obb':
worker.removeListener('message', handler)
res.status(msg.status || 500)
return res.json(msg.data)
}
}
worker.on('message', handler)
worker.send({
id: 'obb',
state,
size
})
} catch (ex) {
res.status(ex.status || 500)
res.json(ex)
}
})
return router
}
Is the feature really useful by itself...? Not really at that point, but I think this demo validates the approach and opens the road for more advanced use of the OpenCV library with Forge models in the future. This was at least an interesting and fun sample to create!
You can find the full source code at https://github.com/leefsmp/forge-cv and the live demo at https://forge-cv.autodesk.io. Below is a recording of the sample in action ...