diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d7f93d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/.babelrc +/.cache +/.yalc +/common +/css +/demo_util.js +/dist +/js +/node_modules +/yalc.lock +/yarn.lock \ No newline at end of file diff --git a/camera.js b/camera.js new file mode 100644 index 0000000..fb8fdd3 --- /dev/null +++ b/camera.js @@ -0,0 +1,584 @@ +/* eslint-disable max-len */ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import * as posenet from '@tensorflow-models/posenet'; +import '@tensorflow/tfjs-backend-webgl'; +import dat from 'dat.gui'; +import Stats from 'stats.js'; + +import {pointGetangle, rayCasting, drawBoundingBox, drawKeypoints, drawSkeleton, isMobile, toggleLoadingUI, tryResNetButtonName, tryResNetButtonText, updateTryResNetButtonDatGuiCss} from './demo_util'; + +const videoWidth = 540; +const videoHeight = 960; +const stats = new Stats(); + +/** + * Loads a the camera to be used in the demo + * + */ +async function setupCamera() { + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + throw new Error( + 'this browser does not support video capture,or this device does not have a camera'); + } + + const video = document.getElementById('video'); + video.width = videoWidth; + video.height = videoHeight; + + const mobile = isMobile(); + const stream = await navigator.mediaDevices.getUserMedia({ + 'audio': false, + 'video': { + facingMode: 'user', + width: mobile ? undefined : videoWidth, + height: mobile ? undefined : videoHeight, + }, + }); + video.srcObject = stream; + + return new Promise((resolve) => { + video.onloadedmetadata = () => { + resolve(video); + }; + }); +} + +async function loadVideo() { + const video = await setupCamera(); + video.play(); + + return video; +} + +const defaultQuantBytes = 2; + +const defaultMobileNetMultiplier = isMobile() ? 0.50 : 0.75; +const defaultMobileNetStride = 16; +const defaultMobileNetInputResolution = 500; + +const defaultResNetMultiplier = 1.0; +const defaultResNetStride = 32; +const defaultResNetInputResolution = 250; +let sitsecond=0; +let sittotalsecond=parseInt( document.getElementById('sittotalsecond').value); +let getinflag=false; +const setgetincount=3; +const setgetinsecond=3; +let getinsecond=0; +let timer1 =null; +let timer2 =null; + +const guiState = { + algorithm: 'multi-pose', + input: { + architecture: 'MobileNetV1', + outputStride: defaultMobileNetStride, + inputResolution: defaultMobileNetInputResolution, + multiplier: defaultMobileNetMultiplier, + quantBytes: defaultQuantBytes, + }, + singlePoseDetection: { + minPoseConfidence: 0.1, + minPartConfidence: 0.5, + }, + multiPoseDetection: { + maxPoseDetections: 5, + minPoseConfidence: 0.15, + minPartConfidence: 0.1, + nmsRadius: 30.0, + }, + output: { + showVideo: true, + showSkeleton: true, + showPoints: true, + showBoundingBox: false, + }, + net: null, +}; + +/** + * Sets up dat.gui controller on the top-right of the window + */ +function setupGui(cameras, net) { + guiState.net = net; + + if (cameras.length > 0) { + guiState.camera = cameras[0].deviceId; + } + + const gui = new dat.GUI({width: 300}); + + let architectureController = null; + guiState[tryResNetButtonName] = function() { + architectureController.setValue('ResNet50'); + }; + gui.add(guiState, tryResNetButtonName).name(tryResNetButtonText); + updateTryResNetButtonDatGuiCss(); + + // The single-pose algorithm is faster and simpler but requires only one + // person to be in the frame or results will be innaccurate. Multi-pose works + // for more than 1 person + const algorithmController = + gui.add(guiState, 'algorithm', ['single-pose', 'multi-pose']); + + // The input parameters have the most effect on accuracy and speed of the + // network + let input = gui.addFolder('Input'); + // Architecture: there are a few PoseNet models varying in size and + // accuracy. 1.01 is the largest, but will be the slowest. 0.50 is the + // fastest, but least accurate. + architectureController = + input.add(guiState.input, 'architecture', ['MobileNetV1', 'ResNet50']); + guiState.architecture = guiState.input.architecture; + // Input resolution: Internally, this parameter affects the height and width + // of the layers in the neural network. The higher the value of the input + // resolution the better the accuracy but slower the speed. + let inputResolutionController = null; + function updateGuiInputResolution( + inputResolution, + inputResolutionArray, + ) { + if (inputResolutionController) { + inputResolutionController.remove(); + } + guiState.inputResolution = inputResolution; + guiState.input.inputResolution = inputResolution; + inputResolutionController = + input.add(guiState.input, 'inputResolution', inputResolutionArray); + inputResolutionController.onChange(function(inputResolution) { + guiState.changeToInputResolution = inputResolution; + }); + } + + // Output stride: Internally, this parameter affects the height and width of + // the layers in the neural network. The lower the value of the output stride + // the higher the accuracy but slower the speed, the higher the value the + // faster the speed but lower the accuracy. + let outputStrideController = null; + function updateGuiOutputStride(outputStride, outputStrideArray) { + if (outputStrideController) { + outputStrideController.remove(); + } + guiState.outputStride = outputStride; + guiState.input.outputStride = outputStride; + outputStrideController = + input.add(guiState.input, 'outputStride', outputStrideArray); + outputStrideController.onChange(function(outputStride) { + guiState.changeToOutputStride = outputStride; + }); + } + + // Multiplier: this parameter affects the number of feature map channels in + // the MobileNet. The higher the value, the higher the accuracy but slower the + // speed, the lower the value the faster the speed but lower the accuracy. + let multiplierController = null; + function updateGuiMultiplier(multiplier, multiplierArray) { + if (multiplierController) { + multiplierController.remove(); + } + guiState.multiplier = multiplier; + guiState.input.multiplier = multiplier; + multiplierController = + input.add(guiState.input, 'multiplier', multiplierArray); + multiplierController.onChange(function(multiplier) { + guiState.changeToMultiplier = multiplier; + }); + } + + // QuantBytes: this parameter affects weight quantization in the ResNet50 + // model. The available options are 1 byte, 2 bytes, and 4 bytes. The higher + // the value, the larger the model size and thus the longer the loading time, + // the lower the value, the shorter the loading time but lower the accuracy. + let quantBytesController = null; + function updateGuiQuantBytes(quantBytes, quantBytesArray) { + if (quantBytesController) { + quantBytesController.remove(); + } + guiState.quantBytes = +quantBytes; + guiState.input.quantBytes = +quantBytes; + quantBytesController = + input.add(guiState.input, 'quantBytes', quantBytesArray); + quantBytesController.onChange(function(quantBytes) { + guiState.changeToQuantBytes = +quantBytes; + }); + } + + function updateGui() { + if (guiState.input.architecture === 'MobileNetV1') { + updateGuiInputResolution( + defaultMobileNetInputResolution, + [200, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 800]); + updateGuiOutputStride(defaultMobileNetStride, [8, 16]); + updateGuiMultiplier(defaultMobileNetMultiplier, [0.50, 0.75, 1.0]); + } else { // guiState.input.architecture === "ResNet50" + updateGuiInputResolution( + defaultResNetInputResolution, + [200, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 800]); + updateGuiOutputStride(defaultResNetStride, [32, 16]); + updateGuiMultiplier(defaultResNetMultiplier, [1.0]); + } + updateGuiQuantBytes(defaultQuantBytes, [1, 2, 4]); + } + + updateGui(); + input.open(); + // Pose confidence: the overall confidence in the estimation of a person's + // pose (i.e. a person detected in a frame) + // Min part confidence: the confidence that a particular estimated keypoint + // position is accurate (i.e. the elbow's position) + let single = gui.addFolder('Single Pose Detection'); + single.add(guiState.singlePoseDetection, 'minPoseConfidence', 0.0, 1.0); + single.add(guiState.singlePoseDetection, 'minPartConfidence', 0.0, 1.0); + + let multi = gui.addFolder('Multi Pose Detection'); + multi.add(guiState.multiPoseDetection, 'maxPoseDetections') + .min(1) + .max(20) + .step(1); + multi.add(guiState.multiPoseDetection, 'minPoseConfidence', 0.0, 1.0); + multi.add(guiState.multiPoseDetection, 'minPartConfidence', 0.0, 1.0); + // nms Radius: controls the minimum distance between poses that are returned + // defaults to 20, which is probably fine for most use cases + multi.add(guiState.multiPoseDetection, 'nmsRadius').min(0.0).max(40.0); + multi.open(); + + let output = gui.addFolder('Output'); + output.add(guiState.output, 'showVideo'); + output.add(guiState.output, 'showSkeleton'); + output.add(guiState.output, 'showPoints'); + output.add(guiState.output, 'showBoundingBox'); + output.open(); + + + architectureController.onChange(function(architecture) { + // if architecture is ResNet50, then show ResNet50 options + updateGui(); + guiState.changeToArchitecture = architecture; + }); + + algorithmController.onChange(function(value) { + switch (guiState.algorithm) { + case 'single-pose': + multi.close(); + single.open(); + break; + case 'multi-pose': + single.close(); + multi.open(); + break; + } + }); +} + +/** + * Sets up a frames per second panel on the top-left of the window + */ +function setupFPS() { + stats.showPanel(0); // 0: fps, 1: ms, 2: mb, 3+: custom + document.getElementById('main').appendChild(stats.dom); +} + +/** + * Feeds an image to posenet to estimate poses - this is where the magic + * happens. This function loops with a requestAnimationFrame method. + */ +function detectPoseInRealTime(video, net) { + const canvas = document.getElementById('output'); + const ctx = canvas.getContext('2d'); + + // since images are being fed from a webcam, we want to feed in the + // original image and then just flip the keypoints' x coordinates. If instead + // we flip the image, then correcting left-right keypoint pairs requires a + // permutation on all the keypoints. + const flipPoseHorizontal = true; + + canvas.width = videoWidth; + canvas.height = videoHeight; + + async function poseDetectionFrame() { + if (guiState.changeToArchitecture) { + // Important to purge variables and free up GPU memory + guiState.net.dispose(); + toggleLoadingUI(true); + guiState.net = await posenet.load({ + architecture: guiState.changeToArchitecture, + outputStride: guiState.outputStride, + inputResolution: guiState.inputResolution, + multiplier: guiState.multiplier, + }); + toggleLoadingUI(false); + guiState.architecture = guiState.changeToArchitecture; + guiState.changeToArchitecture = null; + } + + if (guiState.changeToMultiplier) { + guiState.net.dispose(); + toggleLoadingUI(true); + guiState.net = await posenet.load({ + architecture: guiState.architecture, + outputStride: guiState.outputStride, + inputResolution: guiState.inputResolution, + multiplier: +guiState.changeToMultiplier, + quantBytes: guiState.quantBytes, + }); + toggleLoadingUI(false); + guiState.multiplier = +guiState.changeToMultiplier; + guiState.changeToMultiplier = null; + } + + if (guiState.changeToOutputStride) { + // Important to purge variables and free up GPU memory + guiState.net.dispose(); + toggleLoadingUI(true); + guiState.net = await posenet.load({ + architecture: guiState.architecture, + outputStride: +guiState.changeToOutputStride, + inputResolution: guiState.inputResolution, + multiplier: guiState.multiplier, + quantBytes: guiState.quantBytes, + }); + toggleLoadingUI(false); + guiState.outputStride = +guiState.changeToOutputStride; + guiState.changeToOutputStride = null; + } + + if (guiState.changeToInputResolution) { + // Important to purge variables and free up GPU memory + guiState.net.dispose(); + toggleLoadingUI(true); + guiState.net = await posenet.load({ + architecture: guiState.architecture, + outputStride: guiState.outputStride, + inputResolution: +guiState.changeToInputResolution, + multiplier: guiState.multiplier, + quantBytes: guiState.quantBytes, + }); + toggleLoadingUI(false); + guiState.inputResolution = +guiState.changeToInputResolution; + guiState.changeToInputResolution = null; + } + + if (guiState.changeToQuantBytes) { + // Important to purge variables and free up GPU memory + guiState.net.dispose(); + toggleLoadingUI(true); + guiState.net = await posenet.load({ + architecture: guiState.architecture, + outputStride: guiState.outputStride, + inputResolution: guiState.inputResolution, + multiplier: guiState.multiplier, + quantBytes: guiState.changeToQuantBytes, + }); + toggleLoadingUI(false); + guiState.quantBytes = guiState.changeToQuantBytes; + guiState.changeToQuantBytes = null; + } + + // Begin monitoring code for frames per second + stats.begin(); + + let poses = []; + let minPoseConfidence; + let minPartConfidence; + ctx.rect( 20, 20, 200, 200); + ctx.strokeStyle = 'blue'; + ctx.stroke(); + switch (guiState.algorithm) { + case 'single-pose': + const pose = await guiState.net.estimatePoses(video, { + flipHorizontal: flipPoseHorizontal, + decodingMethod: 'single-person', + }); + poses = poses.concat(pose); + minPoseConfidence = +guiState.singlePoseDetection.minPoseConfidence; + minPartConfidence = +guiState.singlePoseDetection.minPartConfidence; + break; + case 'multi-pose': + let all_poses = await guiState.net.estimatePoses(video, { + flipHorizontal: flipPoseHorizontal, + decodingMethod: 'multi-person', + maxDetections: guiState.multiPoseDetection.maxPoseDetections, + scoreThreshold: guiState.multiPoseDetection.minPartConfidence, + nmsRadius: guiState.multiPoseDetection.nmsRadius, + }); + + poses = poses.concat(all_poses); + minPoseConfidence = +guiState.multiPoseDetection.minPoseConfidence; + minPartConfidence = +guiState.multiPoseDetection.minPartConfidence; + break; + } + + ctx.clearRect(0, 0, videoWidth, videoHeight); + + if (guiState.output.showVideo) { + ctx.save(); + ctx.scale(-1, 1); + ctx.translate(-videoWidth, 0); + ctx.drawImage(video, 0, 0, videoWidth, videoHeight); + ctx.restore(); + } + let msgbox=document.getElementById('msgbox'); + let leftShoulder=[];// 左肩 + let rightShoulder=[];// 右肩 + let leftHip=[];// 左臀 + let rightHip=[];// 右臀 + let leftKnee=[];// 左膝 + let rightKnee=[];// 右膝 + let leftAnkle=[];// 左脚踝 + let rightAnkle=[];// 有脚踝 + + let getincount=0; + // For each pose (i.e. person) detected in an image, loop through the poses + // and draw the resulting skeleton and keypoints if over certain confidence + // scores + poses.forEach(({score, keypoints}) => { + if (score >= minPoseConfidence) { + if (guiState.output.showPoints) { + drawKeypoints(keypoints, minPartConfidence, ctx); + } + if (guiState.output.showSkeleton) { + drawSkeleton(keypoints, minPartConfidence, ctx); + } + if (guiState.output.showBoundingBox) { + drawBoundingBox(keypoints, ctx); + } + // 区域判断 + keypoints.forEach(({position, part, score})=>{ + if (part=='leftShoulder'&&score>0.8) { + leftShoulder=[position.x, position.y]; + }; + if (part=='rightShoulder'&&score>0.8) { + rightShoulder=[position.x, position.y]; + }; + if (part=='leftHip'&&score>0.8) { + leftHip=[position.x, position.y]; + }; + if (part=='rightHip'&&score>0.8) { + rightHip=[position.x, position.y]; + }; + if (part=='leftKnee'&&score>0.8) { + leftKnee=[position.x, position.y]; + }; + if (part=='rightKnee'&&score>0.8) { + rightKnee=[position.x, position.y]; + }; + if (part=='leftAnkle'&&score>0.8) { + leftAnkle=[position.x, position.y]; + }; + if (part=='rightAnkle'&&score>0.8) { + rightAnkle=[position.x, position.y]; + }; + if (rayCasting([position.x, position.y], [[20, 20], [20, 220], [220, 220], [220, 20]])) { + getincount+=1; + } + }); + } + }); + + if (getincount>=setgetincount&&getinsecond==0&&getinflag==false) { + getinflag=true; + document.getElementById('getinbox').style.display='block'; + timer2= window.setInterval(function() { + getinsecond+=1; + if (getinsecond>=setgetinsecond) { + window.clearInterval(timer2); + getinsecond=0; + getinflag=false; + document.getElementById('getinbox').style.display='none'; + } + }, 1000); + } + + // 左边判断 + if (leftShoulder!=[]&&leftHip!=[]&&leftKnee!=[]&&leftAnkle!=[]) { + const anglea= pointGetangle(leftShoulder, leftHip, leftKnee); + const angleb=pointGetangle(leftHip, leftKnee, leftAnkle); + if (anglea>=80&&anglea<=100&&angleb>=80&&angleb<=100) { + if (sitsecond==0) { + timer1=window.setInterval(function() { + sitsecond+=1; + }, 1000); + } + } else { + window.clearInterval(timer1); + if (sitsecond>3) { + msgbox.innerHTML=msgbox.innerHTML+'
静坐结束！
'; + } + sitsecond=0; + } + } + if (sitsecond==3) { + msgbox.innerHTML=msgbox.innerHTML+'
静坐开始！
'; + } + if (sitsecond==sittotalsecond) { + msgbox.innerHTML=msgbox.innerHTML+'
静坐完成！
'; + window.clearInterval(timer1); + sitsecond=0; + } + // End monitoring code for frames per second + stats.end(); + + requestAnimationFrame(poseDetectionFrame); + } + + poseDetectionFrame(); +} + +/** + * Kicks off the demo by loading the posenet model, finding and loading + * available camera devices, and setting off the detectPoseInRealTime function. + */ +export async function bindPage() { + toggleLoadingUI(true); + const net = await posenet.load({ + architecture: guiState.input.architecture, + outputStride: guiState.input.outputStride, + inputResolution: guiState.input.inputResolution, + multiplier: guiState.input.multiplier, + quantBytes: guiState.input.quantBytes, + }); + toggleLoadingUI(false); + + let video; + + try { + video = await loadVideo(); + } catch (e) { + let info = document.getElementById('info'); + info.textContent = 'this browser does not support video capture,' + + 'or this device does not have a camera'; + info.style.display = 'block'; + throw e; + } + + setupGui([], net); + setupFPS(); + detectPoseInRealTime(video, net); +} + +navigator.getUserMedia = navigator.getUserMedia || + navigator.webkitGetUserMedia || navigator.mozGetUserMedia; +// kick off the demo + +document.getElementById('begin').onclick=function() { + bindPage(); +}; +document.getElementById('end').onclick=function() { + bindPage(); +}; + diff --git a/image.html b/image.html new file mode 100644 index 0000000..6f58262 --- /dev/null +++ b/image.html @@ -0,0 +1,83 @@ + + + + + + + + + + +
+ +
+ + + + \ No newline at end of file diff --git a/images/qp04.jpg b/images/qp04.jpg new file mode 100644 index 0000000..60a2cf7 Binary files /dev/null and b/images/qp04.jpg differ diff --git a/index.html b/index.html new file mode 100644 index 0000000..1cf52c4 --- /dev/null +++ b/index.html @@ -0,0 +1,108 @@ + + + + + PoseNet - Camera Feed Demo + + + + + + + +
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ + + + diff --git a/modules/posenet/base_model.d.ts b/modules/posenet/base_model.d.ts new file mode 100644 index 0000000..dd8e514 --- /dev/null +++ b/modules/posenet/base_model.d.ts @@ -0,0 +1,63 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import * as tfconv from '@tensorflow/tfjs-converter'; +import * as tf from '@tensorflow/tfjs-core'; +import { PoseNetOutputStride } from './types'; +/** + * PoseNet supports using various convolution neural network models + * (e.g. ResNet and MobileNetV1) as its underlying base model. + * The following BaseModel interface defines a unified interface for + * creating such PoseNet base models. Currently both MobileNet (in + * ./mobilenet.ts) and ResNet (in ./resnet.ts) implements the BaseModel + * interface. New base models that conform to the BaseModel interface can be + * added to PoseNet. + */ +export declare abstract class BaseModel { + protected readonly model: tfconv.GraphModel; + readonly outputStride: PoseNetOutputStride; + constructor(model: tfconv.GraphModel, outputStride: PoseNetOutputStride); + abstract preprocessInput(input: tf.Tensor3D): tf.Tensor3D; + /** + * Predicts intermediate Tensor representations. + * + * @param input The input RGB image of the base model. + * A Tensor of shape: [`inputResolution`, `inputResolution`, 3]. + * + * @return A dictionary of base model's intermediate predictions. + * The returned dictionary should contains the following elements: + * heatmapScores: A Tensor3D that represents the heatmapScores. + * offsets: A Tensor3D that represents the offsets. + * displacementFwd: A Tensor3D that represents the forward displacement. + * displacementBwd: A Tensor3D that represents the backward displacement. + */ + predict(input: tf.Tensor3D): { + heatmapScores: tf.Tensor3D; + offsets: tf.Tensor3D; + displacementFwd: tf.Tensor3D; + displacementBwd: tf.Tensor3D; + }; + abstract nameOutputResults(results: tf.Tensor3D[]): { + heatmap: tf.Tensor3D; + offsets: tf.Tensor3D; + displacementFwd: tf.Tensor3D; + displacementBwd: tf.Tensor3D; + }; + /** + * Releases the CPU and GPU memory allocated by the model. + */ + dispose(): void; +} diff --git a/modules/posenet/base_model.js b/modules/posenet/base_model.js new file mode 100644 index 0000000..1148352 --- /dev/null +++ b/modules/posenet/base_model.js @@ -0,0 +1,75 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var tf = require("@tensorflow/tfjs-core"); +/** + * PoseNet supports using various convolution neural network models + * (e.g. ResNet and MobileNetV1) as its underlying base model. + * The following BaseModel interface defines a unified interface for + * creating such PoseNet base models. Currently both MobileNet (in + * ./mobilenet.ts) and ResNet (in ./resnet.ts) implements the BaseModel + * interface. New base models that conform to the BaseModel interface can be + * added to PoseNet. + */ +var BaseModel = /** @class */ (function () { + function BaseModel(model, outputStride) { + this.model = model; + this.outputStride = outputStride; + var inputShape = this.model.inputs[0].shape; + tf.util.assert((inputShape[1] === -1) && (inputShape[2] === -1), function () { return "Input shape [" + inputShape[1] + ", " + inputShape[2] + "] " + + "must both be equal to or -1"; }); + } + /** + * Predicts intermediate Tensor representations. + * + * @param input The input RGB image of the base model. + * A Tensor of shape: [`inputResolution`, `inputResolution`, 3]. + * + * @return A dictionary of base model's intermediate predictions. + * The returned dictionary should contains the following elements: + * heatmapScores: A Tensor3D that represents the heatmapScores. + * offsets: A Tensor3D that represents the offsets. + * displacementFwd: A Tensor3D that represents the forward displacement. + * displacementBwd: A Tensor3D that represents the backward displacement. + */ + BaseModel.prototype.predict = function (input) { + var _this = this; + return tf.tidy(function () { + var asFloat = _this.preprocessInput(tf.cast(input, 'float32')); + var asBatch = tf.expandDims(asFloat, 0); + var results = _this.model.predict(asBatch); + var results3d = results.map(function (y) { return tf.squeeze(y, [0]); }); + var namedResults = _this.nameOutputResults(results3d); + return { + heatmapScores: tf.sigmoid(namedResults.heatmap), + offsets: namedResults.offsets, + displacementFwd: namedResults.displacementFwd, + displacementBwd: namedResults.displacementBwd + }; + }); + }; + /** + * Releases the CPU and GPU memory allocated by the model. + */ + BaseModel.prototype.dispose = function () { + this.model.dispose(); + }; + return BaseModel; +}()); +exports.BaseModel = BaseModel; +//# sourceMappingURL=base_model.js.map \ No newline at end of file diff --git a/modules/posenet/base_model.js.map b/modules/posenet/base_model.js.map new file mode 100644 index 0000000..9870e40 --- /dev/null +++ b/modules/posenet/base_model.js.map @@ -0,0 +1 @@ +{"version":3,"file":"base_model.js","sourceRoot":"","sources":["../src/base_model.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAGH,0CAA4C;AAG5C;;;;;;;;GAQG;AACH;IACE,mBACuB,KAAwB,EAC3B,YAAiC;QAD9B,UAAK,GAAL,KAAK,CAAmB;QAC3B,iBAAY,GAAZ,YAAY,CAAqB;QACnD,IAAM,UAAU,GACZ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAyC,CAAC;QACnE,EAAE,CAAC,IAAI,CAAC,MAAM,CACV,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAChD,cAAM,OAAA,kBAAgB,UAAU,CAAC,CAAC,CAAC,UAAK,UAAU,CAAC,CAAC,CAAC,OAAI;YACrD,6BAA6B,EAD3B,CAC2B,CAAC,CAAC;IACzC,CAAC;IAID;;;;;;;;;;;;OAYG;IACH,2BAAO,GAAP,UAAQ,KAAkB;QAA1B,iBAqBC;QAfC,OAAO,EAAE,CAAC,IAAI,CAAC;YACb,IAAM,OAAO,GAAG,KAAI,CAAC,eAAe,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;YAChE,IAAM,OAAO,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC1C,IAAM,OAAO,GAAG,KAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAkB,CAAC;YAC7D,IAAM,SAAS,GAAkB,OAAO,CAAC,GAAG,CAAC,UAAA,CAAC,IAAI,OAAA,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAlB,CAAkB,CAAC,CAAC;YAEtE,IAAM,YAAY,GAAG,KAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEvD,OAAO;gBACL,aAAa,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC;gBAC/C,OAAO,EAAE,YAAY,CAAC,OAAO;gBAC7B,eAAe,EAAE,YAAY,CAAC,eAAe;gBAC7C,eAAe,EAAE,YAAY,CAAC,eAAe;aAC9C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAWD;;OAEG;IACH,2BAAO,GAAP;QACE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IACvB,CAAC;IACH,gBAAC;AAAD,CAAC,AAjED,IAiEC;AAjEqB,8BAAS"} \ No newline at end of file diff --git a/modules/posenet/checkpoints.d.ts b/modules/posenet/checkpoints.d.ts new file mode 100644 index 0000000..2e948b6 --- /dev/null +++ b/modules/posenet/checkpoints.d.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +export declare function resNet50Checkpoint(stride: number, quantBytes: number): string; +export declare function mobileNetCheckpoint(stride: number, multiplier: number, quantBytes: number): string; diff --git a/modules/posenet/checkpoints.js b/modules/posenet/checkpoints.js new file mode 100644 index 0000000..627d54e --- /dev/null +++ b/modules/posenet/checkpoints.js @@ -0,0 +1,49 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var MOBILENET_BASE_URL = 'https://storage.googleapis.com/tfjs-models/savedmodel/posenet/mobilenet/'; +var RESNET50_BASE_URL = 'https://storage.googleapis.com/tfjs-models/savedmodel/posenet/resnet50/'; +// The PoseNet 2.0 ResNet50 models use the latest TensorFlow.js 1.0 model +// format. +function resNet50Checkpoint(stride, quantBytes) { + var graphJson = "model-stride" + stride + ".json"; + // quantBytes=4 corresponding to the non-quantized full-precision checkpoints. + if (quantBytes === 4) { + return RESNET50_BASE_URL + "float/" + graphJson; + } + else { + return RESNET50_BASE_URL + ("quant" + quantBytes + "/") + graphJson; + } +} +exports.resNet50Checkpoint = resNet50Checkpoint; +// The PoseNet 2.0 MobileNetV1 models use the latest TensorFlow.js 1.0 model +// format. +function mobileNetCheckpoint(stride, multiplier, quantBytes) { + var toStr = { 1.0: '100', 0.75: '075', 0.50: '050' }; + var graphJson = "model-stride" + stride + ".json"; + // quantBytes=4 corresponding to the non-quantized full-precision checkpoints. + if (quantBytes === 4) { + return MOBILENET_BASE_URL + ("float/" + toStr[multiplier] + "/") + graphJson; + } + else { + return MOBILENET_BASE_URL + ("quant" + quantBytes + "/" + toStr[multiplier] + "/") + + graphJson; + } +} +exports.mobileNetCheckpoint = mobileNetCheckpoint; +//# sourceMappingURL=checkpoints.js.map \ No newline at end of file diff --git a/modules/posenet/checkpoints.js.map b/modules/posenet/checkpoints.js.map new file mode 100644 index 0000000..b36e477 --- /dev/null +++ b/modules/posenet/checkpoints.js.map @@ -0,0 +1 @@ +{"version":3,"file":"checkpoints.js","sourceRoot":"","sources":["../src/checkpoints.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAEH,IAAM,kBAAkB,GACpB,0EAA0E,CAAC;AAC/E,IAAM,iBAAiB,GACnB,yEAAyE,CAAC;AAE9E,yEAAyE;AACzE,UAAU;AACV,SAAgB,kBAAkB,CAAC,MAAc,EAAE,UAAkB;IACnE,IAAM,SAAS,GAAG,iBAAe,MAAM,UAAO,CAAC;IAC/C,8EAA8E;IAC9E,IAAI,UAAU,KAAK,CAAC,EAAE;QACpB,OAAO,iBAAiB,GAAG,QAAQ,GAAG,SAAS,CAAC;KACjD;SAAM;QACL,OAAO,iBAAiB,IAAG,UAAQ,UAAU,MAAG,CAAA,GAAG,SAAS,CAAC;KAC9D;AACH,CAAC;AARD,gDAQC;AAED,4EAA4E;AAC5E,UAAU;AACV,SAAgB,mBAAmB,CAC/B,MAAc,EAAE,UAAkB,EAAE,UAAkB;IACxD,IAAM,KAAK,GAA4B,EAAC,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAC,CAAC;IAC9E,IAAM,SAAS,GAAG,iBAAe,MAAM,UAAO,CAAC;IAC/C,8EAA8E;IAC9E,IAAI,UAAU,KAAK,CAAC,EAAE;QACpB,OAAO,kBAAkB,IAAG,WAAS,KAAK,CAAC,UAAU,CAAC,MAAG,CAAA,GAAG,SAAS,CAAC;KACvE;SAAM;QACL,OAAO,kBAAkB,IAAG,UAAQ,UAAU,SAAI,KAAK,CAAC,UAAU,CAAC,MAAG,CAAA;YAClE,SAAS,CAAC;KACf;AACH,CAAC;AAXD,kDAWC"} \ No newline at end of file diff --git a/modules/posenet/index.d.ts b/modules/posenet/index.d.ts new file mode 100644 index 0000000..7107c96 --- /dev/null +++ b/modules/posenet/index.d.ts @@ -0,0 +1,25 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import { MobileNet } from './mobilenet'; +import { decodeMultiplePoses } from './multi_pose/decode_multiple_poses'; +import { decodeSinglePose } from './single_pose/decode_single_pose'; +export { partChannels, partIds, partNames, poseChain } from './keypoints'; +export { load, ModelConfig, MultiPersonInferenceConfig, PoseNet, SinglePersonInterfaceConfig } from './posenet_model'; +export { InputResolution, Keypoint, MobileNetMultiplier, Pose, PoseNetOutputStride } from './types'; +export { getAdjacentKeyPoints, getBoundingBox, getBoundingBoxPoints, scaleAndFlipPoses, scalePose } from './util'; +export { version } from './version'; +export { decodeMultiplePoses, decodeSinglePose, MobileNet }; diff --git a/modules/posenet/index.js b/modules/posenet/index.js new file mode 100644 index 0000000..e9e60e5 --- /dev/null +++ b/modules/posenet/index.js @@ -0,0 +1,41 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var mobilenet_1 = require("./mobilenet"); +exports.MobileNet = mobilenet_1.MobileNet; +var decode_multiple_poses_1 = require("./multi_pose/decode_multiple_poses"); +exports.decodeMultiplePoses = decode_multiple_poses_1.decodeMultiplePoses; +var decode_single_pose_1 = require("./single_pose/decode_single_pose"); +exports.decodeSinglePose = decode_single_pose_1.decodeSinglePose; +var keypoints_1 = require("./keypoints"); +exports.partChannels = keypoints_1.partChannels; +exports.partIds = keypoints_1.partIds; +exports.partNames = keypoints_1.partNames; +exports.poseChain = keypoints_1.poseChain; +var posenet_model_1 = require("./posenet_model"); +exports.load = posenet_model_1.load; +exports.PoseNet = posenet_model_1.PoseNet; +var util_1 = require("./util"); +exports.getAdjacentKeyPoints = util_1.getAdjacentKeyPoints; +exports.getBoundingBox = util_1.getBoundingBox; +exports.getBoundingBoxPoints = util_1.getBoundingBoxPoints; +exports.scaleAndFlipPoses = util_1.scaleAndFlipPoses; +exports.scalePose = util_1.scalePose; +var version_1 = require("./version"); +exports.version = version_1.version; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/modules/posenet/index.js.map b/modules/posenet/index.js.map new file mode 100644 index 0000000..42a51cc --- /dev/null +++ b/modules/posenet/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAEH,yCAAsC;AASS,oBATvC,qBAAS,CASuC;AARxD,4EAAuE;AAQ/D,8BARA,2CAAmB,CAQA;AAP3B,uEAAkE;AAOrC,2BAPrB,qCAAgB,CAOqB;AAL7C,yCAAwE;AAAhE,mCAAA,YAAY,CAAA;AAAE,8BAAA,OAAO,CAAA;AAAE,gCAAA,SAAS,CAAA;AAAE,gCAAA,SAAS,CAAA;AACnD,iDAAoH;AAA5G,+BAAA,IAAI,CAAA;AAA2C,kCAAA,OAAO,CAAA;AAE9D,+BAAgH;AAAxG,sCAAA,oBAAoB,CAAA;AAAE,gCAAA,cAAc,CAAA;AAAE,sCAAA,oBAAoB,CAAA;AAAE,mCAAA,iBAAiB,CAAA;AAAE,2BAAA,SAAS,CAAA;AAChG,qCAAkC;AAA1B,4BAAA,OAAO,CAAA"} \ No newline at end of file diff --git a/modules/posenet/keypoints.d.ts b/modules/posenet/keypoints.d.ts new file mode 100644 index 0000000..a446ca5 --- /dev/null +++ b/modules/posenet/keypoints.d.ts @@ -0,0 +1,28 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +export declare type Tuple = [T, T]; +export declare type StringTuple = Tuple; +export declare type NumberTuple = Tuple; +export declare const partNames: string[]; +export declare const NUM_KEYPOINTS: number; +export interface NumberDict { + [jointName: string]: number; +} +export declare const partIds: NumberDict; +export declare const poseChain: StringTuple[]; +export declare const connectedPartIndices: number[][]; +export declare const partChannels: string[]; diff --git a/modules/posenet/keypoints.js b/modules/posenet/keypoints.js new file mode 100644 index 0000000..e80f316 --- /dev/null +++ b/modules/posenet/keypoints.js @@ -0,0 +1,83 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.partNames = [ + 'nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftShoulder', + 'rightShoulder', 'leftElbow', 'rightElbow', 'leftWrist', 'rightWrist', + 'leftHip', 'rightHip', 'leftKnee', 'rightKnee', 'leftAnkle', 'rightAnkle' +]; +exports.NUM_KEYPOINTS = exports.partNames.length; +exports.partIds = exports.partNames.reduce(function (result, jointName, i) { + result[jointName] = i; + return result; +}, {}); +var connectedPartNames = [ + ['leftHip', 'leftShoulder'], ['leftElbow', 'leftShoulder'], + ['leftElbow', 'leftWrist'], ['leftHip', 'leftKnee'], + ['leftKnee', 'leftAnkle'], ['rightHip', 'rightShoulder'], + ['rightElbow', 'rightShoulder'], ['rightElbow', 'rightWrist'], + ['rightHip', 'rightKnee'], ['rightKnee', 'rightAnkle'], + ['leftShoulder', 'rightShoulder'], ['leftHip', 'rightHip'] +]; +/* + * Define the skeleton. This defines the parent->child relationships of our + * tree. Arbitrarily this defines the nose as the root of the tree, however + * since we will infer the displacement for both parent->child and + * child->parent, we can define the tree root as any node. + */ +exports.poseChain = [ + ['nose', 'leftEye'], ['leftEye', 'leftEar'], ['nose', 'rightEye'], + ['rightEye', 'rightEar'], ['nose', 'leftShoulder'], + ['leftShoulder', 'leftElbow'], ['leftElbow', 'leftWrist'], + ['leftShoulder', 'leftHip'], ['leftHip', 'leftKnee'], + ['leftKnee', 'leftAnkle'], ['nose', 'rightShoulder'], + ['rightShoulder', 'rightElbow'], ['rightElbow', 'rightWrist'], + ['rightShoulder', 'rightHip'], ['rightHip', 'rightKnee'], + ['rightKnee', 'rightAnkle'] +]; +exports.connectedPartIndices = connectedPartNames.map(function (_a) { + var jointNameA = _a[0], jointNameB = _a[1]; + return ([exports.partIds[jointNameA], exports.partIds[jointNameB]]); +}); +exports.partChannels = [ + 'left_face', + 'right_face', + 'right_upper_leg_front', + 'right_lower_leg_back', + 'right_upper_leg_back', + 'left_lower_leg_front', + 'left_upper_leg_front', + 'left_upper_leg_back', + 'left_lower_leg_back', + 'right_feet', + 'right_lower_leg_front', + 'left_feet', + 'torso_front', + 'torso_back', + 'right_upper_arm_front', + 'right_upper_arm_back', + 'right_lower_arm_back', + 'left_lower_arm_front', + 'left_upper_arm_front', + 'left_upper_arm_back', + 'left_lower_arm_back', + 'right_hand', + 'right_lower_arm_front', + 'left_hand' +]; +//# sourceMappingURL=keypoints.js.map \ No newline at end of file diff --git a/modules/posenet/keypoints.js.map b/modules/posenet/keypoints.js.map new file mode 100644 index 0000000..888d9d5 --- /dev/null +++ b/modules/posenet/keypoints.js.map @@ -0,0 +1 @@ +{"version":3,"file":"keypoints.js","sourceRoot":"","sources":["../src/keypoints.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAMU,QAAA,SAAS,GAAG;IACvB,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc;IACpE,eAAe,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY;IACrE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY;CAC1E,CAAC;AAEW,QAAA,aAAa,GAAG,iBAAS,CAAC,MAAM,CAAC;AAMjC,QAAA,OAAO,GAChB,iBAAS,CAAC,MAAM,CAAC,UAAC,MAAkB,EAAE,SAAS,EAAE,CAAC;IAChD,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC;AAChB,CAAC,EAAE,EAAE,CAAe,CAAC;AAEzB,IAAM,kBAAkB,GAAkB;IACxC,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC,WAAW,EAAE,cAAc,CAAC;IAC1D,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;IACnD,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,UAAU,EAAE,eAAe,CAAC;IACxD,CAAC,YAAY,EAAE,eAAe,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,CAAC;IAC7D,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,WAAW,EAAE,YAAY,CAAC;IACtD,CAAC,cAAc,EAAE,eAAe,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;CAC3D,CAAC;AAEF;;;;;GAKG;AACU,QAAA,SAAS,GAAkB;IACtC,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC;IACjE,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,cAAc,CAAC;IAClD,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;IACzD,CAAC,cAAc,EAAE,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;IACpD,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,CAAC;IACpD,CAAC,eAAe,EAAE,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,CAAC;IAC7D,CAAC,eAAe,EAAE,UAAU,CAAC,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC;IACxD,CAAC,WAAW,EAAE,YAAY,CAAC;CAC5B,CAAC;AAEW,QAAA,oBAAoB,GAAG,kBAAkB,CAAC,GAAG,CACtD,UAAC,EAAwB;QAAvB,kBAAU,EAAE,kBAAU;IAAM,OAAA,CAAC,CAAC,eAAO,CAAC,UAAU,CAAC,EAAE,eAAO,CAAC,UAAU,CAAC,CAAC,CAAC;AAA5C,CAA4C,CAAC,CAAC;AAEnE,QAAA,YAAY,GAAa;IACpC,WAAW;IACX,YAAY;IACZ,uBAAuB;IACvB,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,qBAAqB;IACrB,qBAAqB;IACrB,YAAY;IACZ,uBAAuB;IACvB,WAAW;IACX,aAAa;IACb,YAAY;IACZ,uBAAuB;IACvB,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,qBAAqB;IACrB,qBAAqB;IACrB,YAAY;IACZ,uBAAuB;IACvB,WAAW;CACZ,CAAC"} \ No newline at end of file diff --git a/modules/posenet/mobilenet.d.ts b/modules/posenet/mobilenet.d.ts new file mode 100644 index 0000000..9eaaff1 --- /dev/null +++ b/modules/posenet/mobilenet.d.ts @@ -0,0 +1,27 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import * as tf from '@tensorflow/tfjs-core'; +import { BaseModel } from './base_model'; +export declare class MobileNet extends BaseModel { + preprocessInput(input: tf.Tensor3D): tf.Tensor3D; + nameOutputResults(results: tf.Tensor3D[]): { + offsets: tf.Tensor; + heatmap: tf.Tensor; + displacementFwd: tf.Tensor; + displacementBwd: tf.Tensor; + }; +} diff --git a/modules/posenet/mobilenet.js b/modules/posenet/mobilenet.js new file mode 100644 index 0000000..d091f49 --- /dev/null +++ b/modules/posenet/mobilenet.js @@ -0,0 +1,50 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +var tf = require("@tensorflow/tfjs-core"); +var base_model_1 = require("./base_model"); +var MobileNet = /** @class */ (function (_super) { + __extends(MobileNet, _super); + function MobileNet() { + return _super !== null && _super.apply(this, arguments) || this; + } + MobileNet.prototype.preprocessInput = function (input) { + // Normalize the pixels [0, 255] to be between [-1, 1]. + return tf.tidy(function () { return tf.sub(tf.div(input, 127.5), 1.0); }); + }; + MobileNet.prototype.nameOutputResults = function (results) { + var offsets = results[0], heatmap = results[1], displacementFwd = results[2], displacementBwd = results[3]; + return { offsets: offsets, heatmap: heatmap, displacementFwd: displacementFwd, displacementBwd: displacementBwd }; + }; + return MobileNet; +}(base_model_1.BaseModel)); +exports.MobileNet = MobileNet; +//# sourceMappingURL=mobilenet.js.map \ No newline at end of file diff --git a/modules/posenet/mobilenet.js.map b/modules/posenet/mobilenet.js.map new file mode 100644 index 0000000..6bec315 --- /dev/null +++ b/modules/posenet/mobilenet.js.map @@ -0,0 +1 @@ +{"version":3,"file":"mobilenet.js","sourceRoot":"","sources":["../src/mobilenet.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;;;;;;;;;;;;;;AAEH,0CAA4C;AAC5C,2CAAuC;AAEvC;IAA+B,6BAAS;IAAxC;;IAUA,CAAC;IATC,mCAAe,GAAf,UAAgB,KAAkB;QAChC,uDAAuD;QACvD,OAAO,EAAE,CAAC,IAAI,CAAC,cAAM,OAAA,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,GAAG,CAAC,EAAjC,CAAiC,CAAC,CAAC;IAC1D,CAAC;IAED,qCAAiB,GAAjB,UAAkB,OAAsB;QAC/B,IAAA,oBAAO,EAAE,oBAAO,EAAE,4BAAe,EAAE,4BAAe,CAAY;QACrE,OAAO,EAAC,OAAO,SAAA,EAAE,OAAO,SAAA,EAAE,eAAe,iBAAA,EAAE,eAAe,iBAAA,EAAC,CAAC;IAC9D,CAAC;IACH,gBAAC;AAAD,CAAC,AAVD,CAA+B,sBAAS,GAUvC;AAVY,8BAAS"} \ No newline at end of file diff --git a/modules/posenet/model_weights.d.ts b/modules/posenet/model_weights.d.ts new file mode 100644 index 0000000..700fa27 --- /dev/null +++ b/modules/posenet/model_weights.d.ts @@ -0,0 +1,28 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import * as tf from '@tensorflow/tfjs-core'; +export declare class ModelWeights { + private variables; + constructor(variables: { + [varName: string]: tf.Tensor; + }); + weights(layerName: string): tf.Tensor; + depthwiseBias(layerName: string): tf.Tensor; + convBias(layerName: string): tf.Tensor; + depthwiseWeights(layerName: string): tf.Tensor; + dispose(): void; +} diff --git a/modules/posenet/model_weights.js b/modules/posenet/model_weights.js new file mode 100644 index 0000000..b5528c1 --- /dev/null +++ b/modules/posenet/model_weights.js @@ -0,0 +1,43 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var ModelWeights = /** @class */ (function () { + function ModelWeights(variables) { + this.variables = variables; + } + ModelWeights.prototype.weights = function (layerName) { + return this.variables["MobilenetV1/" + layerName + "/weights"]; + }; + ModelWeights.prototype.depthwiseBias = function (layerName) { + return this.variables["MobilenetV1/" + layerName + "/biases"]; + }; + ModelWeights.prototype.convBias = function (layerName) { + return this.depthwiseBias(layerName); + }; + ModelWeights.prototype.depthwiseWeights = function (layerName) { + return this.variables["MobilenetV1/" + layerName + "/depthwise_weights"]; + }; + ModelWeights.prototype.dispose = function () { + for (var varName in this.variables) { + this.variables[varName].dispose(); + } + }; + return ModelWeights; +}()); +exports.ModelWeights = ModelWeights; +//# sourceMappingURL=model_weights.js.map \ No newline at end of file diff --git a/modules/posenet/model_weights.js.map b/modules/posenet/model_weights.js.map new file mode 100644 index 0000000..b13448f --- /dev/null +++ b/modules/posenet/model_weights.js.map @@ -0,0 +1 @@ +{"version":3,"file":"model_weights.js","sourceRoot":"","sources":["../src/model_weights.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAIH;IAGE,sBAAY,SAAyC;QACnD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,8BAAO,GAAP,UAAQ,SAAiB;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,iBAAe,SAAS,aAAU,CAAgB,CAAC;IAC3E,CAAC;IAED,oCAAa,GAAb,UAAc,SAAiB;QAC7B,OAAO,IAAI,CAAC,SAAS,CAAC,iBAAe,SAAS,YAAS,CAAgB,CAAC;IAC1E,CAAC;IAED,+BAAQ,GAAR,UAAS,SAAiB;QACxB,OAAO,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;IACvC,CAAC;IAED,uCAAgB,GAAhB,UAAiB,SAAiB;QAChC,OAAO,IAAI,CAAC,SAAS,CAAC,iBAAe,SAAS,uBAAoB,CACnD,CAAC;IAClB,CAAC;IAED,8BAAO,GAAP;QACE,KAAK,IAAM,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE;YACpC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;SACnC;IACH,CAAC;IACH,mBAAC;AAAD,CAAC,AA7BD,IA6BC;AA7BY,oCAAY"} \ No newline at end of file diff --git a/modules/posenet/multi_pose/build_part_with_score_queue.d.ts b/modules/posenet/multi_pose/build_part_with_score_queue.d.ts new file mode 100644 index 0000000..e82d8bf --- /dev/null +++ b/modules/posenet/multi_pose/build_part_with_score_queue.d.ts @@ -0,0 +1,24 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import { PartWithScore, TensorBuffer3D } from '../types'; +import { MaxHeap } from './max_heap'; +/** + * Builds a priority queue with part candidate positions for a specific image in + * the batch. For this we find all local maxima in the score maps with score + * values above a threshold. We create a single priority queue across all parts. + */ +export declare function buildPartWithScoreQueue(scoreThreshold: number, localMaximumRadius: number, scores: TensorBuffer3D): MaxHeap; diff --git a/modules/posenet/multi_pose/build_part_with_score_queue.js b/modules/posenet/multi_pose/build_part_with_score_queue.js new file mode 100644 index 0000000..5014f50 --- /dev/null +++ b/modules/posenet/multi_pose/build_part_with_score_queue.js @@ -0,0 +1,70 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var max_heap_1 = require("./max_heap"); +function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, localMaximumRadius, scores) { + var _a = scores.shape, height = _a[0], width = _a[1]; + var localMaximum = true; + var yStart = Math.max(heatmapY - localMaximumRadius, 0); + var yEnd = Math.min(heatmapY + localMaximumRadius + 1, height); + for (var yCurrent = yStart; yCurrent < yEnd; ++yCurrent) { + var xStart = Math.max(heatmapX - localMaximumRadius, 0); + var xEnd = Math.min(heatmapX + localMaximumRadius + 1, width); + for (var xCurrent = xStart; xCurrent < xEnd; ++xCurrent) { + if (scores.get(yCurrent, xCurrent, keypointId) > score) { + localMaximum = false; + break; + } + } + if (!localMaximum) { + break; + } + } + return localMaximum; +} +/** + * Builds a priority queue with part candidate positions for a specific image in + * the batch. For this we find all local maxima in the score maps with score + * values above a threshold. We create a single priority queue across all parts. + */ +function buildPartWithScoreQueue(scoreThreshold, localMaximumRadius, scores) { + var _a = scores.shape, height = _a[0], width = _a[1], numKeypoints = _a[2]; + var queue = new max_heap_1.MaxHeap(height * width * numKeypoints, function (_a) { + var score = _a.score; + return score; + }); + for (var heatmapY = 0; heatmapY < height; ++heatmapY) { + for (var heatmapX = 0; heatmapX < width; ++heatmapX) { + for (var keypointId = 0; keypointId < numKeypoints; ++keypointId) { + var score = scores.get(heatmapY, heatmapX, keypointId); + // Only consider parts with score greater or equal to threshold as + // root candidates. + if (score < scoreThreshold) { + continue; + } + // Only consider keypoints whose score is maximum in a local window. + if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, localMaximumRadius, scores)) { + queue.enqueue({ score: score, part: { heatmapY: heatmapY, heatmapX: heatmapX, id: keypointId } }); + } + } + } + } + return queue; +} +exports.buildPartWithScoreQueue = buildPartWithScoreQueue; +//# sourceMappingURL=build_part_with_score_queue.js.map \ No newline at end of file diff --git a/modules/posenet/multi_pose/build_part_with_score_queue.js.map b/modules/posenet/multi_pose/build_part_with_score_queue.js.map new file mode 100644 index 0000000..85f8084 --- /dev/null +++ b/modules/posenet/multi_pose/build_part_with_score_queue.js.map @@ -0,0 +1 @@ +{"version":3,"file":"build_part_with_score_queue.js","sourceRoot":"","sources":["../../src/multi_pose/build_part_with_score_queue.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAIH,uCAAmC;AAEnC,SAAS,2BAA2B,CAChC,UAAkB,EAAE,KAAa,EAAE,QAAgB,EAAE,QAAgB,EACrE,kBAA0B,EAAE,MAAsB;IAC9C,IAAA,iBAA8B,EAA7B,cAAM,EAAE,aAAqB,CAAC;IAErC,IAAI,YAAY,GAAG,IAAI,CAAC;IACxB,IAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,kBAAkB,EAAE,CAAC,CAAC,CAAC;IAC1D,IAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,kBAAkB,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;IACjE,KAAK,IAAI,QAAQ,GAAG,MAAM,EAAE,QAAQ,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE;QACvD,IAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,kBAAkB,EAAE,CAAC,CAAC,CAAC;QAC1D,IAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,kBAAkB,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QAChE,KAAK,IAAI,QAAQ,GAAG,MAAM,EAAE,QAAQ,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE;YACvD,IAAI,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,KAAK,EAAE;gBACtD,YAAY,GAAG,KAAK,CAAC;gBACrB,MAAM;aACP;SACF;QACD,IAAI,CAAC,YAAY,EAAE;YACjB,MAAM;SACP;KACF;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,SAAgB,uBAAuB,CACnC,cAAsB,EAAE,kBAA0B,EAClD,MAAsB;IAClB,IAAA,iBAA4C,EAA3C,cAAM,EAAE,aAAK,EAAE,oBAA4B,CAAC;IAEnD,IAAM,KAAK,GAAG,IAAI,kBAAO,CACrB,MAAM,GAAG,KAAK,GAAG,YAAY,EAAE,UAAC,EAAO;YAAN,gBAAK;QAAM,OAAA,KAAK;IAAL,CAAK,CAAC,CAAC;IAEvD,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE;QACpD,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,KAAK,EAAE,EAAE,QAAQ,EAAE;YACnD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,YAAY,EAAE,EAAE,UAAU,EAAE;gBAChE,IAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAEzD,kEAAkE;gBAClE,mBAAmB;gBACnB,IAAI,KAAK,GAAG,cAAc,EAAE;oBAC1B,SAAS;iBACV;gBAED,oEAAoE;gBACpE,IAAI,2BAA2B,CACvB,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,kBAAkB,EACzD,MAAM,CAAC,EAAE;oBACf,KAAK,CAAC,OAAO,CAAC,EAAC,KAAK,OAAA,EAAE,IAAI,EAAE,EAAC,QAAQ,UAAA,EAAE,QAAQ,UAAA,EAAE,EAAE,EAAE,UAAU,EAAC,EAAC,CAAC,CAAC;iBACpE;aACF;SACF;KACF;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AA9BD,0DA8BC"} \ No newline at end of file diff --git a/modules/posenet/multi_pose/decode_multiple_poses.d.ts b/modules/posenet/multi_pose/decode_multiple_poses.d.ts new file mode 100644 index 0000000..6f54241 --- /dev/null +++ b/modules/posenet/multi_pose/decode_multiple_poses.d.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import { Pose, TensorBuffer3D } from '../types'; +/** + * Detects multiple poses and finds their parts from part scores and + * displacement vectors. It returns up to `maxDetections` object instance + * detections in decreasing root score order. It works as follows: We first + * create a priority queue with local part score maxima above + * `scoreThreshold`, considering all parts at the same time. Then we + * iteratively pull the top element of the queue (in decreasing score order) + * and treat it as a root candidate for a new object instance. To avoid + * duplicate detections, we reject the root candidate if it is within a disk + * of `nmsRadius` pixels from the corresponding part of a previously detected + * instance, which is a form of part-based non-maximum suppression (NMS). If + * the root candidate passes the NMS check, we start a new object instance + * detection, treating the corresponding part as root and finding the + * positions of the remaining parts by following the displacement vectors + * along the tree-structured part graph. We assign to the newly detected + * instance a score equal to the sum of scores of its parts which have not + * been claimed by a previous instance (i.e., those at least `nmsRadius` + * pixels away from the corresponding part of all previously detected + * instances), divided by the total number of parts `numParts`. + * + * @param heatmapScores 3-D tensor with shape `[height, width, numParts]`. + * The value of heatmapScores[y, x, k]` is the score of placing the `k`-th + * object part at position `(y, x)`. + * + * @param offsets 3-D tensor with shape `[height, width, numParts * 2]`. + * The value of [offsets[y, x, k], offsets[y, x, k + numParts]]` is the + * short range offset vector of the `k`-th object part at heatmap + * position `(y, x)`. + * + * @param displacementsFwd 3-D tensor of shape + * `[height, width, 2 * num_edges]`, where `num_edges = num_parts - 1` is the + * number of edges (parent-child pairs) in the tree. It contains the forward + * displacements between consecutive part from the root towards the leaves. + * + * @param displacementsBwd 3-D tensor of shape + * `[height, width, 2 * num_edges]`, where `num_edges = num_parts - 1` is the + * number of edges (parent-child pairs) in the tree. It contains the backward + * displacements between consecutive part from the root towards the leaves. + * + * @param outputStride The output stride that was used when feed-forwarding + * through the PoseNet model. Must be 32, 16, or 8. + * + * @param maxPoseDetections Maximum number of returned instance detections per + * image. + * + * @param scoreThreshold Only return instance detections that have root part + * score greater or equal to this value. Defaults to 0.5. + * + * @param nmsRadius Non-maximum suppression part distance. It needs to be + * strictly positive. Two parts suppress each other if they are less than + * `nmsRadius` pixels away. Defaults to 20. + * + * @return An array of poses and their scores, each containing keypoints and + * the corresponding keypoint scores. + */ +export declare function decodeMultiplePoses(scoresBuffer: TensorBuffer3D, offsetsBuffer: TensorBuffer3D, displacementsFwdBuffer: TensorBuffer3D, displacementsBwdBuffer: TensorBuffer3D, outputStride: number, maxPoseDetections: number, scoreThreshold?: number, nmsRadius?: number): Pose[]; diff --git a/modules/posenet/multi_pose/decode_multiple_poses.js b/modules/posenet/multi_pose/decode_multiple_poses.js new file mode 100644 index 0000000..626f5b3 --- /dev/null +++ b/modules/posenet/multi_pose/decode_multiple_poses.js @@ -0,0 +1,130 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var build_part_with_score_queue_1 = require("./build_part_with_score_queue"); +var decode_pose_1 = require("./decode_pose"); +var util_1 = require("./util"); +function withinNmsRadiusOfCorrespondingPoint(poses, squaredNmsRadius, _a, keypointId) { + var x = _a.x, y = _a.y; + return poses.some(function (_a) { + var keypoints = _a.keypoints; + var correspondingKeypoint = keypoints[keypointId].position; + return util_1.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= + squaredNmsRadius; + }); +} +/* Score the newly proposed object instance without taking into account + * the scores of the parts that overlap with any previously detected + * instance. + */ +function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) { + var notOverlappedKeypointScores = instanceKeypoints.reduce(function (result, _a, keypointId) { + var position = _a.position, score = _a.score; + if (!withinNmsRadiusOfCorrespondingPoint(existingPoses, squaredNmsRadius, position, keypointId)) { + result += score; + } + return result; + }, 0.0); + return notOverlappedKeypointScores /= instanceKeypoints.length; +} +// A point (y, x) is considered as root part candidate if its score is a +// maximum in a window |y - y'| <= kLocalMaximumRadius, |x - x'| <= +// kLocalMaximumRadius. +var kLocalMaximumRadius = 1; +/** + * Detects multiple poses and finds their parts from part scores and + * displacement vectors. It returns up to `maxDetections` object instance + * detections in decreasing root score order. It works as follows: We first + * create a priority queue with local part score maxima above + * `scoreThreshold`, considering all parts at the same time. Then we + * iteratively pull the top element of the queue (in decreasing score order) + * and treat it as a root candidate for a new object instance. To avoid + * duplicate detections, we reject the root candidate if it is within a disk + * of `nmsRadius` pixels from the corresponding part of a previously detected + * instance, which is a form of part-based non-maximum suppression (NMS). If + * the root candidate passes the NMS check, we start a new object instance + * detection, treating the corresponding part as root and finding the + * positions of the remaining parts by following the displacement vectors + * along the tree-structured part graph. We assign to the newly detected + * instance a score equal to the sum of scores of its parts which have not + * been claimed by a previous instance (i.e., those at least `nmsRadius` + * pixels away from the corresponding part of all previously detected + * instances), divided by the total number of parts `numParts`. + * + * @param heatmapScores 3-D tensor with shape `[height, width, numParts]`. + * The value of heatmapScores[y, x, k]` is the score of placing the `k`-th + * object part at position `(y, x)`. + * + * @param offsets 3-D tensor with shape `[height, width, numParts * 2]`. + * The value of [offsets[y, x, k], offsets[y, x, k + numParts]]` is the + * short range offset vector of the `k`-th object part at heatmap + * position `(y, x)`. + * + * @param displacementsFwd 3-D tensor of shape + * `[height, width, 2 * num_edges]`, where `num_edges = num_parts - 1` is the + * number of edges (parent-child pairs) in the tree. It contains the forward + * displacements between consecutive part from the root towards the leaves. + * + * @param displacementsBwd 3-D tensor of shape + * `[height, width, 2 * num_edges]`, where `num_edges = num_parts - 1` is the + * number of edges (parent-child pairs) in the tree. It contains the backward + * displacements between consecutive part from the root towards the leaves. + * + * @param outputStride The output stride that was used when feed-forwarding + * through the PoseNet model. Must be 32, 16, or 8. + * + * @param maxPoseDetections Maximum number of returned instance detections per + * image. + * + * @param scoreThreshold Only return instance detections that have root part + * score greater or equal to this value. Defaults to 0.5. + * + * @param nmsRadius Non-maximum suppression part distance. It needs to be + * strictly positive. Two parts suppress each other if they are less than + * `nmsRadius` pixels away. Defaults to 20. + * + * @return An array of poses and their scores, each containing keypoints and + * the corresponding keypoint scores. + */ +function decodeMultiplePoses(scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer, outputStride, maxPoseDetections, scoreThreshold, nmsRadius) { + if (scoreThreshold === void 0) { scoreThreshold = 0.5; } + if (nmsRadius === void 0) { nmsRadius = 20; } + var poses = []; + var queue = build_part_with_score_queue_1.buildPartWithScoreQueue(scoreThreshold, kLocalMaximumRadius, scoresBuffer); + var squaredNmsRadius = nmsRadius * nmsRadius; + // Generate at most maxDetections object instances per image in + // decreasing root part score order. + while (poses.length < maxPoseDetections && !queue.empty()) { + // The top element in the queue is the next root candidate. + var root = queue.dequeue(); + // Part-based non-maximum suppression: We reject a root candidate if it + // is within a disk of `nmsRadius` pixels from the corresponding part of + // a previously detected instance. + var rootImageCoords = util_1.getImageCoords(root.part, outputStride, offsetsBuffer); + if (withinNmsRadiusOfCorrespondingPoint(poses, squaredNmsRadius, rootImageCoords, root.part.id)) { + continue; + } + // Start a new detection instance at the position of the root. + var keypoints = decode_pose_1.decodePose(root, scoresBuffer, offsetsBuffer, outputStride, displacementsFwdBuffer, displacementsBwdBuffer); + var score = getInstanceScore(poses, squaredNmsRadius, keypoints); + poses.push({ keypoints: keypoints, score: score }); + } + return poses; +} +exports.decodeMultiplePoses = decodeMultiplePoses; +//# sourceMappingURL=decode_multiple_poses.js.map \ No newline at end of file diff --git a/modules/posenet/multi_pose/decode_multiple_poses.js.map b/modules/posenet/multi_pose/decode_multiple_poses.js.map new file mode 100644 index 0000000..ac86ad2 --- /dev/null +++ b/modules/posenet/multi_pose/decode_multiple_poses.js.map @@ -0,0 +1 @@ +{"version":3,"file":"decode_multiple_poses.js","sourceRoot":"","sources":["../../src/multi_pose/decode_multiple_poses.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAIH,6EAAsE;AACtE,6CAAyC;AACzC,+BAAuD;AAEvD,SAAS,mCAAmC,CACxC,KAAa,EAAE,gBAAwB,EAAE,EAA8B,EACvE,UAAkB;QADwB,QAAC,EAAE,QAAC;IAEhD,OAAO,KAAK,CAAC,IAAI,CAAC,UAAC,EAAW;YAAV,wBAAS;QAC3B,IAAM,qBAAqB,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC;QAC7D,OAAO,sBAAe,CACX,CAAC,EAAE,CAAC,EAAE,qBAAqB,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC;YAC9D,gBAAgB,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CACrB,aAAqB,EAAE,gBAAwB,EAC/C,iBAA6B;IAC/B,IAAI,2BAA2B,GAAG,iBAAiB,CAAC,MAAM,CACtD,UAAC,MAAM,EAAE,EAAiB,EAAE,UAAU;YAA5B,sBAAQ,EAAE,gBAAK;QACvB,IAAI,CAAC,mCAAmC,CAChC,aAAa,EAAE,gBAAgB,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE;YAC9D,MAAM,IAAI,KAAK,CAAC;SACjB;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,EAAE,GAAG,CAAC,CAAC;IAEZ,OAAO,2BAA2B,IAAI,iBAAiB,CAAC,MAAM,CAAC;AACjE,CAAC;AAED,wEAAwE;AACxE,mEAAmE;AACnE,uBAAuB;AACvB,IAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,SAAgB,mBAAmB,CAC/B,YAA4B,EAAE,aAA6B,EAC3D,sBAAsC,EACtC,sBAAsC,EAAE,YAAoB,EAC5D,iBAAyB,EAAE,cAAoB,EAAE,SAAc;IAApC,+BAAA,EAAA,oBAAoB;IAAE,0BAAA,EAAA,cAAc;IACjE,IAAM,KAAK,GAAW,EAAE,CAAC;IAEzB,IAAM,KAAK,GAAG,qDAAuB,CACjC,cAAc,EAAE,mBAAmB,EAAE,YAAY,CAAC,CAAC;IAEvD,IAAM,gBAAgB,GAAG,SAAS,GAAG,SAAS,CAAC;IAE/C,+DAA+D;IAC/D,oCAAoC;IACpC,OAAO,KAAK,CAAC,MAAM,GAAG,iBAAiB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE;QACzD,2DAA2D;QAC3D,IAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;QAE7B,uEAAuE;QACvE,wEAAwE;QACxE,kCAAkC;QAClC,IAAM,eAAe,GACjB,qBAAc,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;QAC3D,IAAI,mCAAmC,CAC/B,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YAC/D,SAAS;SACV;QAED,8DAA8D;QAC9D,IAAM,SAAS,GAAG,wBAAU,CACxB,IAAI,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,sBAAsB,EACvE,sBAAsB,CAAC,CAAC;QAE5B,IAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC;QAEnE,KAAK,CAAC,IAAI,CAAC,EAAC,SAAS,WAAA,EAAE,KAAK,OAAA,EAAC,CAAC,CAAC;KAChC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAvCD,kDAuCC"} \ No newline at end of file diff --git a/modules/posenet/multi_pose/decode_pose.d.ts b/modules/posenet/multi_pose/decode_pose.d.ts new file mode 100644 index 0000000..15c4cef --- /dev/null +++ b/modules/posenet/multi_pose/decode_pose.d.ts @@ -0,0 +1,24 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import { Keypoint, PartWithScore, TensorBuffer3D } from '../types'; +/** + * Follows the displacement fields to decode the full pose of the object + * instance given the position of a part that acts as root. + * + * @return An array of decoded keypoints and their scores for a single pose + */ +export declare function decodePose(root: PartWithScore, scores: TensorBuffer3D, offsets: TensorBuffer3D, outputStride: number, displacementsFwd: TensorBuffer3D, displacementsBwd: TensorBuffer3D): Keypoint[]; diff --git a/modules/posenet/multi_pose/decode_pose.js b/modules/posenet/multi_pose/decode_pose.js new file mode 100644 index 0000000..8d96c4c --- /dev/null +++ b/modules/posenet/multi_pose/decode_pose.js @@ -0,0 +1,115 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var keypoints_1 = require("../keypoints"); +var util_1 = require("./util"); +var util_2 = require("./util"); +var parentChildrenTuples = keypoints_1.poseChain.map(function (_a) { + var parentJoinName = _a[0], childJoinName = _a[1]; + return ([keypoints_1.partIds[parentJoinName], keypoints_1.partIds[childJoinName]]); +}); +var parentToChildEdges = parentChildrenTuples.map(function (_a) { + var childJointId = _a[1]; + return childJointId; +}); +var childToParentEdges = parentChildrenTuples.map(function (_a) { + var parentJointId = _a[0]; + return parentJointId; +}); +function getDisplacement(edgeId, point, displacements) { + var numEdges = displacements.shape[2] / 2; + return { + y: displacements.get(point.y, point.x, edgeId), + x: displacements.get(point.y, point.x, numEdges + edgeId) + }; +} +function getStridedIndexNearPoint(point, outputStride, height, width) { + return { + y: util_1.clamp(Math.round(point.y / outputStride), 0, height - 1), + x: util_1.clamp(Math.round(point.x / outputStride), 0, width - 1) + }; +} +/** + * We get a new keypoint along the `edgeId` for the pose instance, assuming + * that the position of the `idSource` part is already known. For this, we + * follow the displacement vector from the source to target part (stored in + * the `i`-t channel of the displacement tensor). The displaced keypoint + * vector is refined using the offset vector by `offsetRefineStep` times. + */ +function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep) { + if (offsetRefineStep === void 0) { offsetRefineStep = 2; } + var _a = scoresBuffer.shape, height = _a[0], width = _a[1]; + // Nearest neighbor interpolation for the source->target displacements. + var sourceKeypointIndices = getStridedIndexNearPoint(sourceKeypoint.position, outputStride, height, width); + var displacement = getDisplacement(edgeId, sourceKeypointIndices, displacements); + var displacedPoint = util_2.addVectors(sourceKeypoint.position, displacement); + var targetKeypoint = displacedPoint; + for (var i = 0; i < offsetRefineStep; i++) { + var targetKeypointIndices = getStridedIndexNearPoint(targetKeypoint, outputStride, height, width); + var offsetPoint = util_1.getOffsetPoint(targetKeypointIndices.y, targetKeypointIndices.x, targetKeypointId, offsets); + targetKeypoint = util_2.addVectors({ + x: targetKeypointIndices.x * outputStride, + y: targetKeypointIndices.y * outputStride + }, { x: offsetPoint.x, y: offsetPoint.y }); + } + var targetKeyPointIndices = getStridedIndexNearPoint(targetKeypoint, outputStride, height, width); + var score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId); + return { position: targetKeypoint, part: keypoints_1.partNames[targetKeypointId], score: score }; +} +/** + * Follows the displacement fields to decode the full pose of the object + * instance given the position of a part that acts as root. + * + * @return An array of decoded keypoints and their scores for a single pose + */ +function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) { + var numParts = scores.shape[2]; + var numEdges = parentToChildEdges.length; + var instanceKeypoints = new Array(numParts); + // Start a new detection instance at the position of the root. + var rootPart = root.part, rootScore = root.score; + var rootPoint = util_2.getImageCoords(rootPart, outputStride, offsets); + instanceKeypoints[rootPart.id] = { + score: rootScore, + part: keypoints_1.partNames[rootPart.id], + position: rootPoint + }; + // Decode the part positions upwards in the tree, following the backward + // displacements. + for (var edge = numEdges - 1; edge >= 0; --edge) { + var sourceKeypointId = parentToChildEdges[edge]; + var targetKeypointId = childToParentEdges[edge]; + if (instanceKeypoints[sourceKeypointId] && + !instanceKeypoints[targetKeypointId]) { + instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd); + } + } + // Decode the part positions downwards in the tree, following the forward + // displacements. + for (var edge = 0; edge < numEdges; ++edge) { + var sourceKeypointId = childToParentEdges[edge]; + var targetKeypointId = parentToChildEdges[edge]; + if (instanceKeypoints[sourceKeypointId] && + !instanceKeypoints[targetKeypointId]) { + instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd); + } + } + return instanceKeypoints; +} +exports.decodePose = decodePose; +//# sourceMappingURL=decode_pose.js.map \ No newline at end of file diff --git a/modules/posenet/multi_pose/decode_pose.js.map b/modules/posenet/multi_pose/decode_pose.js.map new file mode 100644 index 0000000..2738899 --- /dev/null +++ b/modules/posenet/multi_pose/decode_pose.js.map @@ -0,0 +1 @@ +{"version":3,"file":"decode_pose.js","sourceRoot":"","sources":["../../src/multi_pose/decode_pose.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAEH,0CAAwE;AAGxE,+BAA6C;AAC7C,+BAAkD;AAElD,IAAM,oBAAoB,GAAkB,qBAAS,CAAC,GAAG,CACrD,UAAC,EAA+B;QAA9B,sBAAc,EAAE,qBAAa;IAC3B,OAAA,CAAC,CAAC,mBAAO,CAAC,cAAc,CAAC,EAAE,mBAAO,CAAC,aAAa,CAAC,CAAC,CAAC;AAAnD,CAAmD,CAAC,CAAC;AAE7D,IAAM,kBAAkB,GACpB,oBAAoB,CAAC,GAAG,CAAC,UAAC,EAAgB;QAAb,oBAAY;IAAM,OAAA,YAAY;AAAZ,CAAY,CAAC,CAAC;AAEjE,IAAM,kBAAkB,GACpB,oBAAoB,CAAC,GAAG,CAAC,UAAC,EAEA;QADC,qBAAa;IACT,OAAA,aAAa;AAAb,CAAa,CAAC,CAAC;AAElD,SAAS,eAAe,CACpB,MAAc,EAAE,KAAe,EAAE,aAA6B;IAChE,IAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5C,OAAO;QACL,CAAC,EAAE,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;QAC9C,CAAC,EAAE,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC7B,KAAe,EAAE,YAAoB,EAAE,MAAc,EACrD,KAAa;IACf,OAAO;QACL,CAAC,EAAE,YAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,YAAY,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC;QAC3D,CAAC,EAAE,YAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,YAAY,CAAC,EAAE,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;KAC3D,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,wBAAwB,CAC7B,MAAc,EAAE,cAAwB,EAAE,gBAAwB,EAClE,YAA4B,EAAE,OAAuB,EAAE,YAAoB,EAC3E,aAA6B,EAAE,gBAAoB;IAApB,iCAAA,EAAA,oBAAoB;IAC/C,IAAA,uBAAoC,EAAnC,cAAM,EAAE,aAA2B,CAAC;IAE3C,uEAAuE;IACvE,IAAM,qBAAqB,GAAG,wBAAwB,CAClD,cAAc,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAE1D,IAAM,YAAY,GACd,eAAe,CAAC,MAAM,EAAE,qBAAqB,EAAE,aAAa,CAAC,CAAC;IAElE,IAAM,cAAc,GAAG,iBAAU,CAAC,cAAc,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACzE,IAAI,cAAc,GAAG,cAAc,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,EAAE,CAAC,EAAE,EAAE;QACzC,IAAM,qBAAqB,GACvB,wBAAwB,CAAC,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAE1E,IAAM,WAAW,GAAG,qBAAc,CAC9B,qBAAqB,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,EAAE,gBAAgB,EAClE,OAAO,CAAC,CAAC;QAEb,cAAc,GAAG,iBAAU,CACvB;YACE,CAAC,EAAE,qBAAqB,CAAC,CAAC,GAAG,YAAY;YACzC,CAAC,EAAE,qBAAqB,CAAC,CAAC,GAAG,YAAY;SAC1C,EACD,EAAC,CAAC,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC,EAAC,CAAC,CAAC;KAC3C;IACD,IAAM,qBAAqB,GACvB,wBAAwB,CAAC,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC1E,IAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAC1B,qBAAqB,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAExE,OAAO,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,qBAAS,CAAC,gBAAgB,CAAC,EAAE,KAAK,OAAA,EAAC,CAAC;AAC9E,CAAC;AAED;;;;;GAKG;AACH,SAAgB,UAAU,CACtB,IAAmB,EAAE,MAAsB,EAAE,OAAuB,EACpE,YAAoB,EAAE,gBAAgC,EACtD,gBAAgC;IAClC,IAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjC,IAAM,QAAQ,GAAG,kBAAkB,CAAC,MAAM,CAAC;IAE3C,IAAM,iBAAiB,GAAe,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC1D,8DAA8D;IACvD,IAAA,oBAAc,EAAE,sBAAgB,CAAS;IAChD,IAAM,SAAS,GAAG,qBAAc,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAElE,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG;QAC/B,KAAK,EAAE,SAAS;QAChB,IAAI,EAAE,qBAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,QAAQ,EAAE,SAAS;KACpB,CAAC;IAEF,wEAAwE;IACxE,iBAAiB;IACjB,KAAK,IAAI,IAAI,GAAG,QAAQ,GAAG,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE;QAC/C,IAAM,gBAAgB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClD,IAAM,gBAAgB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,iBAAiB,CAAC,gBAAgB,CAAC;YACnC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,EAAE;YACxC,iBAAiB,CAAC,gBAAgB,CAAC,GAAG,wBAAwB,CAC1D,IAAI,EAAE,iBAAiB,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,MAAM,EACnE,OAAO,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC;SAC9C;KACF;IAED,yEAAyE;IACzE,iBAAiB;IACjB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE;QAC1C,IAAM,gBAAgB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClD,IAAM,gBAAgB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,iBAAiB,CAAC,gBAAgB,CAAC;YACnC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,EAAE;YACxC,iBAAiB,CAAC,gBAAgB,CAAC,GAAG,wBAAwB,CAC1D,IAAI,EAAE,iBAAiB,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,MAAM,EACnE,OAAO,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC;SAC9C;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AA7CD,gCA6CC"} \ No newline at end of file diff --git a/modules/posenet/multi_pose/max_heap.d.ts b/modules/posenet/multi_pose/max_heap.d.ts new file mode 100644 index 0000000..78858d7 --- /dev/null +++ b/modules/posenet/multi_pose/max_heap.d.ts @@ -0,0 +1,33 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +export declare class MaxHeap { + private priorityQueue; + private numberOfElements; + private getElementValue; + constructor(maxSize: number, getElementValue: (element: T) => number); + enqueue(x: T): void; + dequeue(): T; + empty(): boolean; + size(): number; + all(): T[]; + max(): T; + private swim; + private sink; + private getValueAt; + private less; + private exchange; +} diff --git a/modules/posenet/multi_pose/max_heap.js b/modules/posenet/multi_pose/max_heap.js new file mode 100644 index 0000000..2471352 --- /dev/null +++ b/modules/posenet/multi_pose/max_heap.js @@ -0,0 +1,86 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +// algorithm based on Coursera Lecture from Algorithms, Part 1: +// https://www.coursera.org/learn/algorithms-part1/lecture/ZjoSM/heapsort +function half(k) { + return Math.floor(k / 2); +} +var MaxHeap = /** @class */ (function () { + function MaxHeap(maxSize, getElementValue) { + this.priorityQueue = new Array(maxSize); + this.numberOfElements = -1; + this.getElementValue = getElementValue; + } + MaxHeap.prototype.enqueue = function (x) { + this.priorityQueue[++this.numberOfElements] = x; + this.swim(this.numberOfElements); + }; + MaxHeap.prototype.dequeue = function () { + var max = this.priorityQueue[0]; + this.exchange(0, this.numberOfElements--); + this.sink(0); + this.priorityQueue[this.numberOfElements + 1] = null; + return max; + }; + MaxHeap.prototype.empty = function () { + return this.numberOfElements === -1; + }; + MaxHeap.prototype.size = function () { + return this.numberOfElements + 1; + }; + MaxHeap.prototype.all = function () { + return this.priorityQueue.slice(0, this.numberOfElements + 1); + }; + MaxHeap.prototype.max = function () { + return this.priorityQueue[0]; + }; + MaxHeap.prototype.swim = function (k) { + while (k > 0 && this.less(half(k), k)) { + this.exchange(k, half(k)); + k = half(k); + } + }; + MaxHeap.prototype.sink = function (k) { + while (2 * k <= this.numberOfElements) { + var j = 2 * k; + if (j < this.numberOfElements && this.less(j, j + 1)) { + j++; + } + if (!this.less(k, j)) { + break; + } + this.exchange(k, j); + k = j; + } + }; + MaxHeap.prototype.getValueAt = function (i) { + return this.getElementValue(this.priorityQueue[i]); + }; + MaxHeap.prototype.less = function (i, j) { + return this.getValueAt(i) < this.getValueAt(j); + }; + MaxHeap.prototype.exchange = function (i, j) { + var t = this.priorityQueue[i]; + this.priorityQueue[i] = this.priorityQueue[j]; + this.priorityQueue[j] = t; + }; + return MaxHeap; +}()); +exports.MaxHeap = MaxHeap; +//# sourceMappingURL=max_heap.js.map \ No newline at end of file diff --git a/modules/posenet/multi_pose/max_heap.js.map b/modules/posenet/multi_pose/max_heap.js.map new file mode 100644 index 0000000..3864615 --- /dev/null +++ b/modules/posenet/multi_pose/max_heap.js.map @@ -0,0 +1 @@ +{"version":3,"file":"max_heap.js","sourceRoot":"","sources":["../../src/multi_pose/max_heap.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAEH,+DAA+D;AAC/D,yEAAyE;AAEzE,SAAS,IAAI,CAAC,CAAS;IACrB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED;IAKE,iBAAY,OAAe,EAAE,eAAuC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAEM,yBAAO,GAAd,UAAe,CAAI;QACjB,IAAI,CAAC,aAAa,CAAC,EAAE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACnC,CAAC;IAEM,yBAAO,GAAd;QACE,IAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;QACrD,OAAO,GAAG,CAAC;IACb,CAAC;IAEM,uBAAK,GAAZ;QACE,OAAO,IAAI,CAAC,gBAAgB,KAAK,CAAC,CAAC,CAAC;IACtC,CAAC;IAEM,sBAAI,GAAX;QACE,OAAO,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;IACnC,CAAC;IAEM,qBAAG,GAAV;QACE,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAChE,CAAC;IAEM,qBAAG,GAAV;QACE,OAAO,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAEO,sBAAI,GAAZ,UAAa,CAAS;QACpB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;YACrC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SACb;IACH,CAAC;IAEO,sBAAI,GAAZ,UAAa,CAAS;QACpB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE;YACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE;gBACpD,CAAC,EAAE,CAAC;aACL;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;gBACpB,MAAM;aACP;YACD,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpB,CAAC,GAAG,CAAC,CAAC;SACP;IACH,CAAC;IAEO,4BAAU,GAAlB,UAAmB,CAAS;QAC1B,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IAEO,sBAAI,GAAZ,UAAa,CAAS,EAAE,CAAS;QAC/B,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAEO,0BAAQ,GAAhB,UAAiB,CAAS,EAAE,CAAS;QACnC,IAAM,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACH,cAAC;AAAD,CAAC,AA1ED,IA0EC;AA1EY,0BAAO"} \ No newline at end of file diff --git a/modules/posenet/multi_pose/util.d.ts b/modules/posenet/multi_pose/util.d.ts new file mode 100644 index 0000000..5b6c661 --- /dev/null +++ b/modules/posenet/multi_pose/util.d.ts @@ -0,0 +1,24 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import { Part, TensorBuffer3D, Vector2D } from '../types'; +export declare function getOffsetPoint(y: number, x: number, keypoint: number, offsets: TensorBuffer3D): Vector2D; +export declare function getImageCoords(part: Part, outputStride: number, offsets: TensorBuffer3D): Vector2D; +export declare function fillArray(element: T, size: number): T[]; +export declare function clamp(a: number, min: number, max: number): number; +export declare function squaredDistance(y1: number, x1: number, y2: number, x2: number): number; +export declare function addVectors(a: Vector2D, b: Vector2D): Vector2D; +export declare function clampVector(a: Vector2D, min: number, max: number): Vector2D; diff --git a/modules/posenet/multi_pose/util.js b/modules/posenet/multi_pose/util.js new file mode 100644 index 0000000..adbd3aa --- /dev/null +++ b/modules/posenet/multi_pose/util.js @@ -0,0 +1,68 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +Object.defineProperty(exports, "__esModule", { value: true }); +var keypoints_1 = require("../keypoints"); +function getOffsetPoint(y, x, keypoint, offsets) { + return { + y: offsets.get(y, x, keypoint), + x: offsets.get(y, x, keypoint + keypoints_1.NUM_KEYPOINTS) + }; +} +exports.getOffsetPoint = getOffsetPoint; +function getImageCoords(part, outputStride, offsets) { + var heatmapY = part.heatmapY, heatmapX = part.heatmapX, keypoint = part.id; + var _a = getOffsetPoint(heatmapY, heatmapX, keypoint, offsets), y = _a.y, x = _a.x; + return { + x: part.heatmapX * outputStride + x, + y: part.heatmapY * outputStride + y + }; +} +exports.getImageCoords = getImageCoords; +function fillArray(element, size) { + var result = new Array(size); + for (var i = 0; i < size; i++) { + result[i] = element; + } + return result; +} +exports.fillArray = fillArray; +function clamp(a, min, max) { + if (a < min) { + return min; + } + if (a > max) { + return max; + } + return a; +} +exports.clamp = clamp; +function squaredDistance(y1, x1, y2, x2) { + var dy = y2 - y1; + var dx = x2 - x1; + return dy * dy + dx * dx; +} +exports.squaredDistance = squaredDistance; +function addVectors(a, b) { + return { x: a.x + b.x, y: a.y + b.y }; +} +exports.addVectors = addVectors; +function clampVector(a, min, max) { + return { y: clamp(a.y, min, max), x: clamp(a.x, min, max) }; +} +exports.clampVector = clampVector; +//# sourceMappingURL=util.js.map \ No newline at end of file diff --git a/modules/posenet/multi_pose/util.js.map b/modules/posenet/multi_pose/util.js.map new file mode 100644 index 0000000..f35088e --- /dev/null +++ b/modules/posenet/multi_pose/util.js.map @@ -0,0 +1 @@ +{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/multi_pose/util.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAEH,0CAA2C;AAG3C,SAAgB,cAAc,CAC1B,CAAS,EAAE,CAAS,EAAE,QAAgB,EAAE,OAAuB;IACjE,OAAO;QACL,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC;QAC9B,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,GAAG,yBAAa,CAAC;KAC/C,CAAC;AACJ,CAAC;AAND,wCAMC;AAED,SAAgB,cAAc,CAC1B,IAAU,EAAE,YAAoB,EAAE,OAAuB;IACpD,IAAA,wBAAQ,EAAE,wBAAQ,EAAE,kBAAY,CAAS;IAC1C,IAAA,0DAA8D,EAA7D,QAAC,EAAE,QAA0D,CAAC;IACrE,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,YAAY,GAAG,CAAC;QACnC,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,YAAY,GAAG,CAAC;KACpC,CAAC;AACJ,CAAC;AARD,wCAQC;AAED,SAAgB,SAAS,CAAI,OAAU,EAAE,IAAY;IACnD,IAAM,MAAM,GAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE;QAC7B,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;KACrB;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AARD,8BAQC;AAED,SAAgB,KAAK,CAAC,CAAS,EAAE,GAAW,EAAE,GAAW;IACvD,IAAI,CAAC,GAAG,GAAG,EAAE;QACX,OAAO,GAAG,CAAC;KACZ;IACD,IAAI,CAAC,GAAG,GAAG,EAAE;QACX,OAAO,GAAG,CAAC;KACZ;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AARD,sBAQC;AAED,SAAgB,eAAe,CAC3B,EAAU,EAAE,EAAU,EAAE,EAAU,EAAE,EAAU;IAChD,IAAM,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IACnB,IAAM,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IACnB,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AAC3B,CAAC;AALD,0CAKC;AAED,SAAgB,UAAU,CAAC,CAAW,EAAE,CAAW;IACjD,OAAO,EAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAC,CAAC;AACtC,CAAC;AAFD,gCAEC;AAED,SAAgB,WAAW,CAAC,CAAW,EAAE,GAAW,EAAE,GAAW;IAC/D,OAAO,EAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAC,CAAC;AAC5D,CAAC;AAFD,kCAEC"} \ No newline at end of file diff --git a/modules/posenet/posenet_model.d.ts b/modules/posenet/posenet_model.d.ts new file mode 100644 index 0000000..6a7b2d4 --- /dev/null +++ b/modules/posenet/posenet_model.d.ts @@ -0,0 +1,167 @@ +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import { BaseModel } from './base_model'; +import { InputResolution, MobileNetMultiplier, Pose, PoseNetArchitecture, PosenetInput, PoseNetOutputStride, PoseNetQuantBytes } from './types'; +/** + * PoseNet model loading is configurable using the following config dictionary. + * + * `architecture`: PoseNetArchitecture. It determines wich PoseNet architecture + * to load. The supported architectures are: MobileNetV1 and ResNet. + * + * `outputStride`: Specifies the output stride of the PoseNet model. + * The smaller the value, the larger the output resolution, and more accurate + * the model at the cost of speed. Set this to a larger value to increase speed + * at the cost of accuracy. Stride 32 is supported for ResNet and + * stride 8,16,32 are supported for various MobileNetV1 models. + * + * * `inputResolution`: A number or an Object of type {width: number, height: + * number}. Specifies the size the input image is scaled to before feeding it + * through the PoseNet model. The larger the value, more accurate the model at + * the cost of speed. Set this to a smaller value to increase speed at the cost + * of accuracy. If a number is provided, the input will be resized and padded to + * be a square with the same width and height. If width and height are + * provided, the input will be resized and padded to the specified width and + * height. + * + * `multiplier`: An optional number with values: 1.01, 1.0, 0.75, or + * 0.50. The value is used only by MobileNet architecture. It is the float + * multiplier for the depth (number of channels) for all convolution ops. + * The larger the value, the larger the size of the layers, and more accurate + * the model at the cost of speed. Set this to a smaller value to increase speed + * at the cost of accuracy. + * + * `modelUrl`: An optional string that specifies custom url of the model. This + * is useful for area/countries that don't have access to the model hosted on + * GCP. + * + * `quantBytes`: An opional number with values: 1, 2, or 4. This parameter + * affects weight quantization in the models. The available options are + * 1 byte, 2 bytes, and 4 bytes. The higher the value, the larger the model size + * and thus the longer the loading time, the lower the value, the shorter the + * loading time but lower the accuracy. + */ +export interface ModelConfig { + architecture: PoseNetArchitecture; + outputStride: PoseNetOutputStride; + inputResolution: InputResolution; + multiplier?: MobileNetMultiplier; + modelUrl?: string; + quantBytes?: PoseNetQuantBytes; +} +/** + * PoseNet inference is configurable using the following config dictionary. + * + * `flipHorizontal`: If the poses should be flipped/mirrored horizontally. + * This should be set to true for videos where the video is by default flipped + * horizontally (i.e. a webcam), and you want the poses to be returned in the + * proper orientation. + * + */ +export interface InferenceConfig { + flipHorizontal: boolean; +} +/** + * Single Person Inference Config + */ +export interface SinglePersonInterfaceConfig extends InferenceConfig { +} +/** + * Multiple Person Inference Config + * + * `maxDetections`: Maximum number of returned instance detections per image. + * + * `scoreThreshold`: Only return instance detections that have root part + * score greater or equal to this value. Defaults to 0.5 + * + * `nmsRadius`: Non-maximum suppression part distance in pixels. It needs + * to be strictly positive. Two parts suppress each other if they are less + * than `nmsRadius` pixels away. Defaults to 20. + */ +export interface MultiPersonInferenceConfig extends InferenceConfig { + maxDetections?: number; + scoreThreshold?: number; + nmsRadius?: number; +} +export interface LegacyMultiPersonInferenceConfig extends MultiPersonInferenceConfig { + decodingMethod: 'multi-person'; +} +export interface LegacySinglePersonInferenceConfig extends SinglePersonInterfaceConfig { + decodingMethod: 'single-person'; +} +export declare const SINGLE_PERSON_INFERENCE_CONFIG: SinglePersonInterfaceConfig; +export declare const MULTI_PERSON_INFERENCE_CONFIG: MultiPersonInferenceConfig; +export declare class PoseNet { + readonly baseModel: BaseModel; + readonly inputResolution: [number, number]; + constructor(net: BaseModel, inputResolution: [number, number]); + /** + * Infer through PoseNet, and estimates multiple poses using the outputs. + * This does standard ImageNet pre-processing before inferring through the + * model. The image should pixels should have values [0-255]. It detects + * multiple poses and finds their parts from part scores and displacement + * vectors using a fast greedy decoding algorithm. It returns up to + * `config.maxDetections` object instance detections in decreasing root + * score order. + * + * @param input + * ImageData|HTMLImageElement|HTMLCanvasElement|HTMLVideoElement) The input + * image to feed through the network. + * + * @param config MultiPoseEstimationConfig object that contains parameters + * for the PoseNet inference using multiple pose estimation. + * + * @return An array of poses and their scores, each containing keypoints and + * the corresponding keypoint scores. The positions of the keypoints are + * in the same scale as the original image + */ + estimateMultiplePoses(input: PosenetInput, config?: MultiPersonInferenceConfig): Promise; + /** + * Infer through PoseNet, and estimates a single pose using the outputs. + * This does standard ImageNet pre-processing before inferring through the + * model. The image should pixels should have values [0-255]. It detects + * multiple poses and finds their parts from part scores and displacement + * vectors using a fast greedy decoding algorithm. It returns a single pose + * + * @param input + * ImageData|HTMLImageElement|HTMLCanvasElement|HTMLVideoElement) The input + * image to feed through the network. + * + * @param config SinglePersonEstimationConfig object that contains + * parameters for the PoseNet inference using single pose estimation. + * + * @return An pose and its scores, containing keypoints and + * the corresponding keypoint scores. The positions of the keypoints are + * in the same scale as the original image + */ + estimateSinglePose(input: PosenetInput, config?: SinglePersonInterfaceConfig): Promise; + /** Deprecated: Use either estimateSinglePose or estimateMultiplePoses */ + estimatePoses(input: PosenetInput, config: LegacySinglePersonInferenceConfig | LegacyMultiPersonInferenceConfig): Promise; + dispose(): void; +} +/** + * Loads the PoseNet model instance from a checkpoint, with the ResNet + * or MobileNet architecture. The model to be loaded is configurable using the + * config dictionary ModelConfig. Please find more details in the + * documentation of the ModelConfig. + * + * @param config ModelConfig dictionary that contains parameters for + * the PoseNet loading process. Please find more details of each parameters + * in the documentation of the ModelConfig interface. The predefined + * `MOBILENET_V1_CONFIG` and `RESNET_CONFIG` can also be used as references + * for defining your customized config. + */ +export declare function load(config?: ModelConfig): Promise; diff --git a/modules/posenet/posenet_model.js b/modules/posenet/posenet_model.js new file mode 100644 index 0000000..fdf8563 --- /dev/null +++ b/modules/posenet/posenet_model.js @@ -0,0 +1,379 @@ +"use strict"; +/** + * @license + * Copyright 2019 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +var __assign = (this && this.__assign) || function () { + __assign = Object.assign || function(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) + t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); +}; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +var tfconv = require("@tensorflow/tfjs-converter"); +var tf = require("@tensorflow/tfjs-core"); +var checkpoints_1 = require("./checkpoints"); +var mobilenet_1 = require("./mobilenet"); +var decode_multiple_poses_1 = require("./multi_pose/decode_multiple_poses"); +var resnet_1 = require("./resnet"); +var decode_single_pose_1 = require("./single_pose/decode_single_pose"); +var util_1 = require("./util"); +// The default configuration for loading MobileNetV1 based PoseNet. +// +// (And for references, the default configuration for loading ResNet +// based PoseNet is also included). +// +// ``` +// const RESNET_CONFIG = { +// architecture: 'ResNet50', +// outputStride: 32, +// quantBytes: 2, +// } as ModelConfig; +// ``` +var MOBILENET_V1_CONFIG = { + architecture: 'MobileNetV1', + outputStride: 16, + multiplier: 0.75, + inputResolution: 257, +}; +var VALID_ARCHITECTURE = ['MobileNetV1', 'ResNet50']; +var VALID_STRIDE = { + 'MobileNetV1': [8, 16, 32], + 'ResNet50': [32, 16] +}; +var VALID_MULTIPLIER = { + 'MobileNetV1': [0.50, 0.75, 1.0], + 'ResNet50': [1.0] +}; +var VALID_QUANT_BYTES = [1, 2, 4]; +function validateModelConfig(config) { + config = config || MOBILENET_V1_CONFIG; + if (config.architecture == null) { + config.architecture = 'MobileNetV1'; + } + if (VALID_ARCHITECTURE.indexOf(config.architecture) < 0) { + throw new Error("Invalid architecture " + config.architecture + ". " + + ("Should be one of " + VALID_ARCHITECTURE)); + } + if (config.inputResolution == null) { + config.inputResolution = 257; + } + util_1.validateInputResolution(config.inputResolution); + if (config.outputStride == null) { + config.outputStride = 16; + } + if (VALID_STRIDE[config.architecture].indexOf(config.outputStride) < 0) { + throw new Error("Invalid outputStride " + config.outputStride + ". " + + ("Should be one of " + VALID_STRIDE[config.architecture] + " ") + + ("for architecture " + config.architecture + ".")); + } + if (config.multiplier == null) { + config.multiplier = 1.0; + } + if (VALID_MULTIPLIER[config.architecture].indexOf(config.multiplier) < 0) { + throw new Error("Invalid multiplier " + config.multiplier + ". " + + ("Should be one of " + VALID_MULTIPLIER[config.architecture] + " ") + + ("for architecture " + config.architecture + ".")); + } + if (config.quantBytes == null) { + config.quantBytes = 4; + } + if (VALID_QUANT_BYTES.indexOf(config.quantBytes) < 0) { + throw new Error("Invalid quantBytes " + config.quantBytes + ". " + + ("Should be one of " + VALID_QUANT_BYTES + " ") + + ("for architecture " + config.architecture + ".")); + } + if (config.architecture === 'MobileNetV1' && config.outputStride === 32 && + config.multiplier !== 1) { + throw new Error("When using an output stride of 32, " + + "you must select 1 as the multiplier."); + } + return config; +} +exports.SINGLE_PERSON_INFERENCE_CONFIG = { + flipHorizontal: false +}; +exports.MULTI_PERSON_INFERENCE_CONFIG = { + flipHorizontal: false, + maxDetections: 5, + scoreThreshold: 0.5, + nmsRadius: 20 +}; +function validateSinglePersonInferenceConfig(config) { } +function validateMultiPersonInputConfig(config) { + var maxDetections = config.maxDetections, scoreThreshold = config.scoreThreshold, nmsRadius = config.nmsRadius; + if (maxDetections <= 0) { + throw new Error("Invalid maxDetections " + maxDetections + ". " + + "Should be > 0"); + } + if (scoreThreshold < 0.0 || scoreThreshold > 1.0) { + throw new Error("Invalid scoreThreshold " + scoreThreshold + ". " + + "Should be in range [0.0, 1.0]"); + } + if (nmsRadius <= 0) { + throw new Error("Invalid nmsRadius " + nmsRadius + "."); + } +} +var PoseNet = /** @class */ (function () { + function PoseNet(net, inputResolution) { + util_1.assertValidOutputStride(net.outputStride); + util_1.assertValidResolution(inputResolution, net.outputStride); + this.baseModel = net; + this.inputResolution = inputResolution; + } + /** + * Infer through PoseNet, and estimates multiple poses using the outputs. + * This does standard ImageNet pre-processing before inferring through the + * model. The image should pixels should have values [0-255]. It detects + * multiple poses and finds their parts from part scores and displacement + * vectors using a fast greedy decoding algorithm. It returns up to + * `config.maxDetections` object instance detections in decreasing root + * score order. + * + * @param input + * ImageData|HTMLImageElement|HTMLCanvasElement|HTMLVideoElement) The input + * image to feed through the network. + * + * @param config MultiPoseEstimationConfig object that contains parameters + * for the PoseNet inference using multiple pose estimation. + * + * @return An array of poses and their scores, each containing keypoints and + * the corresponding keypoint scores. The positions of the keypoints are + * in the same scale as the original image + */ + PoseNet.prototype.estimateMultiplePoses = function (input, config) { + if (config === void 0) { config = exports.MULTI_PERSON_INFERENCE_CONFIG; } + return __awaiter(this, void 0, void 0, function () { + var configWithDefaults, outputStride, inputResolution, _a, height, width, _b, resized, padding, _c, heatmapScores, offsets, displacementFwd, displacementBwd, allTensorBuffers, scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer, poses, resultPoses; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + configWithDefaults = __assign({}, exports.MULTI_PERSON_INFERENCE_CONFIG, config); + validateMultiPersonInputConfig(config); + outputStride = this.baseModel.outputStride; + inputResolution = this.inputResolution; + _a = util_1.getInputTensorDimensions(input), height = _a[0], width = _a[1]; + _b = util_1.padAndResizeTo(input, inputResolution), resized = _b.resized, padding = _b.padding; + _c = this.baseModel.predict(resized), heatmapScores = _c.heatmapScores, offsets = _c.offsets, displacementFwd = _c.displacementFwd, displacementBwd = _c.displacementBwd; + return [4 /*yield*/, util_1.toTensorBuffers3D([heatmapScores, offsets, displacementFwd, displacementBwd])]; + case 1: + allTensorBuffers = _d.sent(); + scoresBuffer = allTensorBuffers[0]; + offsetsBuffer = allTensorBuffers[1]; + displacementsFwdBuffer = allTensorBuffers[2]; + displacementsBwdBuffer = allTensorBuffers[3]; + return [4 /*yield*/, decode_multiple_poses_1.decodeMultiplePoses(scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer, outputStride, configWithDefaults.maxDetections, configWithDefaults.scoreThreshold, configWithDefaults.nmsRadius)]; + case 2: + poses = _d.sent(); + resultPoses = util_1.scaleAndFlipPoses(poses, [height, width], inputResolution, padding, configWithDefaults.flipHorizontal); + heatmapScores.dispose(); + offsets.dispose(); + displacementFwd.dispose(); + displacementBwd.dispose(); + resized.dispose(); + return [2 /*return*/, resultPoses]; + } + }); + }); + }; + /** + * Infer through PoseNet, and estimates a single pose using the outputs. + * This does standard ImageNet pre-processing before inferring through the + * model. The image should pixels should have values [0-255]. It detects + * multiple poses and finds their parts from part scores and displacement + * vectors using a fast greedy decoding algorithm. It returns a single pose + * + * @param input + * ImageData|HTMLImageElement|HTMLCanvasElement|HTMLVideoElement) The input + * image to feed through the network. + * + * @param config SinglePersonEstimationConfig object that contains + * parameters for the PoseNet inference using single pose estimation. + * + * @return An pose and its scores, containing keypoints and + * the corresponding keypoint scores. The positions of the keypoints are + * in the same scale as the original image + */ + PoseNet.prototype.estimateSinglePose = function (input, config) { + if (config === void 0) { config = exports.SINGLE_PERSON_INFERENCE_CONFIG; } + return __awaiter(this, void 0, void 0, function () { + var configWithDefaults, outputStride, inputResolution, _a, height, width, _b, resized, padding, _c, heatmapScores, offsets, displacementFwd, displacementBwd, pose, poses, resultPoses; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + configWithDefaults = __assign({}, exports.SINGLE_PERSON_INFERENCE_CONFIG, config); + validateSinglePersonInferenceConfig(configWithDefaults); + outputStride = this.baseModel.outputStride; + inputResolution = this.inputResolution; + _a = util_1.getInputTensorDimensions(input), height = _a[0], width = _a[1]; + _b = util_1.padAndResizeTo(input, inputResolution), resized = _b.resized, padding = _b.padding; + _c = this.baseModel.predict(resized), heatmapScores = _c.heatmapScores, offsets = _c.offsets, displacementFwd = _c.displacementFwd, displacementBwd = _c.displacementBwd; + return [4 /*yield*/, decode_single_pose_1.decodeSinglePose(heatmapScores, offsets, outputStride)]; + case 1: + pose = _d.sent(); + poses = [pose]; + resultPoses = util_1.scaleAndFlipPoses(poses, [height, width], inputResolution, padding, configWithDefaults.flipHorizontal); + heatmapScores.dispose(); + offsets.dispose(); + displacementFwd.dispose(); + displacementBwd.dispose(); + resized.dispose(); + return [2 /*return*/, resultPoses[0]]; + } + }); + }); + }; + /** Deprecated: Use either estimateSinglePose or estimateMultiplePoses */ + PoseNet.prototype.estimatePoses = function (input, config) { + return __awaiter(this, void 0, void 0, function () { + var pose; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + if (!(config.decodingMethod === 'single-person')) return [3 /*break*/, 2]; + return [4 /*yield*/, this.estimateSinglePose(input, config)]; + case 1: + pose = _a.sent(); + return [2 /*return*/, [pose]]; + case 2: return [2 /*return*/, this.estimateMultiplePoses(input, config)]; + } + }); + }); + }; + PoseNet.prototype.dispose = function () { + this.baseModel.dispose(); + }; + return PoseNet; +}()); +exports.PoseNet = PoseNet; +function loadMobileNet(config) { + return __awaiter(this, void 0, void 0, function () { + var outputStride, quantBytes, multiplier, url, graphModel, mobilenet, validInputResolution; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + outputStride = config.outputStride; + quantBytes = config.quantBytes; + multiplier = config.multiplier; + if (tf == null) { + throw new Error("Cannot find TensorFlow.js. If you are using a