diff --git a/landmarks.js b/landmarks.js index cbbb3e9..0c6ca21 100644 --- a/landmarks.js +++ b/landmarks.js @@ -1,218 +1,289 @@ -const fs = require('fs'); -const path = require('path'); -const ArgumentParser = require('argparse'); -const cv = require('opencv'); -const yaml = require('js-yaml'); -const { Pose, POSE_LANDMARKS } = require('@mediapipe/pose'); +const fs = require("fs"); +const path = require("path"); +const { ArgumentParser } = require("argparse"); +const cv = require("@techstark/opencv-js"); +const yaml = require("js-yaml"); +const { Pose, POSE_LANDMARKS } = require("@mediapipe/pose"); const logging = console; const warnings = console; class Landmarker { - static resizedHeight = 256; - static resizedWidth = 256; + static resizedHeight = 256; + static resizedWidth = 256; - constructor() { - this.args = this.parseArgs(); - this.measurements = this.loadLandmarks(); - if (!this.args.frontImage) { - throw new Error("Front image needs to be passed"); - } - if (!this.args.sideImage) { - throw new Error("Side image needs to be passed"); - } - - this.frontImage = cv.imread(this.args.frontImage); - this.sideImage = cv.imread(this.args.sideImage); - - this.frontImageResized = this.frontImage.resize(Landmarker.resizedHeight, Landmarker.resizedWidth); - this.sideImageResized = this.sideImage.resize(Landmarker.resizedHeight, Landmarker.resizedWidth); - - this.distances = {}; - - this.personHeight = this.args.personHeight; - this.pixelHeight = this.args.pixelHeight; - - this.pose = new Pose({ - locateFile: (file) => { - return `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`; - } - }); - - this.landmarksIndices = [ - POSE_LANDMARKS.LEFT_SHOULDER, - POSE_LANDMARKS.RIGHT_SHOULDER, - POSE_LANDMARKS.LEFT_ELBOW, - POSE_LANDMARKS.RIGHT_ELBOW, - POSE_LANDMARKS.LEFT_WRIST, - POSE_LANDMARKS.RIGHT_WRIST, - POSE_LANDMARKS.LEFT_HIP, - POSE_LANDMARKS.RIGHT_HIP, - POSE_LANDMARKS.LEFT_KNEE, - POSE_LANDMARKS.RIGHT_KNEE, - POSE_LANDMARKS.LEFT_ANKLE, - POSE_LANDMARKS.RIGHT_ANKLE - ]; + constructor() { + this.args = this.parseArgs(); + this.measurements = this.loadLandmarks(); + if (!this.args.frontImage) { + throw new Error("Front image needs to be passed"); + } + if (!this.args.sideImage) { + throw new Error("Side image needs to be passed"); } - loadLandmarks() { - const file = fs.readFileSync(this.args.yamlFile, 'utf8'); - const landmarksData = yaml.load(file); - const measurements = {}; - for (const measurement of landmarksData.measurements) { - measurements[measurement.name] = measurement.landmarks; - } - return measurements; + this.frontImage = cv.imread(this.args.frontImage); + this.sideImage = cv.imread(this.args.sideImage); + + this.frontImageResized = cv.resize( + this.frontImage, + new cv.Size(Landmarker.resizedWidth, Landmarker.resizedHeight), + ); + this.sideImageResized = cv.resize( + this.sideImage, + new cv.Size(Landmarker.resizedWidth, Landmarker.resizedHeight), + ); + + this.distances = {}; + + this.personHeight = this.args.personHeight; + this.pixelHeight = this.args.pixelHeight; + + this.pose = new Pose({ + locateFile: (file) => { + return `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`; + }, + }); + + this.landmarksIndices = [ + POSE_LANDMARKS.LEFT_SHOULDER, + POSE_LANDMARKS.RIGHT_SHOULDER, + POSE_LANDMARKS.LEFT_ELBOW, + POSE_LANDMARKS.RIGHT_ELBOW, + POSE_LANDMARKS.LEFT_WRIST, + POSE_LANDMARKS.RIGHT_WRIST, + POSE_LANDMARKS.LEFT_HIP, + POSE_LANDMARKS.RIGHT_HIP, + POSE_LANDMARKS.LEFT_KNEE, + POSE_LANDMARKS.RIGHT_KNEE, + POSE_LANDMARKS.LEFT_ANKLE, + POSE_LANDMARKS.RIGHT_ANKLE, + ]; + } + + loadLandmarks() { + const file = fs.readFileSync(this.args.yamlFile, "utf8"); + const landmarksData = yaml.load(file); + const measurements = {}; + for (const measurement of landmarksData.measurements) { + measurements[measurement.name] = measurement.landmarks; } + return measurements; + } - parseArgs() { - const parser = new ArgumentParser(); - parser.add_argument('--front', { dest: 'frontImage', type: 'str', help: 'Front image' }); - parser.add_argument('--side', { dest: 'sideImage', type: 'str', help: 'Side image' }); - parser.add_argument('--poseDetectionConfidence', { dest: 'poseDetectionConfidence', default: 0.5, type: 'float', help: 'Confidence score for pose detection' }); - parser.add_argument('--poseTrackingConfidence', { dest: 'poseTrackingConfidence', default: 0.5, type: 'float', help: 'Confidence score for pose tracking' }); - parser.add_argument('--personHeight', { dest: 'personHeight', type: 'int', help: 'Person height in cm' }); - parser.add_argument('--pixelHeight', { dest: 'pixelHeight', type: 'int', help: 'Pixel height of person' }); - parser.add_argument('--measurement', { dest: 'measurement', nargs: '+', type: 'str', help: 'Type of measurement' }); - parser.add_argument('--yamlFile', { dest: 'yamlFile', type: 'str', help: 'Path to the YAML file containing landmarks' }); - return parser.parse_args(); - } + parseArgs() { + const parser = new ArgumentParser({ + description: "Process images and calculate measurements", + }); + parser.add_argument("--front", { + dest: "frontImage", + required: true, + help: "Path to the front image", + }); + parser.add_argument("--side", { + dest: "sideImage", + required: true, + help: "Path to the side image", + }); + parser.add_argument("--poseDetectionConfidence", { + dest: "poseDetectionConfidence", + default: 0.5, + type: "float", + help: "Confidence score for pose detection", + }); + parser.add_argument("--poseTrackingConfidence", { + dest: "poseTrackingConfidence", + default: 0.5, + type: "float", + help: "Confidence score for pose tracking", + }); + parser.add_argument("--personHeight", { + dest: "personHeight", + required: true, + type: "int", + help: "Person height in cm", + }); + parser.add_argument("--pixelHeight", { + dest: "pixelHeight", + type: "int", + help: "Pixel height of person", + }); + parser.add_argument("--measurement", { + dest: "measurement", + nargs: "+", + type: "str", + help: "Type of measurement", + }); + parser.add_argument("--yamlFile", { + dest: "yamlFile", + required: true, + help: "Path to the YAML file containing landmarks", + }); + return parser.parse_args(); + } - async run() { - await this.pose.initialize(); - const { frontResults, sideResults } = await this.processImages(); + async run() { + await this.pose.initialize(); + const { frontResults, sideResults } = await this.processImages(); - this.getCenterTopPoint(sideResults); + this.getCenterTopPoint(sideResults); - const table = []; - if (this.args.measurement) { - for (const m of this.args.measurement) { - if (!this.measurements[m]) { - throw new Error("Incorrect input (input not present in config.yml)"); - } else { - const distance = this.calculateDistanceBetweenLandmarks(frontResults, m); - table.push([m, distance]); - } - } + const table = []; + if (this.args.measurement) { + for (const m of this.args.measurement) { + if (!this.measurements[m]) { + throw new Error("Incorrect input (input not present in config.yml)"); } else { - for (const m in this.measurements) { - const distance = this.calculateDistanceBetweenLandmarks(frontResults, m); - table.push([m, distance]); - } + const distance = this.calculateDistanceBetweenLandmarks( + frontResults, + m, + ); + table.push([m, distance]); } - - console.table(table); - - this.pose.close(); + } + } else { + for (const m in this.measurements) { + const distance = this.calculateDistanceBetweenLandmarks( + frontResults, + m, + ); + table.push([m, distance]); + } } - async processImages() { - const frontResults = await this.pose.estimatePoses(this.frontImageResized); - const sideResults = await this.pose.estimatePoses(this.sideImageResized); + console.table(table); - this.sideImageKeypoints = this.sideImageResized.copy(); - this.frontImageKeypoints = this.frontImageResized.copy(); + this.pose.close(); + } - if (frontResults[0].landmarks) { - this.drawLandmarks(this.frontImageKeypoints, frontResults[0].landmarks, this.landmarksIndices); - } - if (sideResults[0].landmarks) { - this.drawLandmarks(this.sideImageKeypoints, sideResults[0].landmarks, this.landmarksIndices); - } - return { - frontResults: frontResults[0], - sideResults: sideResults[0] - }; + async processImages() { + const frontResults = await this.pose.estimatePoses(this.frontImageResized); + const sideResults = await this.pose.estimatePoses(this.sideImageResized); + + this.sideImageKeypoints = this.sideImageResized.clone(); + this.frontImageKeypoints = this.frontImageResized.clone(); + + if (frontResults[0].landmarks) { + this.drawLandmarks( + this.frontImageKeypoints, + frontResults[0].landmarks, + this.landmarksIndices, + ); + } + if (sideResults[0].landmarks) { + this.drawLandmarks( + this.sideImageKeypoints, + sideResults[0].landmarks, + this.landmarksIndices, + ); + } + return { + frontResults: frontResults[0], + sideResults: sideResults[0], + }; + } + + pixelToMetricRatio() { + const pixelToMetricRatio = this.personHeight / this.pixelHeight; + logging.debug("pixelToMetricRatio %s", pixelToMetricRatio); + return pixelToMetricRatio; + } + + drawLandmarks(image, landmarks, indices) { + for (const idx of indices) { + const landmark = landmarks[idx]; + const h = image.rows; + const w = image.cols; + const cx = Math.round(landmark.x * w); + const cy = Math.round(landmark.y * h); + this.circle(image, cx, cy); + } + } + + circle(image, cx, cy) { + cv.circle(image, new cv.Point(cx, cy), 2, new cv.Scalar(255, 0, 0), -1); + } + + calculateDistanceBetweenLandmarks(frontResults, measurementName) { + if (!frontResults.landmarks) { + return; } - pixelToMetricRatio() { - this.pixelHeight = this.pixelDistance * 2; - const pixelToMetricRatio = this.personHeight / this.pixelHeight; - logging.debug("pixelToMetricRatio %s", pixelToMetricRatio); - return pixelToMetricRatio; + const landmarks = frontResults.landmarks; + const landmarkNames = this.measurements[measurementName]; + + let totalDistance = 0; + for (let i = 0; i < landmarkNames.length - 1; i++) { + const current = landmarks[landmarkNames[i]]; + const next = landmarks[landmarkNames[i + 1]]; + const pixelDistance = this.euclideanDistance( + current.x * Landmarker.resizedWidth, + current.y * Landmarker.resizedHeight, + next.x * Landmarker.resizedWidth, + next.y * Landmarker.resizedHeight, + ); + const realDistance = pixelDistance * this.pixelToMetricRatio(); + totalDistance += realDistance; + } + return totalDistance; + } + + euclideanDistance(x1, y1, x2, y2) { + return Math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2); + } + + getCenterTopPoint(sideResults) { + const grayImage = cv.cvtColor(this.sideImageKeypoints, cv.COLOR_BGR2GRAY); + const blurredImage = cv.GaussianBlur(grayImage, new cv.Size(5, 5), 0); + const roi = blurredImage.roi( + new cv.Rect( + 0, + 0, + this.sideImageResized.cols, + Math.floor(this.sideImageResized.rows / 2), + ), + ); + this.edges = cv.Canny(roi, 50, 150); + const contours = this.edges.findContours( + cv.RETR_EXTERNAL, + cv.CHAIN_APPROX_SIMPLE, + ); + + let xt, yt; + this.topmostPoint = null; + + for (const contour of contours) { + const [xt, yt] = contour.minEnclosingCircle(); + if (this.topmostPoint === null || yt < this.topmostPoint[1]) { + this.topmostPoint = [xt, yt]; + } } - drawLandmarks(image, landmarks, indices) { - for (const idx of indices) { - const landmark = landmarks[idx]; - const h = image.rows; - const w = image.cols; - const cx = Math.round(landmark.x * w); - const cy = Math.round(landmark.y * h); - this.circle(image, cx, cy); - } - } + const { x, y } = sideResults.landmarks[POSE_LANDMARKS.NOSE]; + const centerPoint = [ + x * Landmarker.resizedWidth, + y * Landmarker.resizedHeight, + ]; + this.pixelHeight = Math.abs(centerPoint[1] - this.topmostPoint[1]); - circle(image, cx, cy) { - return image.drawCircle(new cv.Point(cx, cy), 2, new cv.Vec(255, 0, 0), -1); - } - - calculateDistanceBetweenLandmarks(frontResults, measurementName) { - if (!frontResults.landmarks) { - return; - } - - const landmarks = frontResults.landmarks; - const landmarkNames = this.measurements[measurementName]; - - let totalDistance = 0; - for (let i = 0; i < landmarkNames.length - 1; i++) { - const current = landmarks[landmarkNames[i]]; - const next = landmarks[landmarkNames[i + 1]]; - const pixelDistance = this.euclideanDistance( - current.x * Landmarker.resizedWidth, - current.y * Landmarker.resizedHeight, - next.x * Landmarker.resizedWidth, - next.y * Landmarker.resizedHeight - ); - const realDistance = pixelDistance * this.pixelToMetricRatio(); - totalDistance += realDistance; - } - return totalDistance; - } - - euclideanDistance(x1, y1, x2, y2) { - return Math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2); - } - - getCenterTopPoint(sideResults) { - const grayImage = this.sideImageKeypoints.cvtColor(cv.COLOR_BGR2GRAY); - const blurredImage = grayImage.gaussianBlur(new cv.Size(5, 5), 0); - const roi = blurredImage.getRegion(new cv.Rect(0, 0, this.sideImageResized.cols, Math.floor(this.sideImageResized.rows / 2))); - this.edges = roi.canny(50, 150); - const contours = this.edges.findContours(cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE); - - let xt, yt; - this.topmostPoint = null; - - if (contours.length > 0) { - const largestContour = contours.sort((a, b) => b.area - a.area)[0]; - this.topmostPoint = largestContour.minBy(pt => pt.y); - xt = this.topmostPoint.x; - yt = this.topmostPoint.y; - - this.circle(this.sideImageKeypoints, xt, yt); - } - - let xc, yc; - if (sideResults.landmarks) { - const leftHip = sideResults.landmarks[POSE_LANDMARKS.LEFT_HIP]; - const rightHip = sideResults.landmarks[POSE_LANDMARKS.RIGHT_HIP]; - const centerPoint = [ - (leftHip.x + rightHip.x) / 2, - (leftHip.y + rightHip.y) / 2 - ]; - xc = Math.round(centerPoint[0] * this.sideImageResized.cols); - yc = Math.round(centerPoint[1] * this.sideImageResized.rows); - this.circle(this.sideImageKeypoints, xc, yc); - } - - this.pixelDistance = this.euclideanDistance(xt, yt, xc, yc); - this.pixelDistance *= 2; - } + cv.circle( + this.sideImageKeypoints, + new cv.Point(centerPoint[0], centerPoint[1]), + 2, + new cv.Scalar(255, 0, 0), + -1, + ); + cv.circle( + this.sideImageKeypoints, + new cv.Point(this.topmostPoint[0], this.topmostPoint[1]), + 2, + new cv.Scalar(255, 0, 0), + -1, + ); + } } -(async () => { - const landmarker = new Landmarker(); - await landmarker.run(); -})(); +const landmarker = new Landmarker(); +landmarker.run().catch((error) => { + console.error(error); +}); diff --git a/landmarks.py b/landmarks.py index 66137a5..893face 100644 --- a/landmarks.py +++ b/landmarks.py @@ -189,6 +189,7 @@ class Landmarker: tablefmt="plain", ) print(output) + self.pose.close() def process_images(self): @@ -279,6 +280,7 @@ class Landmarker: distance = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2) return distance + def get_center_top_point(self, side_results): gray_image = cv2.cvtColor( self.side_image_keypoints,