Use MoveNet with Tensorflow.js and tfjs layers model

TensorFlow
, ,
1

Hi everybody. The problem would be that I want to do a human pose estimation application that uses Movenet (tfjs-models/pose-detection/src/movenet at master · tensorflow/tfjs-models · GitHub), Tensorflow.js, and a trained model (tfjs layers model). I created the model (https://teachablemachine.withgoogle.com/), did the movenet detector and its detector config, loaded the model, but the model predict doesn’t work. If anyone could help, I would be grateful.

my code:

let detector;
    let poses;
    let video;

    async function init() {
		const detectorConfig = {
			modelType: poseDetection.movenet.modelType.SINGLEPOSE_LIGHTNING,
			enableSmoothing: true,
			multiPoseMaxDimension: 256,
			enableTracking: true,
			trackerType: poseDetection.TrackerType.BoundingBox
		};
		detector = await poseDetection.createDetector(
			poseDetection.SupportedModels.MoveNet,
			detectorConfig
		);

    }

    async function videoReady() {
      console.log("video ready");
      await getPoses();
      pc = new RTCPeerConnection().addTrack(video.elt.srcObject.getVideoTracks()[0]);
    }

    async function setup() {
      createCanvas(640, 480);    
      video = createCapture(VIDEO, videoReady);
      console.log(video.height);
      video.hide();
      await init();
    }

    async function getPoses() {
		
		const modelUrl = 'model/model.json';
		const model = await tf.loadLayersModel(modelUrl);
		model.summary();
		const { pose, posenetOutput } = await detector.estimatePoses(video.elt);

		const imageFromWebcam = tf.tidy(() => {
		var img = tf.browser.fromPixels(video.elt);
		
		var smallimg = tf.image.resizeBilinear(img, [224, 224]);
		var resized = tf.cast(smallimg, 'float32');
		img = tf.reshape(resized, ([-1,224,224,3]));

		return img;
		})	  
	  
	const prediction = await model.predict(imageFromWebcam);

    }

    function draw() {
      background(220);
      image(video, 0, 0);
      if (poses && poses.length > 0) {
        for (let kp of poses[0].keypoints) {
          const { x, y, score } = kp;
          if (score > 0.5) {
            fill(255);
            stroke(0);
            strokeWeight(4);
            circle(x, y, 16);                                   
          }
        }
      }
    }

I’m getting this error:

Uncaught (in promise) Error: Error when checking : expected input_1 to have 2 dimension(s), but got array with shape [1,224,224,3]

I tried a lot of things but was unsuccessful

3

@Jason might be able to help here

4

I am a little confused. TeachableMachine uses PoseNet - entirely different model yet you state you are using MoveNet with TM - something seems wrong here. Either you make your own DNN attached to outputs you have collected from MoveNet and make your own version of teachable machine that uses MoveNet as the base model, or you use the teachable machine code from end to end for which they give example code after you have trained a model to use instantly.

You may want to take my course here that goes into these things in more detail if you are new here:

https://www.edx.org/course/google-ai-for-javascript-developers-with-tensorflowjs

And shows how to create your very own Teachable Machine from a blank canvas.