Hi the community,
I trained a Keras model in python and converted it to TFJS layers format in order to make inferences in one web page I hosted onto Github pages. But even if my code looks good it does not work. It looks like all the code is paused or even blocked cause of this single line: import * as tf from ‘@tensorflow/tfjs’; (So probably the most important line in order to use my model 
). I don’t find anything relating incompatibility issue between Tensorflow.js and touch event listeners but that what I was assuming in a first time. It’s quite worrying because the lines of my code that use tfjs are nearly identical to the ones in the official tutorial
import * as tf from '@tensorflow/tfjs';
class TrackingSession {
/*
This TrackingSession offers us a general way to collect data from the screen through the app
The most important method there is the *handle* method through which we collect data like position and timestamp
*/
// To track the active touch
activeTouch = {}
// To store all the records
records = []
// Our app are not going to use the whole screen. This will set the app screen dimensions
screenScale = window.devicePixelRatio
screenSize = [
screen.width,
screen.height
]
// The handle method
handle(event, touches) {
if (touches.length !== 1) {
// Ignore if there are multiple touches on the screen
return
}
const touch = touches.item(0)
switch (event) {
case "start":
const id = Math.floor(1e8 * Math.random()) + ""
this.activeTouch[touch.identifier] = id
this.records.push(new TouchRecord(event, touch, id))
break
case "move":
if (this.activeTouch[touch.identifier]) {
this.records.push(new TouchRecord(event, touch, this.activeTouch[touch.identifier]))
}
break
case "end":
if (this.activeTouch[touch.identifier]) {
this.records.push(new TouchRecord(event, touch, this.activeTouch[touch.identifier]))
delete this.activeTouch[touch.identifier]
this.export()
this.activeTouch = {}
this.records = []
}
break
}
}
// Recognition
async recognizeUser() {
const model = await tf.loadLayersModel('https://diarray-hub.github.io/TouchPatternRecognition/Models/tfjs_model/model.json');
const data = preprocess(this.touchTracks);
const outcome = await model.predict([tf.tensor(data)]);
if (outcome[0][0] >= 0.90) {
// Redirect the user to another page
window.location.href = "https://diarray-hub.github.io/TouchPatternRecognition/theapp/Welcome.html";
}
else {
window.location.href = "https://diarray-hub.github.io/TouchPatternRecognition/theapp/Error.html";
}
}
// This method will use the *download* function defined below to export data in .json file format
export() {
const name = "TouchTracker_Export";
const touchTrackings = {};
let currentTouchId;
let currentTouchTimestamp;
let currentPosition;
let lastPosition;
let currentSpeed;
let currentDirection;
// Process the touch records to create touch tracking objects
this.records.forEach(record => {
if (record.event === "start") {
currentTouchId = record.touchId;
currentTouchTimestamp = record.timestamp;
currentPosition = record.position;
touchTrackings[currentTouchId] = {id: currentTouchId, positions: [currentPosition],
speeds: [], directions: [], startTimestamp: currentTouchTimestamp};
} else if (record.event === "move") {
lastPosition = currentPosition;
currentPosition = record.position;
currentSpeed = calculateSpeed(currentPosition, lastPosition, record.timestamp, currentTouchTimestamp);
currentTouchTimestamp = record.timestamp;
currentDirection = calculateDirection(currentPosition, lastPosition);
touchTrackings[currentTouchId].positions.push(currentPosition);
touchTrackings[currentTouchId].speeds.push(currentSpeed);
touchTrackings[currentTouchId].directions.push(currentDirection);
} else if (record.event === "end") {
touchTrackings[currentTouchId].endTimestamp = record.timestamp;
}
});
// Create an array of touch tracking objects
const touchTrackingsArray = Object.values(touchTrackings);
// Generate the output object
const output = {
name: name,
duration: touchTrackingsArray[0].endTimestamp - touchTrackingsArray[0].starTimestamp,
touchTrackings: touchTrackingsArray,
screenSize: this.screenSize,
screenScale: this.screenScale
};
//this.recognizeUser()
download(JSON.stringify(output, null, 2), name + " " + new Date().toLocaleString(), "application/json");
function calculateSpeed(currentPosition, lastPosition, timestamp, lastimestamp) {
const distance = Math.sqrt((currentPosition[0] - lastPosition[0]) ** 2 + (currentPosition[1] - lastPosition[1]) ** 2);
const timeElapsed = timestamp - lastimestamp;
return distance / timeElapsed; // Eucludian speed calculus
}
function calculateDirection(currentPosition, lastPosition) {
/*
Note that the angle returned by Math.atan2 is not the same as the direction in degrees (i.e. north, south, east, west). Instead,
it represents the angle between the two points in the coordinate system, with the positive x-axis as the reference.
*/
const deltaX = currentPosition[0] - lastPosition[0];
const deltaY = currentPosition[1] - lastPosition[1];
return Math.atan2(deltaY, deltaX);
}
}
}
class TouchRecord {
/*
A TouchRecord class that we'll use as represention of the collected data.
This class' structure represent how data will look like in the .json file
*/
touchId
event
position
timestamp
constructor(event, touch, id) {
this.touchId = id
this.event = event
const topOffset = screen.height - window.innerHeight
this.position = [
touch.screenX,
touch.screenY + topOffset
]
this.timestamp = new Date().getTime() / 1000
}
}
// Implement a part of preprocessing.py stats_summary function in JS
function preprocess(touchTrackingArray){
var touch = touchTrackingArray[0],
positionX = [],
positionY = [],
speeds = touch.speeds,
directions = touch.directions,
latency = touch.endTimestamp - touch.startTimestamp
touch.positions.forEach(position => {
positionX.push(position[0])
positionY.push(position[1])
});
fields = [positionX, positionY, speeds, directions];
// Calculate the features
const features = fields.map(field => {
const mean = field.reduce((a, b) => a + b) / field.length;
const stdDev = Math.sqrt(field.map(x => Math.pow(x - mean, 2)).reduce((a, b) => a + b) / field.length);
const min = Math.min(...field);
const max = Math.max(...field);
const range = max - min;
return [mean, stdDev, min, max, range];
});
// Flatten the features into a single list
const flattenedFeatures = features.reduce((acc, val) => acc.concat(val), []);
flattenedFeatures.push(latency);
return flattenedFeatures
}
// Defining a function that will allows us to export collected data in .json file from the browser
function download(data, filename, type) {
var file = new Blob([data], {type: type});
if (window.navigator.msSaveOrOpenBlob) // IE10+
window.navigator.msSaveOrOpenBlob(file, filename);
else { // Others
var a = document.createElement("a"),
url = URL.createObjectURL(file);
a.href = url;
a.download = filename;
document.body.appendChild(a);
a.click();
setTimeout(function() {
document.body.removeChild(a);
window.URL.revokeObjectURL(url);
}, 0);
}
}
// Creating a instance of our Trackingsession class
const session = new TrackingSession()
document.body.addEventListener('touchstart', function(e){
e.preventDefault()
session.handle("start", e.touches)
});
document.body.addEventListener('touchmove', function(e){
e.preventDefault()
session.handle("move", e.changedTouches)
}, { passive: false });
document.body.addEventListener('touchend', function(e){
e.preventDefault()
console.log(e.changedTouches)
session.handle("end", e.changedTouches)
});
document.body.addEventListener('touchcancel', function(e){
e.preventDefault()
session.handle("end", e.changedTouches)
});
Trying to debug the code I wrote it with script tags in the html file and I noticed that there the problem come from the line: const model = await tf.loadLayersModel(‘https://diarray-hub.github.io/TouchPatternRecognition/Models/tfjs_model/model.json’); that block the code again and even block execution of lines above it but this allows me to know that in this case the import success and I could even use tf.tensor() but when I add the above line the code simply don’t run like when I import tfjs in the script 
.
<!doctype html>
<html lang="en">
<head>
<title>Test</title>
</head>
<body>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4.7.0/dist/tf.min.js"></script>
<h1>Debugging</h1>
<script>
data = preprocess(touchTracks);
document.write(data);
document.write("\n")
data = tf.tensor(data)
document.write(data)
document.write("\n")
const model = tf.loadLayersModel('../../Models/tfjs_model/model.json');
document.write(model.state)
if (model) {
document.write("yo!\n");
result = model.predict([data]);
if(result){
result.array().then(function(scores) {
document.write(scores);
});
}
else{
document.write("Something went wrong!");
}
}
else {
document.write("Oh no!");
}
</script>
</body>
</html>
I need help guys!
