Pfthb

I'm currently attempting to figure out how to convert a input png into a tensor with tensorflow.js so I can feed it into my model for training. Currently I'm capturing the image, saving it locally, reading it with fs.readFileSync, and then creating a buffer. Where i'm a bit lost is normalizing the buffer values from 0-244 to 0-1, then creating a tensor from this buffer to feed into the model.fit function as the X arg. I also don't really know how to set up my labels file and properly convert that into a buffer for the Y arg. (https://js.tensorflow.org/api/0.11.2/#tf.Model.fit) Any insight into the proper usage / configuration of images into tensors for using tensorflow.js would be greatly appreciated.

Repo is here;
https://github.com/Durban-Designer/Fighter-Ai

code for loading local image in data.js;

const tf = require('@tensorflow/tfjs');
const assert = require('assert');
const IMAGE_HEADER_BYTES = 32;
const IMAGE_HEIGHT = 600;
const IMAGE_WIDTH = 800;
const IMAGE_FLAT_SIZE = IMAGE_HEIGHT * IMAGE_WIDTH;

function loadHeaderValues(buffer, headerLength) 
 const headerValues = ;
 for (let i = 0; i < headerLength / 4; i++) 
 headerValues[i] = buffer.readUInt32BE(i * 4);
 
 return headerValues;


...
...
class Dataset 
 async loadLocalImage(filename) 
 const buffer = fs.readFileSync(filename);

 const headerBytes = IMAGE_HEADER_BYTES;
 const recordBytes = IMAGE_HEIGHT * IMAGE_WIDTH;

 const headerValues = loadHeaderValues(buffer, headerBytes);
 console.log(headerValues, buffer);
 assert.equal(headerValues[5], IMAGE_HEIGHT);
 assert.equal(headerValues[4], IMAGE_WIDTH);

 const images = ;
 let index = headerBytes;
 while (index < buffer.byteLength) 
 const array = new Float32Array(recordBytes);
 for (let i = 0; i < recordBytes; i++) 
 // Normalize the pixel values into the 0-1 interval, from
 // the original 0-255 interval.
 array[i] = buffer.readUInt8(index++) / 255;
 
 images.push(array);
 

 assert.equal(images.length, headerValues[1]);
 return images;
 

module.exports = new Dataset();

image capture loop in app.js;

const ioHook = require("iohook");
const tf = require('@tensorflow/tfjs');
var screenCap = require('desktop-screenshot');
require('@tensorflow/tfjs-node');
const data = require('./src/data');
const virtKeys = require('./src/virtKeys');
const model = require('./src/model');
var dir = __dirname;
var paused = true;
var loopInterval,
 image,
 imageData,
 result

ioHook.on('keyup', event => 
 if (event.keycode === 88) 
 if (paused) 
 paused = false;
 gameLoop();
 else 
 paused = true;
 
 
);

ioHook.start();
function gameLoop () 
 if (!paused) 
 screenCap(dir + '\image.png', width: 800, height: 600, quality: 60, function (error, complete) 
 if (error) 
 console.log(error);
 else 
 imageData = await data.getImage(dir + '\image.png')
 console.log(imageData);
 result = model.predict(imageData, batchSize: 4);
 console.log(result);
 gameLoop();
 
 )
 

I know I use model.predict here, I wanted to get the actual image to tensor part working then figure out labels and model.fit() in train-tensor.js in the repo. I don't have any actual working code for training so I didn't include it in this question, sorry if it caused any confusion.

Thank you again!

Edit final working code

const Image, createCanvas = require('canvas');
const canvas = createCanvas(800, 600);
const ctx = canvas.getContext('2d');

async function loadLocalImage (filename) 
 try 
 var img = new Image()
 img.onload = () => ctx.drawImage(img, 0, 0);
 img.onerror = err => throw err ;
 img.src = filename;
 image = tf.fromPixels(canvas);
 return image;
 catch (err) 
 console.log(err);
 

...
...
async getImage(filename) 
 try 
 this.image = await loadLocalImage(filename);
 catch (error) 
 console.log('error loading image', error);
 
 return this.image;
 

tensorflowjs already has a method for this: tf.fromPixels().

You just need to load the image into on of the accepted types(ImageData|HTMLImageElement|HTMLCanvasElement|HTMLVideoElement).

Your image loading Promise returns nothing because your async function doesn't return anything, just your callback, to fix this you need to create and resolve a promise yourself:

const imageGet = require('get-image-data');
async loadLocalImage(filename) {
 return new Promise((res,rej)=>
 imageGet(filename, (err, info) => 
 if(err)
 rej(err);
 return;
 
 const image = tf.fromPixels(info.data)
 console.log(image, '127');
 res(image);
 );