app.js

import jsQR from "./jsQR/";

var SCREEN_WIDTH = 256;
var SCREEN_HEIGHT = 240;

const nesCanvas = document.getElementById("nes-canvas");
canvas_ctx = nesCanvas.getContext("2d");
let image = canvas_ctx.getImageData(0, 0, SCREEN_WIDTH, SCREEN_HEIGHT);

canvas_ctx.fillStyle = "black";
canvas_ctx.fillRect(0, 0, SCREEN_WIDTH, SCREEN_HEIGHT);

const nesWorker = new Worker(new URL("nes.js", import.meta.url), {
  type: "module",
});

nesWorker.onmessage = (buffer) => {
  image.data.set(buffer["data"]);
};

function onAnimationFrame() {
  window.requestAnimationFrame(onAnimationFrame);

  canvas_ctx.putImageData(image, 0, 0);
}

onAnimationFrame();

var video = document.createElement("video");
var canvasElement = document.getElementById("canvas");
var canvas = canvasElement.getContext("2d", { willReadFrequently: true });
var loadingMessage = document.getElementById("loadingMessage");
var outputContainer = document.getElementById("output");
var outputData = document.getElementById("outputData");

const latencyDiv = document.getElementById("latency");
const latencyScan = document.getElementById("scan");

// buttons
const up = document.getElementById("up");
const down = document.getElementById("down");
const left = document.getElementById("left");
const right = document.getElementById("right");
const a = document.getElementById("a");
const b = document.getElementById("b");
const select = document.getElementById("select");
const start = document.getElementById("start");

// Recording buttons, i.e. when you click them we'll start recording inputs
// which should later count as that button press when seen on the QR code
const recordNetural = document.getElementById("noInputRec");
const recordUp = document.getElementById("upRec");
const recordDown = document.getElementById("downRec");
const recordLeft = document.getElementById("leftRec");
const recordRight = document.getElementById("rightRec");
const recordA = document.getElementById("aRec");
const recordB = document.getElementById("bRec");
// const recordSelect = document.getElementById("selectRec");
// const recordStart = document.getElementById("startRec");

const isNeutralHeld = false;

const recordingButtons = [
  recordNetural,
  recordUp,
  recordDown,
  recordLeft,
  recordRight,
  recordA,
  recordB,
  // recordSelect,
  // recordStart,
]

const codeDataButtonIndexes = [
  [14, 19], // Neutral
  [15, 19], // Up
  [15, 19], // Down
  [15, 19], // Left
  [15, 19], // Right
  [0, 5],   // A
  [0, 5],   // B
  // [0, 5],   // Select
  // [0, 5],   // Start
]

let recordingButtonIsHeldArr = [
  false, // recordNetural
  false, // recordUp
  false, // recordDown
  false, // recordLeft
  false, // recordRight
  false, // recordA
  false, // recordB
  // false, // recordSelect
  // false, // recordStart
]

function analyze(arr) {
  // Find min length in arr
  let minLength = Infinity;
  arr.forEach(codeData => {
    if (codeData.length < minLength) {
      minLength = codeData.length;
    }
  });
  let answer = "";
  for (let i = 0; i < minLength; i++) {
    let allSame = true;
    let thisLetter = arr[0][i];
    arr.forEach(codeData => {
      if (codeData[i] !== thisLetter) {
        allSame = false;
      }
    });
    if (allSame) {
      answer += thisLetter;
    } else {
      answer += "~";
    }
  }
  console.log(answer);
  return answer;
}

// Add a listener for each recording button to set the corresponding index in the array to true when pressed
recordingButtons.forEach((button, index) => {
  button.onpointerdown = () => {
    recordingButtonIsHeldArr[index] = true;
  }
  button.onpointerup = () => {
    recordingButtonIsHeldArr[index] = false;
    let relevantRecordedData = recordedData[index];
    // Get the top 20% of the recordedData for this button
    // Sum the counts
    let counts = Object.values(relevantRecordedData).map(data => data.count);
    const sum = counts.reduce((partialSum, a) => partialSum + a, 0);
    let threshold = 0;
    if (index === 0) {
      threshold = sum * 0.80;
    } else {
      threshold = sum * 0.40;
    }
    // Get sorted list of recordedData for this button
    let sortedRecordedData = Object.entries(relevantRecordedData).sort((a, b) => b[1].count - a[1].count);
    // Convert sortedRecordedData to an array of arrays
    // where each element is [code.data, { count: count }]
    sortedRecordedData = sortedRecordedData.map(data => [data[0], data[1].count]);

    // Create accumulator and iterate over sortedRecordedData and filter out values
    // below the threshold
    let bestValues = [];
    let accumulator = 0;
    // TODO Replace this with a foreach
    for (let i = 0; i < sortedRecordedData.length; i++) {
      if (accumulator > threshold) {
        break;
      }
      bestValues.push([sortedRecordedData[i][0], getButtonReadouts(sortedRecordedData[i][0])]);
      accumulator += sortedRecordedData[i][1];
    }

    bestValues.forEach((codeData, buttonReadout) => {
      console.log('me');
    });


    let stringBestValues = bestValues.map(a => a[0]);
    stringBestValues.forEach(codeData => {
      let relevantSlice = codeData.substring(
        codeDataButtonIndexes[index][0],
        codeDataButtonIndexes[index][1]
      );
      // Check if relevantSlice is in codeDataButtonSubstrings[index]
      if (!codeDataButtonSubstrings[index].includes(relevantSlice)) {
        codeDataButtonSubstrings[index].push(relevantSlice);
        console.log(`Adding ${relevantSlice} to codeDataButtonSubstrings[${index}]`);
      }
    });

    // if (index === 0) {
    //   // TODO Only add unique ones.
    //   codeDataButtonSubstrings[0].push(...bestValues.map(codeData => codeData[0]));
    //   let junk = analyze(codeDataButtonSubstrings[0]);
    //   debugger;
    // } else {



    // let junk = analyze(codeDataButtonSubstrings[index]);

    // let minDiff = Infinity;
    // let minDiffString = "";
    // bestValues.forEach(possibleInput => {
    //   // Find the string in codeDataButtonSubstrings[0] which differes from
    //   // possibleInput by the least characters
    //   codeDataButtonSubstrings[0].forEach(neutralCodeData => {
    //     let diff = 0;
    //     for (let i = 0; i < neutralCodeData.length; i++) {
    //       if (neutralCodeData[i] !== possibleInput[0][i]) {
    //         diff++;
    //       }
    //     }
    //     if (diff < minDiff) {
    //       minDiff = diff;
    //       minDiffString = neutralCodeData;
    //     }
    //   });
    // });

    debugger;
  }
}
});

// let time;
// recordNetural.onpointerdown = function () {
//   isNeutralHeld = false;
//   // time = Date.now();
// }
// recordNetural.onpointerup = function () {
//   isNeutralHeld = false;
//   // console.log(`you held me down for ${Date.now() - time} milliseconds`);
// }

let qr = {};
let pressedButtons = {};

// SO BAD AAAAAAA
up.onpointerdown = () => {
  pressedButtons.up = true;
  checkButtonStatus();
};

up.onpointerup = () => {
  pressedButtons.up = false;
  checkButtonStatus();
};

down.onpointerdown = () => {
  pressedButtons.down = true;
  checkButtonStatus();
};

down.onpointerup = () => {
  pressedButtons.down = false;
  checkButtonStatus();
};

left.onpointerdown = () => {
  pressedButtons.left = true;
  checkButtonStatus();
};

left.onpointerup = () => {
  pressedButtons.left = false;
  checkButtonStatus();
};

right.onpointerdown = () => {
  pressedButtons.right = true;
  checkButtonStatus();
};

right.onpointerup = () => {
  pressedButtons.right = false;
  checkButtonStatus();
};

a.onpointerdown = () => {
  pressedButtons.a = true;
  checkButtonStatus();
};

a.onpointerup = () => {
  pressedButtons.a = false;
  checkButtonStatus();
};

b.onpointerdown = () => {
  pressedButtons.b = true;
  checkButtonStatus();
};

b.onpointerup = () => {
  pressedButtons.b = false;
  checkButtonStatus();
};

start.onpointerdown = () => {
  pressedButtons.start = true;
  checkButtonStatus();
};

start.onpointerup = () => {
  pressedButtons.start = false;
  checkButtonStatus();
};

select.onpointerdown = () => {
  pressedButtons.select = true;
  checkButtonStatus();
};

select.onpointerup = () => {
  pressedButtons.select = false;
  checkButtonStatus();
};

function checkButtonStatus() {
  const pressed = {
    left: pressedButtons.left || qr.left,
    right: pressedButtons.right || qr.right,
    up: pressedButtons.up || qr.up,
    down: pressedButtons.down || qr.down,
    a: pressedButtons.a || qr.a,
    b: pressedButtons.b || qr.b,
    start: pressedButtons.start || qr.start,
    select: pressedButtons.select || qr.select,
  };

  up.style = pressed.up ? "background-color: red;" : "";
  down.style = pressed.down ? "background-color: red;" : "";
  left.style = pressed.left ? "background-color: red;" : "";
  right.style = pressed.right ? "background-color: red;" : "";
  a.style = pressed.a ? "background-color: red;" : "";
  b.style = pressed.b ? "background-color: red;" : "";
  start.style = pressed.start ? "background-color: red;" : "";
  select.style = pressed.select ? "background-color: red;" : "";

  nesWorker.postMessage({ keys: pressed });
}

let datas = {};
// Initialize recordedData as an array as the same length as recordingButtons
// to store the count of each button press
recordedData = new Array(recordingButtons.length).fill({});

function renderData() {
  let html = "<table><tr><th>Data</th><th>Count</th><th>Buttons</th></tr>";
  const keys = Object.keys(datas);
  for (let i = 0; i < keys.length; i++) {
    const key = keys[i];
    const val = datas[key];

    html += `<tr><td>${key}</td><td>${val.count}</td><td>${val.buttons}</td></tr>`;
  }

  html += "</table>";

  outputData.innerHTML = html;
}

function drawLine(begin, end, color) {
  canvas.beginPath();
  canvas.moveTo(begin.x, begin.y);
  canvas.lineTo(end.x, end.y);
  canvas.lineWidth = 4;
  canvas.strokeStyle = color;
  canvas.stroke();
}

// Use facingMode: environment to attemt to get the front camera on phones
navigator.mediaDevices
  .getUserMedia({ video: { facingMode: "environment" } })
  .then(function (stream) {
    video.srcObject = stream;
    video.setAttribute("playsinline", true); // required to tell iOS safari we don't want fullscreen
    video.play();
    requestAnimationFrame(tick);
  });

let codeDataButtonSubstrings = [
  // Indexes 15-19 inclusive for neutral input
  [
    "OONYS",
    "OOQRS",
    "OPQRS",
    "OOQ/S",
    "OOI3S",
    "OPQ/S",
    "OOZ6S",
    "OOPOS",
    "OPNYS",
    "OOO5S",
    "OOI3S",
    "OOO5S",
  ], // Neutral
  [], // Up
  [], // Down
  [
    "OP3%S",
    "OP9OS",
    "OOP8S",
    "OOI3S",
    "OP9+S",
    "OP8$S",
    "OP8$S",
    "OP98S",
  ], // Left
  [
    // "OO$:S",
    // "OPJMS",
    // "OOZ6S",
    // "OPZ6S",
    // "OPCHS",
    // "OPD S",
    // "OOD S",
    // "OPGTS",
    // "OO PS",
    // "OOYJS",
    // "OOFAS",
    // "OPH0S",
    // "OPDGS",
    // "OPDKS",
    // "OPXWS",
    // "OPU*S",
    // "OOGTS",
    // "OODKS",
  ], // Right
  // Indexes 0-5 inclusive
  [
    // "A7H",
    // "ANH",
    // "A8+",
    // "A8H",
    // "A9H",
    // "AO+",
  ], // A
  [], // B
  // [], // Select
  // [], // Start
]

// Returns an object which contains booleans for each button
function getButtonReadouts(codeData) {
  let localQr = {
    //a: qr.a, // don't reset a!
  };

  let directionComponent = codeData.substring(
    codeDataButtonIndexes[0][0],
    codeDataButtonIndexes[0][1]
  );
  let aComponent = codeData.substring(
    codeDataButtonIndexes[5][0],
    codeDataButtonIndexes[5][1]
  );

  // Left
  if (
    codeDataButtonSubstrings[3].some((substring) => directionComponent === substring)
  ) {
    localQr.left = true;
  }

  // Right
  if (
    codeDataButtonSubstrings[4].some((substring) => directionComponent === substring)
  ) {
    localQr.right = true;
  }

  // A button
  if (
    codeDataButtonSubstrings[5].some((substring) => aComponent === substring)
  ) {
    localQr.a = true;
  }

  return localQr;
}

function tick() {
  loadingMessage.innerText = "⌛ Loading video...";
  if (video.readyState === video.HAVE_ENOUGH_DATA) {
    loadingMessage.hidden = true;
    canvasElement.hidden = false;
    outputContainer.hidden = false;

    canvasElement.height = video.videoHeight;
    canvasElement.width = video.videoWidth;
    canvas.drawImage(video, 0, 0, canvasElement.width, canvasElement.height);
    var imageData = canvas.getImageData(
      0,
      0,
      canvasElement.width,
      canvasElement.height,
    );

    const start = window.performance.now();
    var code = jsQR(imageData.data, imageData.width, imageData.height, {
      inversionAttempts: "dontInvert",
    });
    const end = window.performance.now();
    const delta = end - start;

    latencyDiv.innerText = `Frame latency: ${delta}ms`;

    if (code?.data.length > 0) {
      // reset
      qr = getButtonReadouts(code.data);

      latencyScan.innerText = `Scan latency: ${delta}ms`;

      drawLine(
        code.location.topLeftCorner,
        code.location.topRightCorner,
        "#FF3B58",
      );
      drawLine(
        code.location.topRightCorner,
        code.location.bottomRightCorner,
        "#FF3B58",
      );
      drawLine(
        code.location.bottomRightCorner,
        code.location.bottomLeftCorner,
        "#FF3B58",
      );
      drawLine(
        code.location.bottomLeftCorner,
        code.location.topLeftCorner,
        "#FF3B58",
      );

      // If a given recording button is being held down, then record its count
      // to the recordDatas object
      for (let i = 0; i < recordingButtonIsHeldArr.length; i++) {
        if (recordingButtonIsHeldArr[i]) {
          let buttonDataFreq = recordedData[i];
          // Check if code.data is already in the recordedData object
          // If it is, increment the count
          if (code.data in buttonDataFreq) {
            buttonDataFreq[code.data].count += 1;
          } else {
            // If it isn't, add it with a count of 1
            buttonDataFreq[code.data] = { count: 1 };
          }
        }
      }

      // console.log(recordingButtonIsHeldArr);
      console.log(recordedData[0]);
      // Log sorted recordedData[0] to the console
      console.log(
        Object.entries(recordedData[0]).sort((a, b) => b[1].count - a[1].count),
      );


      if (!datas[code.data]) {
        let buttons = [];
        for (let i = 0; i < Object.keys(qr).length; i++) {
          const k = Object.keys(qr)[i];
          if (qr[k]) {
            buttons.push(k);
          }
        }

        datas[code.data] = { count: 1, buttons };
      } else {
        datas[code.data].count += 1;
      }

      renderData();
    }

    checkButtonStatus();
  }

  requestAnimationFrame(tick);
}