import jsQR from "./jsQR/"; var SCREEN_WIDTH = 256; var SCREEN_HEIGHT = 240; const nesCanvas = document.getElementById("nes-canvas"); canvas_ctx = nesCanvas.getContext("2d"); let image = canvas_ctx.getImageData(0, 0, SCREEN_WIDTH, SCREEN_HEIGHT); canvas_ctx.fillStyle = "black"; canvas_ctx.fillRect(0, 0, SCREEN_WIDTH, SCREEN_HEIGHT); const nesWorker = new Worker(new URL("nes.js", import.meta.url), { type: "module", }); nesWorker.onmessage = (buffer) => { image.data.set(buffer["data"]); }; function onAnimationFrame() { window.requestAnimationFrame(onAnimationFrame); canvas_ctx.putImageData(image, 0, 0); } onAnimationFrame(); var video = document.createElement("video"); var canvasElement = document.getElementById("canvas"); var canvas = canvasElement.getContext("2d", { willReadFrequently: true }); var loadingMessage = document.getElementById("loadingMessage"); var outputContainer = document.getElementById("output"); var outputData = document.getElementById("outputData"); const latencyDiv = document.getElementById("latency"); const latencyScan = document.getElementById("scan"); // buttons const up = document.getElementById("up"); const down = document.getElementById("down"); const left = document.getElementById("left"); const right = document.getElementById("right"); const a = document.getElementById("a"); const b = document.getElementById("b"); const select = document.getElementById("select"); const start = document.getElementById("start"); // Recording buttons, i.e. when you click them we'll start recording inputs // which should later count as that button press when seen on the QR code const recordNetural = document.getElementById("noInputRec"); const recordUp = document.getElementById("upRec"); const recordDown = document.getElementById("downRec"); const recordLeft = document.getElementById("leftRec"); const recordRight = document.getElementById("rightRec"); const recordA = document.getElementById("aRec"); const recordB = document.getElementById("bRec"); // const recordSelect = document.getElementById("selectRec"); // const recordStart = document.getElementById("startRec"); const isNeutralHeld = false; const recordingButtons = [ recordNetural, recordUp, recordDown, recordLeft, recordRight, recordA, recordB, // recordSelect, // recordStart, ] const codeDataButtonIndexes = [ [14, 19], // Neutral [15, 19], // Up [15, 19], // Down [15, 19], // Left [15, 19], // Right [0, 5], // A [0, 5], // B // [0, 5], // Select // [0, 5], // Start ] let recordingButtonIsHeldArr = [ false, // recordNetural false, // recordUp false, // recordDown false, // recordLeft false, // recordRight false, // recordA false, // recordB // false, // recordSelect // false, // recordStart ] function analyze(arr) { // Find min length in arr let minLength = Infinity; arr.forEach(codeData => { if (codeData.length < minLength) { minLength = codeData.length; } }); let answer = ""; for (let i = 0; i < minLength; i++) { let allSame = true; let thisLetter = arr[0][i]; arr.forEach(codeData => { if (codeData[i] !== thisLetter) { allSame = false; } }); if (allSame) { answer += thisLetter; } else { answer += "~"; } } console.log(answer); return answer; } // Add a listener for each recording button to set the corresponding index in the array to true when pressed recordingButtons.forEach((button, index) => { button.onpointerdown = () => { recordingButtonIsHeldArr[index] = true; } button.onpointerup = () => { recordingButtonIsHeldArr[index] = false; let relevantRecordedData = recordedData[index]; // Get the top 20% of the recordedData for this button // Sum the counts let counts = Object.values(relevantRecordedData).map(data => data.count); const sum = counts.reduce((partialSum, a) => partialSum + a, 0); let threshold = 0; if (index === 0) { threshold = sum * 0.80; } else { threshold = sum * 0.40; } // Get sorted list of recordedData for this button let sortedRecordedData = Object.entries(relevantRecordedData).sort((a, b) => b[1].count - a[1].count); // Convert sortedRecordedData to an array of arrays // where each element is [code.data, { count: count }] sortedRecordedData = sortedRecordedData.map(data => [data[0], data[1].count]); // Create accumulator and iterate over sortedRecordedData and filter out values // below the threshold let bestValues = []; let accumulator = 0; // TODO Replace this with a foreach for (let i = 0; i < sortedRecordedData.length; i++) { if (accumulator > threshold) { break; } bestValues.push([sortedRecordedData[i][0], getButtonReadouts(sortedRecordedData[i][0])]); accumulator += sortedRecordedData[i][1]; } bestValues.forEach((codeData, buttonReadout) => { console.log('me'); }); let stringBestValues = bestValues.map(a => a[0]); stringBestValues.forEach(codeData => { let relevantSlice = codeData.substring( codeDataButtonIndexes[index][0], codeDataButtonIndexes[index][1] ); // Check if relevantSlice is in codeDataButtonSubstrings[index] if (!codeDataButtonSubstrings[index].includes(relevantSlice)) { codeDataButtonSubstrings[index].push(relevantSlice); console.log(`Adding ${relevantSlice} to codeDataButtonSubstrings[${index}]`); } }); // if (index === 0) { // // TODO Only add unique ones. // codeDataButtonSubstrings[0].push(...bestValues.map(codeData => codeData[0])); // let junk = analyze(codeDataButtonSubstrings[0]); // debugger; // } else { // let junk = analyze(codeDataButtonSubstrings[index]); // let minDiff = Infinity; // let minDiffString = ""; // bestValues.forEach(possibleInput => { // // Find the string in codeDataButtonSubstrings[0] which differes from // // possibleInput by the least characters // codeDataButtonSubstrings[0].forEach(neutralCodeData => { // let diff = 0; // for (let i = 0; i < neutralCodeData.length; i++) { // if (neutralCodeData[i] !== possibleInput[0][i]) { // diff++; // } // } // if (diff < minDiff) { // minDiff = diff; // minDiffString = neutralCodeData; // } // }); // }); debugger; } } }); // let time; // recordNetural.onpointerdown = function () { // isNeutralHeld = false; // // time = Date.now(); // } // recordNetural.onpointerup = function () { // isNeutralHeld = false; // // console.log(`you held me down for ${Date.now() - time} milliseconds`); // } let qr = {}; let pressedButtons = {}; // SO BAD AAAAAAA up.onpointerdown = () => { pressedButtons.up = true; checkButtonStatus(); }; up.onpointerup = () => { pressedButtons.up = false; checkButtonStatus(); }; down.onpointerdown = () => { pressedButtons.down = true; checkButtonStatus(); }; down.onpointerup = () => { pressedButtons.down = false; checkButtonStatus(); }; left.onpointerdown = () => { pressedButtons.left = true; checkButtonStatus(); }; left.onpointerup = () => { pressedButtons.left = false; checkButtonStatus(); }; right.onpointerdown = () => { pressedButtons.right = true; checkButtonStatus(); }; right.onpointerup = () => { pressedButtons.right = false; checkButtonStatus(); }; a.onpointerdown = () => { pressedButtons.a = true; checkButtonStatus(); }; a.onpointerup = () => { pressedButtons.a = false; checkButtonStatus(); }; b.onpointerdown = () => { pressedButtons.b = true; checkButtonStatus(); }; b.onpointerup = () => { pressedButtons.b = false; checkButtonStatus(); }; start.onpointerdown = () => { pressedButtons.start = true; checkButtonStatus(); }; start.onpointerup = () => { pressedButtons.start = false; checkButtonStatus(); }; select.onpointerdown = () => { pressedButtons.select = true; checkButtonStatus(); }; select.onpointerup = () => { pressedButtons.select = false; checkButtonStatus(); }; function checkButtonStatus() { const pressed = { left: pressedButtons.left || qr.left, right: pressedButtons.right || qr.right, up: pressedButtons.up || qr.up, down: pressedButtons.down || qr.down, a: pressedButtons.a || qr.a, b: pressedButtons.b || qr.b, start: pressedButtons.start || qr.start, select: pressedButtons.select || qr.select, }; up.style = pressed.up ? "background-color: red;" : ""; down.style = pressed.down ? "background-color: red;" : ""; left.style = pressed.left ? "background-color: red;" : ""; right.style = pressed.right ? "background-color: red;" : ""; a.style = pressed.a ? "background-color: red;" : ""; b.style = pressed.b ? "background-color: red;" : ""; start.style = pressed.start ? "background-color: red;" : ""; select.style = pressed.select ? "background-color: red;" : ""; nesWorker.postMessage({ keys: pressed }); } let datas = {}; // Initialize recordedData as an array as the same length as recordingButtons // to store the count of each button press recordedData = new Array(recordingButtons.length).fill({}); function renderData() { let html = "<table><tr><th>Data</th><th>Count</th><th>Buttons</th></tr>"; const keys = Object.keys(datas); for (let i = 0; i < keys.length; i++) { const key = keys[i]; const val = datas[key]; html += `<tr><td>${key}</td><td>${val.count}</td><td>${val.buttons}</td></tr>`; } html += "</table>"; outputData.innerHTML = html; } function drawLine(begin, end, color) { canvas.beginPath(); canvas.moveTo(begin.x, begin.y); canvas.lineTo(end.x, end.y); canvas.lineWidth = 4; canvas.strokeStyle = color; canvas.stroke(); } // Use facingMode: environment to attemt to get the front camera on phones navigator.mediaDevices .getUserMedia({ video: { facingMode: "environment" } }) .then(function (stream) { video.srcObject = stream; video.setAttribute("playsinline", true); // required to tell iOS safari we don't want fullscreen video.play(); requestAnimationFrame(tick); }); let codeDataButtonSubstrings = [ // Indexes 15-19 inclusive for neutral input [ "OONYS", "OOQRS", "OPQRS", "OOQ/S", "OOI3S", "OPQ/S", "OOZ6S", "OOPOS", "OPNYS", "OOO5S", "OOI3S", "OOO5S", ], // Neutral [], // Up [], // Down [ "OP3%S", "OP9OS", "OOP8S", "OOI3S", "OP9+S", "OP8$S", "OP8$S", "OP98S", ], // Left [ // "OO$:S", // "OPJMS", // "OOZ6S", // "OPZ6S", // "OPCHS", // "OPD S", // "OOD S", // "OPGTS", // "OO PS", // "OOYJS", // "OOFAS", // "OPH0S", // "OPDGS", // "OPDKS", // "OPXWS", // "OPU*S", // "OOGTS", // "OODKS", ], // Right // Indexes 0-5 inclusive [ // "A7H", // "ANH", // "A8+", // "A8H", // "A9H", // "AO+", ], // A [], // B // [], // Select // [], // Start ] // Returns an object which contains booleans for each button function getButtonReadouts(codeData) { let localQr = { //a: qr.a, // don't reset a! }; let directionComponent = codeData.substring( codeDataButtonIndexes[0][0], codeDataButtonIndexes[0][1] ); let aComponent = codeData.substring( codeDataButtonIndexes[5][0], codeDataButtonIndexes[5][1] ); // Left if ( codeDataButtonSubstrings[3].some((substring) => directionComponent === substring) ) { localQr.left = true; } // Right if ( codeDataButtonSubstrings[4].some((substring) => directionComponent === substring) ) { localQr.right = true; } // A button if ( codeDataButtonSubstrings[5].some((substring) => aComponent === substring) ) { localQr.a = true; } return localQr; } function tick() { loadingMessage.innerText = "⌛ Loading video..."; if (video.readyState === video.HAVE_ENOUGH_DATA) { loadingMessage.hidden = true; canvasElement.hidden = false; outputContainer.hidden = false; canvasElement.height = video.videoHeight; canvasElement.width = video.videoWidth; canvas.drawImage(video, 0, 0, canvasElement.width, canvasElement.height); var imageData = canvas.getImageData( 0, 0, canvasElement.width, canvasElement.height, ); const start = window.performance.now(); var code = jsQR(imageData.data, imageData.width, imageData.height, { inversionAttempts: "dontInvert", }); const end = window.performance.now(); const delta = end - start; latencyDiv.innerText = `Frame latency: ${delta}ms`; if (code?.data.length > 0) { // reset qr = getButtonReadouts(code.data); latencyScan.innerText = `Scan latency: ${delta}ms`; drawLine( code.location.topLeftCorner, code.location.topRightCorner, "#FF3B58", ); drawLine( code.location.topRightCorner, code.location.bottomRightCorner, "#FF3B58", ); drawLine( code.location.bottomRightCorner, code.location.bottomLeftCorner, "#FF3B58", ); drawLine( code.location.bottomLeftCorner, code.location.topLeftCorner, "#FF3B58", ); // If a given recording button is being held down, then record its count // to the recordDatas object for (let i = 0; i < recordingButtonIsHeldArr.length; i++) { if (recordingButtonIsHeldArr[i]) { let buttonDataFreq = recordedData[i]; // Check if code.data is already in the recordedData object // If it is, increment the count if (code.data in buttonDataFreq) { buttonDataFreq[code.data].count += 1; } else { // If it isn't, add it with a count of 1 buttonDataFreq[code.data] = { count: 1 }; } } } // console.log(recordingButtonIsHeldArr); console.log(recordedData[0]); // Log sorted recordedData[0] to the console console.log( Object.entries(recordedData[0]).sort((a, b) => b[1].count - a[1].count), ); if (!datas[code.data]) { let buttons = []; for (let i = 0; i < Object.keys(qr).length; i++) { const k = Object.keys(qr)[i]; if (qr[k]) { buttons.push(k); } } datas[code.data] = { count: 1, buttons }; } else { datas[code.data].count += 1; } renderData(); } checkButtonStatus(); } requestAnimationFrame(tick); }