Tokenization Evaluator

Tokenization Evaluator

Analyze and compare different text tokenization methods.

Choose Tokenizer(s)

Token Count: ${res.count}

${res.tokens.map(token => `${escapeHtml(token)}`).join('')}
`; }); pdfReportContainer.innerHTML = reportHtml; }; const downloadPdf = async () => { if (!window.jspdf || !window.html2canvas) { pdfMessage.textContent = 'Error: PDF libraries not loaded.'; pdfMessage.classList.add('text-red-500'); return; } downloadPdfBtn.disabled = true; loader.classList.remove('hidden'); pdfMessage.textContent = 'Generating PDF, please wait... This may take a moment.'; pdfMessage.classList.remove('text-red-500', 'text-green-500'); try { const { jsPDF } = window.jspdf; const canvas = await html2canvas(pdfReportContainer, { scale: 2, // Higher scale for better quality useCORS: true, logging: false }); const imgData = canvas.toDataURL('image/png'); const pdf = new jsPDF({ orientation: 'portrait', unit: 'mm', format: 'a4' }); const pdfWidth = pdf.internal.pageSize.getWidth(); const pdfHeight = pdf.internal.pageSize.getHeight(); const imgWidth = canvas.width; const imgHeight = canvas.height; const ratio = imgWidth / imgHeight; const canvasPdfWidth = pdfWidth - 20; // with margin const canvasPdfHeight = canvasPdfWidth / ratio; let position = 0; let heightLeft = canvasPdfHeight; const margin = 10; pdf.addImage(imgData, 'PNG', margin, margin, canvasPdfWidth, canvasPdfHeight); heightLeft -= (pdfHeight - 2 * margin); while (heightLeft > 0) { position -= (pdfHeight - 2 * margin); pdf.addPage(); pdf.addImage(imgData, 'PNG', margin, position + margin, canvasPdfWidth, canvasPdfHeight); heightLeft -= (pdfHeight - 2 * margin); } pdf.save('tokenization-report.pdf'); pdfMessage.textContent = 'PDF downloaded successfully!'; pdfMessage.classList.add('text-green-500'); } catch (error) { console.error("PDF Generation Error:", error); pdfMessage.textContent = 'An error occurred while generating the PDF.'; pdfMessage.classList.add('text-red-500'); } finally { loader.classList.add('hidden'); downloadPdfBtn.disabled = false; setTimeout(() => { pdfMessage.textContent = ''; }, 5000); } }; const escapeHtml = (unsafe) => { return unsafe .replace(/&/g, "&") .replace(//g, ">") .replace(/"/g, """) .replace(/'/g, "'"); }; // --- Event Listeners --- if (evaluateBtn) { evaluateBtn.addEventListener('click', evaluateTokenization); } if (downloadPdfBtn) { downloadPdfBtn.addEventListener('click', downloadPdf); } });
Scroll to Top