This project is in the prototype stage !
Draw svg quick-responsively with a mouse or pen,
Designed for note-taking, formulas drafting
OCR to latex supported now (by Customize-Buttons and myscript API)
open markdown file, right click at some line,
Edit/Create current line as SVG
or press F1
, type svg
, click the command
Compatibility is not guaranteed in prototype stage
Todo:
- fold the svg automatically and add a command to fold all svg
- adjust behavior of pens
- fix the bug of polygon
Welcome star to support the project or issue to provide ideas
I have not decided the interface. In this stage, it could work this way.
Step 1. You should change the following code(icon/title/onclick), run it in chrome console. It prints one line of string.
console.log(JSON.stringify((()=>{
// find a icon from https://fontawesome.com/v5.8.2/icons
const icon = 'beer'
const title = 'a new button'
const addGap = true
const onclick = ()=>{
drawAPI.unstable.setTextContent('a custom button works')
}
document.querySelector('div.svg-operate').insertAdjacentHTML('beforeEnd',(addGap?"<span class='svgiconspliter'></span>":"")+"<div class='svg-btn fa fa-"+icon+"' title='"+title+"'><span></span></div>");
document.querySelector('div.svg-operate > :last-child').onclick=onclick
}).toString().slice(7,-2)))
" // find a icon from https://fontawesome.com/v5.8.2/icons\n const icon = 'beer' \n const title = 'a new button'\n const addGap = true\n const onclick = ()=>{\n drawAPI.unstable.setTextContent('a custom button works')\n }\n\n document.querySelector('div.svg-operate').insertAdjacentHTML('beforeEnd',(addGap?\"<span class='svgiconspliter'></span>\":\"\")+\"<div class='svg-btn fa fa-\"+icon+\"' title='\"+title+\"'><span></span></div>\");\n document.querySelector('div.svg-operate > :last-child').onclick=onclick\n"
Step 2. Push the string into your settings.json
"markdown-draw.customized-buttons": [
{
"type": "script",
"version": "0.1.2",
"function": " // find a icon from https://fontawesome.com/v5.8.2/icons\n const icon = 'beer' \n const title = 'a new button'\n const addGap = true\n const onclick = ()=>{\n drawAPI.unstable.setTextContent('a custom button works')\n }\n\n document.querySelector('div.svg-operate').insertAdjacentHTML('beforeEnd',(addGap?\"<span class='svgiconspliter'></span>\":\"\")+\"<div class='svg-btn fa fa-\"+icon+\"' title='\"+title+\"'><span></span></div>\");\n document.querySelector('div.svg-operate > :last-child').onclick=onclick\n"
}
]
myscript API
Recognitize handwritten text to latex using API of myscript.com
which declares MyScript has been working on this matter for over two decades now and there is no question that our technology is ahead of Microsoft Ink Recognizer.
at their article.
Create a account at https://developer.myscript.com/getting-started/web to get token.
It is free for 2000 calls/month.
Paste your token on PasteYourTokenHere
, and push this into setting.json following the previous section Customize Buttons. The json generated from this script
{"type":"script","version":"0.1.2","function":"var token = {\n applicationKey: 'PasteYourTokenHere',\n hmacKey: 'PasteYourTokenHere',\n}\nvar addGap = true\n\nvar iink_cdn = 'https://cdn.jsdelivr.net/npm/[email protected]/dist/iink.min.js'\n// var iink_cdn = https://unpkg.com/[email protected]/dist/iink.min.js\n// var iink_cdn = https://myscript.github.io/iinkJS/dist/iink.min.js\n\nvar icon = 'square-root-alt'\nvar title = 'Recognize to latex'\n\ndocument.querySelector('div.svg-operate').insertAdjacentHTML('beforeEnd', (addGap ? \"<span class='svgiconspliter'></span>\" : \"\") + \"<div class='svg-btn fa fa-\" + icon + \"' title='\" + title + \"'><span></span></div>\");\nvar btnElement = document.querySelector('div.svg-operate > :last-child')\nbtnElement.onclick = ()=>{drawAPI.unstable.setTextContent('loading script')}\n\nvar svg = drawAPI.unstable.getSVGElement()\n\nvar getStrokeGroups = () => {\n let strokes = []\n for (const item of svg.children) {\n if (item.nodeName === 'path') {\n let points = item\n .getAttributeNS(null, \"d\")\n .split(/M |L /g)\n .slice(1)\n .map(item => {\n return {\n x: parseFloat(item.split(\",\")[0]),\n y: parseFloat(item.split(\",\")[1])\n };\n })\n strokes.push({ x: points.map(v => v.x), y: points.map(v => v.y) })\n }\n }\n console.log(strokes);\n return [{ \"penStyle\": null, \"strokes\": strokes }]\n}\n\nvar s1 = document.createElement('script')\ns1.setAttribute('nonce', drawAPI.unstable.nonce())\ns1.src = iink_cdn\ns1.onload = () => {\n\n const onclick = () => {\n drawAPI.unstable.setTextContent('calling the API')\n // Creating a recognizer\n const iinkRecognizer = iink.DefaultBehaviors.recognizerList.find(x => {\n const infos = x.getInfo();\n return infos.protocol === 'REST';\n });\n\n // Creating a empty model\n const model = iink.InkModel.createModel();\n // Filling the model with the stroke groups\n model.strokeGroups = getStrokeGroups();\n\n // Creating a recognizer context with the configuration attached\n const recognizerContext = iink.RecognizerContext.createEmptyRecognizerContext({\n configuration: iink.DefaultConfiguration\n });\n\n recognizerContext.editor.configuration.recognitionParams = {\n type: 'MATH',\n protocol: 'REST',\n server: {\n scheme: 'https',\n // host: 'webdemoapi.myscript.com',\n host: 'cloud.myscript.com',\n applicationKey: token.applicationKey,\n hmacKey: token.hmacKey\n },\n iink: {\n math: {\n mimeTypes: [\n 'application/x-latex',\n ],\n }\n }\n }\n\n // Assigning a theme to the document\n recognizerContext.editor.theme = iink.DefaultTheme;\n\n // Defining the behaviour on recognition result\n const recognitionCallback = (err, x) => {\n if (!err) {\n Object.entries(x.exports)\n .forEach(([mimeType, exportValue]) => {\n\n let latex = x.exports[mimeType]\n let content = '\\n$$'+latex.trim()+'$$ '+' '\n drawAPI.unstable.setTextContent('')\n drawAPI.unstable.editCurrentLine({\n control: 0,\n text: content\n })\n });\n }\n };\n\n // Triggering the recognition\n iinkRecognizer.export_(recognizerContext, model)\n .then((values) => {\n values.forEach((value) => {\n recognitionCallback(undefined, value);\n });\n })\n .catch(err => recognitionCallback(err, undefined));\n }\n\n btnElement.onclick = onclick\n}\ndocument.body.appendChild(s1)\n"}
Actually it is HTR not OCR.
mathpix API (click to expand)
The mathpix API is a popular choice for OCR to latex.
Its API it free for 1000/month, but you have to provide a card.
They charge a one-time non-refundable setup fee of $1.
The following script also supports drag and paste image to latex by mathpix API.
And I am considering providing another independent extension to convert the clipbord-picture to latex.
Which need not to open a webview panel and provides a command and is able to bind keys.
(It will be https://github.com/zhaouv/vscode-paste-to-latex-mathpix.git if it becomes existence.)
{"type":"script","version":"0.1.2","function":"var token = { app_id: 'PasteYourTokenHere', app_key: 'PasteYourTokenHere' }\nvar addGap = true\nvar icon = 'square-root-alt'\nvar title = 'Recognize to latex'\n\ndocument.querySelector('div.svg-operate').insertAdjacentHTML('beforeEnd', (addGap ? \"<span class='svgiconspliter'></span>\" : \"\") + \"<div class='svg-btn fa fa-\" + icon + \"' title='\" + title + \"'><span></span></div>\");\nvar btnElement = document.querySelector('div.svg-operate > :last-child')\nbtnElement.onclick = () => {\n\n drawAPI.unstable.getPNG((dataURL) => {\n drawAPI.unstable.setTextContent('calling the API')\n xhrPost(dataURL, (err,ret)=>{\n console.log(err,ret)\n let latex = JSON.parse(ret)['latex']\n let content = '\\n$$'+latex.trim()+'$$ '+' '\n drawAPI.unstable.setTextContent('')\n drawAPI.unstable.editCurrentLine({\n control: 0,\n text: content\n })\n })\n })\n}\n\nfunction xhrPost(dataURL, callback) {\n var xhr = new XMLHttpRequest();\n xhr.onreadystatechange = function () {\n if (xhr.readyState == 4) {\n if ((xhr.status >= 200 && xhr.status < 300) || xhr.status == 304) {\n callback(null, xhr.responseText);\n } else {\n callback([xhr.status, xhr.responseText], null);\n }\n }\n }\n xhr.open('post', 'https://api.mathpix.com/v3/latex');\n xhr.setRequestHeader('app_id', token.app_id)\n xhr.setRequestHeader('app_key', token.app_key)\n xhr.setRequestHeader('Content-type', 'application/json')\n xhr.send(JSON.stringify({ 'url': dataURL }));\n}\n\n// drag and paste image\nfunction convertImage(dataURL) {\n drawAPI.unstable.setTextContent('calling the API to convert image')\n xhrPost(dataURL, (err,ret)=>{\n console.log(err,ret)\n let latex = JSON.parse(ret)['latex']\n let content = '\\n$$'+latex.trim()+'$$ '+' '\n drawAPI.unstable.setTextContent('')\n drawAPI.unstable.editCurrentLine({\n control: 0,\n text: content\n })\n })\n\n}\nfunction getImage(items, cb) {\n var file = null;\n if (items && items.length) {\n for (var i = 0; i < items.length; i++) {\n if (items[i].type.indexOf('image') !== -1) {\n file = items[i].getAsFile();\n break;\n }\n }\n }\n // console.log(file);\n if (file) {\n var reader = new FileReader()\n reader.onload = function (event) {\n cb(event.target.result);\n }\n reader.readAsDataURL(file);\n }\n}\nvar bindElement = document.body\nbindElement.addEventListener('paste', function (event) {\n var items = event.clipboardData?.items;\n getImage(items,convertImage);\n});\nbindElement.ondragover = function (ev) {\n ev.preventDefault();\n}\nbindElement.ondrop = function (ev) {\n ev.preventDefault();\n var items = ev.dataTransfer.items;\n getImage(items,convertImage);\n}\n"}
Paste your token on PasteYourTokenHere
, and push this into setting.json following the previous section Customize Buttons. The json generated from this script
board adapted from syfxlin/graffiti-board Apache-2.0
with lots of changes
board/Font-Awesome-5-8-2-all-min.css from Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
replace webfonts/fa-solid-900.woff2 by base64