English Natural Language Processing / Observable

English Natural Language Processingtodo:

[ ] ensure that markdown linebreaks (titles, bullets) are treated as new paragraphs)

viewof inputText = { const value = `Moby-Dick; or, The Whale is an 1851 novel by American writer Herman Melville. The book is sailor Ishmael's narrative of the obsessive quest of Ahab, captain of the whaling ship Pequod, for revenge on Moby Dick, the white whale that on the ship's previous voyage bit off Ahab's leg at the knee. A contribution to the literature of the American Renaissance, the work's genre classifications range from late Romantic to early Symbolist. Moby-Dick was published to mixed reviews, was a commercial failure, and was out of print at the time of the author's death in 1891. Its reputation as a "Great American Novel" was established only in the 20th century, after the centennial of its author's birth. William Faulkner confessed he wished he had written the book himself,[1] and D. H. Lawrence called it "one of the strangest and most wonderful books in the world" and "the greatest book of the sea ever written".[2] Its opening sentence, "Call me Ishmael", is among world literature's most famous.` return textarea({ title: "Text to analyze", value: value, width: "100%", rows: 10, }) }Readabilityhttps://github.com/retextjs/retext-readabilityviewof readabilityAge = slider({ min: 1, max: 25, step: 1, value: 16, title: "Readability age", }){ let div = md`${inputText}` let instance = new Mark(div); instance.unmark() const results = await new Promise((resolve, reject) => { retext().use(retextPlugins.readability, { age: readabilityAge, threshold: 1 / 7 }).process(div.textContent, (err, file) => err ? reject(err) : resolve(file)) }); results.messages.forEach(function(message) { const offset = message.location.start.offset; const length = message.location.end.offset - message.location.start.offset const reason = message.reason const [confidence, confidenceMax] = message.confidence.split('/') const color = d3.scaleSequential(d3.interpolateReds).domain([0, confidenceMax])(confidence) instance.markRanges([{ start: offset, length: length }], { each: function(el) { el.setAttribute('title', message.reason); el.setAttribute('style', `background-color: ${color};`); }, }) }); return div; }Simplifyhttps://github.com/retextjs/retext-simplify{ let div = md`${inputText}` let instance = new Mark(div); const results = await new Promise((resolve, reject) => { retext().use(retextPlugins.simplify).process(div.textContent, (err, file) => err ? reject(err) : resolve(file)) }); results.messages.forEach(function(message) { const offset = message.location.start.offset; const length = message.location.end.offset - message.location.start.offset const reason = message.reason instance.markRanges([{ start: offset, length: length }], { each: function(el) { el.setAttribute('title', message.reason); el.setAttribute('style', `background-color: purple;`); }, }) }); return div; }Sentimenthttps://github.com/retextjs/retext-sentiment{ let div = md`${inputText}` let instance = new Mark(div); const processor = retext().use(retextPlugins.sentiment); const tree = processor.parse(div.textContent) await processor.run(tree) visit(tree, 'SentenceNode', function (node) { const offset = node.position.start.offset; const length = node.position.end.offset - node.position.start.offset const polarity = node.data.polarity const color = d3.scaleSequential(d3.interpolateRdYlGn).domain([-5, 5])(polarity) instance.markRanges([{ start: offset, length: length }], { each: function(el) { el.setAttribute('style', `background-color: ${color};`); }, }) }); return div; }Sentence Length / Density{ let div = md`${inputText}` let instance = new Mark(div); const processor = retext() const tree = processor.parse(div.textContent) await processor.run(tree) let sentences = []; visit(tree, 'SentenceNode', function (node) { sentences.push(node); }); const maxLength = d3.max(sentences, function(sentence) { return sentence.position.end.offset - sentence.position.start.offset }); sentences.forEach(function(sentence) { const offset = sentence.position.start.offset; const length = sentence.position.end.offset - sentence.position.start.offset const color = d3.scaleSequential(d3.interpolateGreys).domain([0, maxLength])(length) instance.markRanges([{ start: offset, length: length }], { each: function(el) { el.setAttribute('style', `background-color: ${color}; color: transparent;`); }, }) }); return div; }Inspiration

Retext: https://github.com/retextjs/retext/blob/master/doc/plugins.md

Write Good: Suggestions for writing: https://github.com/btford/write-good

Things that Marked.app highlights: http://marked2app.com/help/Keyword_Highlighting.html

Keywords
Doubled words ("the the")
Passive Voice
Word Repetition

Existing Editors:

Nitpicker Tool: http://nitpickertool.com
Hemingway App: http://www.hemingwayapp.com/
Grammarly: https://www.grammarly.com

Enchanting Marketing: https://www.enchantingmarketing.com/write-clear-and-concise-sentences/

Plain English Guides: http://www.plainenglish.co.uk/free-guides.html

Blast.js: has a "Density" view that turns sentences into different grayscales depending on length: http://velocityjs.org/blast/

Mark = require('mark.js')import { retext, plugins as retextPlugins, resolve as retextResolve, visit } from '@bensheldon/retext'd3 = require("https://d3js.org/d3.v5.min.js")import { slider, textarea } from "@jashkenas/inputs"