textrank

textrank is a R package that uses the textrank algorithm to 1) rank sentences by their “importance” and 2) identify keywords that appear more frequently in the test. The workflow presented here is a combination of posts by Jan Wijffels and Emil Hvitfeldt.

## Import libraries
library(tidyverse)
library(tidytext)
library(textrank)
library(rvest)

url <- "http://time.com/5196761/fitbit-ace-kids-fitness-tracker/"

article <- read_html(url) %>%
  html_nodes('p')   %>%
  html_text()

article <- paste(article, collapse = "\n")

print(article)

## [1] "Fitbit is launching a new fitness tracker designed for children called the Fitbit Ace, which will go on sale for $99.95 in the second quarter of this year.\nThe Fitbit Ace looks a lot like the company’s Alta tracker, but with a few child-friendly tweaks. The most important of which is Fitbit’s new family account option, which gives parents control over how their child uses their tracker and is compliant with the Children’s Online Privacy Protection Act, or COPPA. Parents must approve who their child can connect with via the Fitbit app and can view their kid’s activity progress and sleep trends, the latter of which can help them manage their children’s bedtimes.\nLike many of Fitbit’s other products, the Fitbit Ace can automatically track steps, monitor active minutes, and remind kids to move when they’ve been still for too long. But while Fitbit’s default move goal is 30 minutes for adult users, the Ace’s will be 60 minutes, in line with the World Health Organization’s recommendation that children between the ages of five and 17 get an hour of daily physical activity per day. Fitbit says the tracker is designed for children eight years old and up.\nFitbit will also be introducing a Family Faceoff feature that lets kids compete in a five-day step challenge against the other members of their family account. The app also will reward children with in-app badges for achieving their health goals. Fitbit’s new child-friendly fitness band will be available in blue and purple, is showerproof, and should last for five days on a single charge.\nThe Ace launch is part of Fitbit’s broader goal of branching out to new audiences. The company also announced a new smartwatch on Tuesday called the Versa, which is being positioned as an everyday smartwatch rather than a fitness-only device or sports watch, like some of the company’s other products.\nAbove all else, the Ace is an effort to get children up and moving. The Centers for Disease Control and Prevention report that the percentage of children and adolescents affected by obesity has more than tripled since the 1970’s. But parents who want to encourage their children to move already have several less expensive options to choose from. Garmin’s $79.99 Vivofit Jr. 2, for example, comes in themed skins like these Minnie Mouse and Star Wars versions, while the wristband entices kids to move by reflecting their fitness achievements in an accompanying smartphone game. The $39.99 Nabi Compete, meanwhile, is sold in pairs so that family members can work together to achieve movement milestones.\nContact us at editors@time.com."

## Import library
library(udpipe)

## Create model
tagger <- udpipe_download_model("english")
tagger <- udpipe_load_model(tagger$file_model)

## Annotate text
article <- udpipe_annotate(tagger, article)

article <- as.data.frame(article)

article %>% 
  head(10) %>% 
  select(-sentence) %>% 
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover"), full_width = F) %>%
  scroll_box(width = "100%")

article_keywords <- textrank_keywords(article$lemma, 
                                      relevant = article$upos %in% c("NOUN", "VERB", "ADJ"))

article_keywords$keywords %>%
  subset(freq > 1) %>%
  kable()

sentences <- unique(article[, c("sentence_id", "sentence")])

terminology <- article %>% 
                  filter(upos %in% c("NOUN", "VERB", "ADJ")) %>%
                  select(sentence_id, lemma) 

tr <- textrank_sentences(data = sentences, 
                         terminology = terminology)

kable(tr[['sentences']] %>% arrange(-textrank))

s <- summary(tr, n = 5, keep.sentence.order = TRUE)

cat(s, sep = "\n\n")

## Fitbit is launching a new fitness tracker designed for children called the Fitbit Ace, which will go on sale for $99.95 in the second quarter of this year.
## 
## The most important of which is Fitbit’s new family account option, which gives parents control over how their child uses their tracker and is compliant with the Children’s Online Privacy Protection Act, or COPPA.
## 
## But while Fitbit’s default move goal is 30 minutes for adult users, the Ace’s will be 60 minutes, in line with the World Health Organization’s recommendation that children between the ages of five and 17 get an hour of daily physical activity per day.
## 
## Fitbit’s new child-friendly fitness band will be available in blue and purple, is showerproof, and should last for five days on a single charge.
## 
## Above all else, the Ace is an effort to get children up and moving.

tr[["sentences"]] %>%
  ggplot(aes(textrank_id, textrank, fill = textrank_id)) +
  geom_col(color="blue", fill=rgb(0.1,0.4,0.5,0.7)) +
  theme_classic() +
  guides(fill = "none") +
  labs(x = "Sentence",
       y = "TextRank score")

link <- "https://ca.indeed.com/viewjob?cmp=Slice-Insurance-Technologies-Inc.&t=Data+Scientist&jk=35c9914ee9b0a052&sjdu=vQIlM60yK_PwYat7ToXhk3myaatk1OsAkhQIw2UEit1eiAfw8fAJl6BKuZ5V5P0TgQjo7h2qNGvAZ_80Lr3XxA&tk=1e1nuvuml0np3000&adid=335410488&pub=4a1b367933fd867b19b072952f68dceb&vjs=3"

description <- tryCatch(
    as.character(link) %>% read_html() %>% html_nodes(xpath='//*[(@id = "jobDescriptionText")]') %>% map(xml_contents),
    error=function(e){NA}
  )
  if (is.null(description)){
    desc <- NA
  }
  
  
final  <- tryCatch(
    paste(as.character(description[[1]]), collapse=' '),
    error=function(e){NA}
  )
  if (is.null(description)){
    NA
  }

job_des_raw <- html_text(read_html(final))
 
print(job_des_raw)

## [1] "WE are Slice Labs (Slice). We’re building a team of innovators and disruptors to change the insurance experience to meet the demands of an on-demand world. We are disrupting the idea that insurance needs to be a fixed product, with a fixed term, with fixed coverage. We believe it can and should be all digital and on-demand, so customers get only the coverage they need, right when they need it. At Slice, we’re focused on ensuring our products provide a positive, individualized customer experience. Our smaller teams invite broader thinking and problem solving, where nobody is pigeonholed into a predetermined role. We work in an open, supportive, environment that values and promotes inclusiveness, innovation, and collaboration. It’s fast-paced, dynamic and fulfilling.\n *The Data Science team is key to Slice’s ongoing success. The data we have and the models we build are foundational to our platform and quite literally drive our business. Our work improves the customer experience, grows our market share and drives business outcomes. This is an opportunity to play an active part in delivering our data vision and determining how we get there. All that to say: Data Science work here at Slice is far from theoretical.\n This ROLE develops and executes Data Science projects across the company. This means hands-on work at all points in the data lifecycle, including data wrangling and mining; feature engineering; model-building and testing; and implementation and communication. Your work will ensure that all our decisions are data-driven - this means that you will have a direct impact on the customer experience by influencing critical decisions on resource deployment and customer engagement. Our immediate applications include marketing analytics, fraud prevention, and risk/value modeling. (Our future applications are boundless!)\n YOU are passionate about data. You enjoy being able to combine your analytical, technical and business skills in one role. You love solving problems and predicting behaviours. You are a collaborator and a communicator and are energized by working with multidisciplinary teams. You are a hands-on learner and are excited by the thought of moving past theory and examples to real-time data science work.\n You bring: \n An effective communication style with an ability to translate “the complex” to “the simple”. You are adept at data visualizations and have experience working with real-time data.4+ years of practical experience using data, models, and common sense to solve tough problems in a collaborative environment. Experience deploying models is a plus!Working experience with Python, including Pandas, NumPy, scikit-learn, NLTK, and Keras/TensorFlow.An understanding of statistical and predictive modeling concepts, machine learning algorithms, clustering and classification techniques.Some exposure to one or more sub-fields of data science, especially GIS/spatial analysis; graph theory/network analysis; or natural language processing (NLP).Working experience with non-Excel BI tools such as Tableau, Looker, Superset, PowerBI, etc.You are comfortable building datasets using traditional relational databases and you’re familiar with alternative databases (noSQL, graph databases) or big data platforms such as Apache Spark.University degree in engineering, applied statistics, data mining, machine learning, mathematics or a related quantitative discipline.*\n Job Types: Full-time, Permanent\n Experience:\n Data Science: 4 years (Required)Location:\n Ottawa, ON (Preferred)"

## Create model
tagger <- udpipe_download_model("english")
tagger <- udpipe_load_model(tagger$file_model)

job_des <- udpipe_annotate(tagger, job_des_raw)

job_des <- as.data.frame(job_des)

job_des %>% 
  head(10) %>% 
  select(-sentence) %>% 
  kable() %>%
  kable_styling(bootstrap_options = c("striped", "hover"), full_width = F) %>%
  scroll_box(width = "100%")

job_des_keywords <- textrank_keywords(job_des$lemma, 
                                      relevant = job_des$upos %in% c("NOUN", "VERB", "ADJ"))

job_des_keywords$keywords %>%
  subset(freq > 1) %>%
  kable()

sentences <- unique(job_des[, c("sentence_id", "sentence")])

terminology <- job_des %>% 
                  filter(upos %in% c("NOUN", "VERB", "ADJ")) %>%
                  select(sentence_id, lemma) 

tr <- textrank_sentences(data = sentences, 
                         terminology = terminology)

kable(tr[['sentences']] %>% arrange(-textrank))

library(magrittr)

x <- tr[['sentences']] %>% arrange(-textrank) %>% head(10)

tr[["sentences"]] %>%
  ggplot(aes(textrank_id, textrank, fill = textrank_id)) +
  geom_col(color="blue", fill=rgb(0.1,0.4,0.5,0.7)) +
  theme_classic() +
  guides(fill = "none") +
  labs(x = "Sentence",
       y = "TextRank score")

pytextrank

import spacy
import pytextrank

# example text
text = r.job_des_raw
 
# load a spaCy model, depending on language, scale, etc.
nlp = spacy.load("en_core_web_sm")
 
# add PyTextRank to the spaCy pipeline
tr = pytextrank.TextRank()

nlp.add_pipe(tr.PipelineComponent, name="textrank", last=True)
 
doc = nlp(text)

# examine the top-ranked phrases in the document
pytextrank = []
for c in list(doc._.textrank.summary(limit_phrases=10, limit_sentences=10)):
        pytextrank.append(str(c))
        print('---')

## ---
## ---
## ---
## ---
## ---
## ---
## ---
## ---
## ---
## ---

for p in doc._.phrases[:10]:
    print('{:.4f} {:5d}  {}'.format(p.rank, p.count, p.text))
    print(p.chunks)

## 0.0789     1  data mining
## [data mining]
## 0.0768     1  data science
## [data science]
## 0.0764     3  data
## [data, data, The data]
## 0.0762     1  experience deploying models
## [Experience deploying models]
## 0.0746     1  data visualizations
## [data visualizations]
## 0.0741     1  big data platforms
## [big data platforms]
## 0.0739     1  data wrangling
## [data wrangling]
## 0.0725     1  experience
## [experience]
## 0.0717     1  nlp).working experience
## [NLP).Working experience]
## 0.0713     1  practical experience
## [practical experience]

gensim

from gensim.summarization.summarizer import summarize

for sent in summarize(text, split=True, ratio=0.3):
  print(sent)
  print('-')

## We’re building a team of innovators and disruptors to change the insurance experience to meet the demands of an on-demand world.
## -
## The data we have and the models we build are foundational to our platform and quite literally drive our business.
## -
## Our work improves the customer experience, grows our market share and drives business outcomes.
## -
## All that to say: Data Science work here at Slice is far from theoretical.
## -
## This means hands-on work at all points in the data lifecycle, including data wrangling and mining; feature engineering; model-building and testing; and implementation and communication.
## -
## Your work will ensure that all our decisions are data-driven - this means that you will have a direct impact on the customer experience by influencing critical decisions on resource deployment and customer engagement.
## -
## You are a collaborator and a communicator and are energized by working with multidisciplinary teams.
## -
## You are a hands-on learner and are excited by the thought of moving past theory and examples to real-time data science work.
## -
## You are adept at data visualizations and have experience working with real-time data.4+ years of practical experience using data, models, and common sense to solve tough problems in a collaborative environment.
## -

summa

from summa.summarizer import summarize

for sent in summarize(text, split=True, ratio=0.3):
  print(sent)
  print('-')

## *The Data Science team is key to Slice’s ongoing success.
## -
## The data we have and the models we build are foundational to our platform and quite literally drive our business.
## -
## Our work improves the customer experience, grows our market share and drives business outcomes.
## -
## All that to say: Data Science work here at Slice is far from theoretical.
## -
## This means hands-on work at all points in the data lifecycle, including data wrangling and mining; feature engineering; model-building and testing; and implementation and communication.
## -
## Your work will ensure that all our decisions are data-driven - this means that you will have a direct impact on the customer experience by influencing critical decisions on resource deployment and customer engagement.
## -
## You are a hands-on learner and are excited by the thought of moving past theory and examples to real-time data science work.
## -
## You are adept at data visualizations and have experience working with real-time data.4+ years of practical experience using data, models, and common sense to solve tough problems in a collaborative environment.
## -
## Experience deploying models is a plus!Working experience with Python, including Pandas, NumPy, scikit-learn, NLTK, and Keras/TensorFlow.An understanding of statistical and predictive modeling concepts, machine learning algorithms, clustering and classification techniques.Some exposure to one or more sub-fields of data science, especially GIS/spatial analysis; graph theory/network analysis; or natural language processing (NLP).Working experience with non-Excel BI tools such as Tableau, Looker, Superset, PowerBI, etc.You are comfortable building datasets using traditional relational databases and you’re familiar with alternative databases (noSQL, graph databases) or big data platforms such as Apache Spark.University degree in engineering, applied statistics, data mining, machine learning, mathematics or a related quantitative discipline.*
## -

Summary

res = r.x

z = res['sentence'].tolist()

import pandas as pd
import collections

 
gensim = summarize(text, split=True, ratio=0.3)

summa = summarize(text, split=True, ratio=0.3)

a = pytextrank + gensim + summa + z

counter=collections.Counter(a)
 
pd.DataFrame(counter, index=[0]).T

##                                                     0
## You are adept at data visualizations and have e...  4
## Some exposure to one or more sub-fields of data...  1
## University degree in engineering, applied stati...  1
## The Data Science team is key to Slice’s ongoing...  1
## YOU are passionate about data.                      1
## You are comfortable building datasets using tra...  1
## This means hands-on work at all points in the d...  4
## WE are Slice Labs (Slice).                          1
## We’re building a team of innovators and disrupt...  2
## We are disrupting the idea that insurance needs...  1
## *The Data Science team is key to Slice’s ongoin...  2
## The data we have and the models we build are fo...  3
## Our work improves the customer experience, grow...  3
## All that to say: Data Science work here at Slic...  2
## Your work will ensure that all our decisions ar...  3
## You are a hands-on learner and are excited by t...  3
## Experience deploying models is a plus!Working e...  2
## Working experience with non-Excel BI tools such...  1
## Working experience with Python, including Panda...  1
## You are a collaborator and a communicator and a...  1

summa_df = pd.DataFrame({'phrases': summa,
                   'summa': [1]*len(summa)})

#summa_df.set_index('phrases')

gensim_df = pd.DataFrame({'phrases': gensim,
                   'gensim': [1]*len(gensim)})

#gensim_df.set_index('phrases')

pytextrank_df = pd.DataFrame({'phrases': pytextrank,
                   'pytextrank': [1]*len(pytextrank)})

#pytextrank_df.set_index('phrases')

textrank_df = pd.DataFrame({'phrases': z,
                   'r_textrank': [1]*len(z)})

from functools import reduce
v = reduce(lambda left,right: pd.merge(left,right,on='phrases', how='outer'), [summa_df, gensim_df, pytextrank_df, textrank_df])

v = v.fillna('0')

v = v.set_index('phrases')

for x in v:
  v[x] = pd.to_numeric(v[x])

import seaborn as sns 
import matplotlib.pyplot as plt
 
 
g=sns.clustermap(v)

g.savefig("output.png")

plt.show()

k <- py$v

kable(k)

sumy

from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals

from sumy.parsers.html import HtmlParser
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.lsa import LsaSummarizer as Summarizer
from sumy.nlp.stemmers import Stemmer
from sumy.utils import get_stop_words

def sumy_function(url, SENTENCES_COUNT = 9, LANGUAGE = "english"):
  
  parser = HtmlParser.from_url(url, Tokenizer(LANGUAGE))
  # or for plain text files
  # parser = PlaintextParser.from_file("document.txt", Tokenizer(LANGUAGE))
  # parser = PlaintextParser.from_string("Check this out.", Tokenizer(LANGUAGE))
  stemmer = Stemmer(LANGUAGE)
  
  summarizer = Summarizer(stemmer)
  summarizer.stop_words = get_stop_words(LANGUAGE)
  
  for sentence in summarizer(parser.document, SENTENCES_COUNT):
      print(sentence)
      print('\n')
      
sumy_function("https://ca.indeed.com/viewjob?cmp=Slice-Insurance-Technologies-Inc.&t=Data+Scientist&jk=35c9914ee9b0a052&sjdu=vQIlM60yK_PwYat7ToXhk3myaatk1OsAkhQIw2UEit1eiAfw8fAJl6BKuZ5V5P0TgQjo7h2qNGvAZ_80Lr3XxA&tk=1e1nuvuml0np3000&adid=335410488&pub=4a1b367933fd867b19b072952f68dceb&vjs=3")

## We’re building a team of innovators and disruptors to change the insurance experience to meet the demands of an on-demand world.
## 
## 
## At Slice, we’re focused on ensuring our products provide a positive, individualized customer experience.
## 
## 
## Our smaller teams invite broader thinking and problem solving, where nobody is pigeonholed into a predetermined role.
## 
## 
## We work in an open, supportive, environment that values and promotes inclusiveness, innovation, and collaboration.
## 
## 
## Our work improves the customer experience, grows our market share and drives business outcomes.
## 
## 
## You are a hands-on learner and are excited by the thought of moving past theory and examples to real-time data science work.
## 
## 
## 4+ years of practical experience using data, models, and common sense to solve tough problems in a collaborative environment.
## 
## 
## An understanding of statistical and predictive modeling concepts, machine learning algorithms, clustering and classification techniques.
## 
## 
## University degree in engineering, applied statistics, data mining, machine learning, mathematics or a related quantitative discipline.