In-class_EX05

pacman::p_load(tidyverse, readtext,
               quanteda, tidytext)

data_folder <- "MC1/articles"

text_data <- readtext(paste0("MC1/articles",
                "/*"))

text_data <- readtext("MC1/articles")

usenet_words <- text_data %>%
  unnest_tokens(word, text) %>%  #reading the text data
  filter(str_detect(word, "[a-z']$"),
         !word %in% stop_words$word) #remove stop words
usenet_words %>%
  count(word, sort = TRUE)

readtext object consisting of 3260 documents and 0 docvars.
# A data frame: 3,260 × 3
  word             n text     
  <chr>        <int> <chr>    
1 fishing       2177 "\"\"..."
2 sustainable   1525 "\"\"..."
3 company       1036 "\"\"..."
4 practices      838 "\"\"..."
5 industry       715 "\"\"..."
6 transactions   696 "\"\"..."
# ℹ 3,254 more rows

temp_table <- usenet_words %>%
  count(word, sort = TRUE)

corpus_text <- corpus(text_data)
summary(corpus_text, 5)

Corpus consisting of 338 documents, showing 5 documents:

                                   Text Types Tokens Sentences
 Alvarez PLC__0__0__Haacklee Herald.txt   206    433        18
    Alvarez PLC__0__0__Lomark Daily.txt   102    170        12
   Alvarez PLC__0__0__The News Buoy.txt    90    200         9
 Alvarez PLC__0__1__Haacklee Herald.txt    96    187         8
    Alvarez PLC__0__1__Lomark Daily.txt   241    504        21

text_data_splitted <- text_data %>%
  separate_wider_delim("doc_id",
                       delim = "__0__",
                       names = c("X", "Y"),
                       too_few = "align_end")

pacman::p_load(jsonlite, tidyverse)

mc1_data <- fromJSON("MC1/mc1.json")