#load files into corpus
#get listing of .txt files in directory
#names<-removeExt(filenames) Line not used
#read files into a character vector
#load text mining library
library(tm)
library(SnowballC)
library(wordcloud)

setwd("/Users/zahra/Dropbox/0Publications/2017/RE17 Data Track/Oli");
getwd()

dataValues <- read.csv("CleanedOutput.csv")
#dim(dataValues)
## Text Pre-processing.
## Creating a Corpus from the Orginal Function
## interprets each element of the vector x as a document
CorpusObj<- VectorSource(dataValues$Requirements);
CorpusObj<-Corpus(CorpusObj);

f <- content_transformer(function(x, pattern) 
  regmatches(x, gregexpr(pattern, x, perl=TRUE, ignore.case=TRUE)))

keep = "CC|CD|DT|EX|FW|IN|JJ|JJR|JJS|LS|MD|NN|NNS|NNP|NNPS|PDT|POS|PRP|PRP|RB|RBR|RBS|RP|SYM|TO|UH|VB|VBD|VBG|VBN|VBP|VBZ|WDT|WP|WP|WRB|NNS"
CorpusObj<-tm_map(CorpusObj, f, keep)

CorpusObj <- tm_map(CorpusObj, PlainTextDocument)
#save in indiv text file
writeCorpus(CorpusObj)
#write 1 file
tmp <- CorpusObj[1]

dataframe<-data.frame(text=unlist(sapply(CorpusObj, `[`, "content")), stringsAsFactors=F)
write.csv(dataframe, "CleanedOutputt.csv")