# Load required libraries

library(tm)
library(RTextTools)
library(e1071)
library(dplyr)
library(caret)
# Library for parallel processing
library(doMC)
registerDoMC(cores=detectCores())  # Use all available cores

setwd("/Users/zahra/Dropbox/0Publications/2017/RE17 Data Track/Iteration2/Naive Baysen")

df<- read.csv("NFR.csv", stringsAsFactors = FALSE)
set.seed(1)
df <- df[sample(nrow(df)), ]
df <- df[sample(nrow(df)), ]
glimpse(df)

# Convert the 'class' variable from character to factor.
df$class <- as.factor(df$class)

corpus <- Corpus(VectorSource(df$text))

corpus <- tm_map(corpus,
               content_transformer(function(x) iconv(x, to='UTF-8-MAC', sub='byte')),
               mc.cores=1)

#start preprocessing
#Transform to lower case
corpus <-tm_map(corpus,content_transformer(tolower))


#remove potentially problematic symbols
toSpace <- content_transformer(function(x, pattern) { return (gsub(pattern, ' ', x))})
toReplace <- content_transformer(function(x, y, pattern) { return (gsub(pattern, y, x))})

#remove HTML formatting and tags
corpus <- tm_map(corpus, toSpace, "<.*?>")
#remove punctuation
corpus <- tm_map(corpus, removeWords, c("user", "id", "requirements", "part", "movie", "software", "the", "shall", "every", "shall", "event", "activity", "on", "must", "be", "able", "can",
                                    "MDI", "events", "of", "projected", "distance", "contacts", "reports", "appointments", "criteria",
                                    "able", "cma", "report", "internet", "connection", "device", "button", "buttons", "navigation", "search",
                                    "classes", "clinicals", "final", "class", "scheduling", "statement", "database", "transaction", "disputes", "letters", "letter",
                                    "user reports", "standards", "standard", "credit card", "dispute", "navigate", "navigating", "screen", "cases", "dbms",
                                    "located", "machine", "cycle", "accounting", "claims", "repair facility", "language", "java runtime", "html browser", "email",
                                    "user ids", "reservations", "conference", "rooms", "room", "building maps", "design patterns", "coding practice", "training",
                                    "animation", "home", "sales", "prepaid", "pre-paid", "card", "administration", "pin number", "email", "website", "pear", "pins",
                                    "excel", "stream", "buffer", "velkat staff", "lead", "cycle time", "evaluation", "leads", "ship", "sink", "sunk", "shot",
                                    "hit", "target", "miss", "sea", "offensive", "grid", "african", "guideline", "response", "advertise", "empty time slots", "ensure", "that",
                                    "browser", "depaul teleservices center", "responce", "messages", "message", "between", "csr", "eam", "ibm", "computer", "operating system",
                                    "windows", "rfs system", "bdw", "budgeting process", "city", "chicago", "application", "development", "guide", "framework", "monitoring",
                                    "services", "monitoring", "error", "tracking", "united states", "ccr", "wcs", "branding","logo", "resolution", "navigation", "mouse",
                                    "keyboard", "developer", "queries", "w3", "league", "button", "nfl season", "network", "player", "architecture", "hard", "disk", "space",
                                    "pcg", "hierarchy", "roles", "enet", "budget", "budgeting", "revenue", "expense", "formula", "pcg", "finance", "time", "part", "without",  
                                    "survey", "via", "market", "file", "week", "day", "month", "one", "system", "product", "user", "Product", "intranet", "information", "User", "role", "roles",
                                    "can", "projectidrequirementsclassid", "requirements","class", "ID", "within", "file", "let", "provide", "detail", "number", "will", "year"))



corpus <- tm_map(corpus, removePunctuation)
#Strip digits
corpus <- tm_map(corpus, removeNumbers)
#remove stopwords
corpus <- tm_map(corpus, removeWords, stopwords("english"))
#remove whitespace
corpus <- tm_map(corpus, stripWhitespace)
# pre stem replace words
#corpus <- tm_map(corpus, toReplace, ' ffunctional ', " functional ")
#Stem document
corpus <- tm_map(corpus, toReplace, 'easy', "easily")
corpus <- tm_map(corpus,stemDocument)
#post stem replace

# corpus <- tm_map(corpus, toReplace, 'implement', "build")
# corpus <- tm_map(corpus, toReplace, 'implement', "creat")
# corpus <- tm_map(corpus, toReplace, 'model', "user stori")
# corpus <- tm_map(corpus, toReplace, 'model', "use case")

corpus <- tm_map(corpus, removeWords, c("user", "case", "year", "data", "day", "per", "customer", "hour", "busi", "store"))
corpus.clean <- corpus

dtm <- DocumentTermMatrix(corpus.clean)
inspect(dtm[40:50, 10:15])

df.train <- df[1:150,]
df.test <- df[151:370,]

dtm.train <- dtm[1:150,]
dtm.test <- dtm[151:370,]

corpus.clean.train <- corpus.clean[1:150]
corpus.clean.test <- corpus.clean[151:370]

dim(dtm.train)

fivefreq <- findFreqTerms(dtm.train, 8)
length((fivefreq))

dtm.train.nb <- DocumentTermMatrix(corpus.clean.train, control=list(dictionary = fivefreq))

dim(dtm.train.nb)

dtm.test.nb <- DocumentTermMatrix(corpus.clean.test, control=list(dictionary = fivefreq))

dim(dtm.train.nb)

# Function to convert the word frequencies to yes (presence) and no (absence) labels
convert_count <- function(x) {
  y <- ifelse(x > 0, 1,0)
  y <- factor(y, levels=c(0,1), labels=c("No", "Yes"))
  y
}

# Apply the convert_count function to get final training and testing DTMs
trainNB <- apply(dtm.train.nb, 2, convert_count)
testNB <- apply(dtm.test.nb, 2, convert_count)


# Train the classifier
system.time( classifier <- naiveBayes(trainNB, df.train$class, laplace = 1) )

# Use the NB classifier we built to make predictions on the test set.
system.time( pred <- predict(classifier, newdata=testNB) )

table("Predictions"= pred,  "Actual" = df.test$class )


conf.mat <- confusionMatrix(pred, df.test$class)

conf.mat