#load files into corpus #get listing of .txt files in directory #names<-removeExt(filenames) Line not used #read files into a character vector #load text mining library library(tm) library(SnowballC) library(wordcloud) #set working directory setwd("/Users/zahra/Dropbox/0Publications/2017/RE'17 Data Track/Iteration2"); #load files into corpus #get listing of .txt files in directory filenames <- list.files(getwd(),pattern="*.txt") #read files into a character vector files <- lapply(filenames,readLines) #create corpus from vector docs <- Corpus(VectorSource(files)) docs <- tm_map(docs, content_transformer(function(x) iconv(x, to='UTF-8-MAC', sub='byte')), mc.cores=1) #start preprocessing #Transform to lower case docs <-tm_map(docs,content_transformer(tolower)) #remove potentially problematic symbols toSpace <- content_transformer(function(x, pattern) { return (gsub(pattern, ' ', x))}) toReplace <- content_transformer(function(x, y, pattern) { return (gsub(pattern, y, x))}) #remove HTML formatting and tags docs <- tm_map(docs, toSpace, "<.*?>") #remove punctuation docs <- tm_map(docs, removeWords, c("user", "part", "movie", "software", "the", "shall", "every", "shall", "event", "activity", "on", "must", "be", "able", "can", "MDI", "events", "of", "projected", "distance", "contacts", "reports", "appointments", "criteria", "able", "cma", "report", "internet", "connection", "device", "button", "buttons", "navigation", "search", "classes", "clinicals", "final", "class", "scheduling", "statement", "database", "transaction", "disputes", "letters", "letter", "user reports", "standards", "standard", "credit card", "dispute", "navigate", "navigating", "screen", "cases", "dbms", "located", "machine", "cycle", "accounting", "claims", "repair facility", "language", "java runtime", "html browser", "email", "user ids", "reservations", "conference", "rooms", "room", "building maps", "design patterns", "coding practice", "training", "animation", "home", "sales", "prepaid", "pre-paid", "card", "administration", "pin number", "email", "website", "pear", "pins", "excel", "stream", "buffer", "velkat staff", "lead", "cycle time", "evaluation", "leads", "ship", "sink", "sunk", "shot", "hit", "target", "miss", "sea", "offensive", "grid", "african", "guideline", "response", "advertise", "empty time slots", "ensure", "that", "browser", "depaul teleservices center", "responce", "messages", "message", "between", "csr", "eam", "ibm", "computer", "operating system", "windows", "rfs system", "bdw", "budgeting process", "city", "chicago", "application", "development", "guide", "framework", "monitoring", "services", "monitoring", "error", "tracking", "united states", "ccr", "wcs", "branding","logo", "resolution", "navigation", "mouse", "keyboard", "developer", "queries", "w3", "league", "button", "nfl season", "network", "player", "architecture", "hard", "disk", "space", "pcg", "hierarchy", "roles", "enet", "budget", "budgeting", "revenue", "expense", "formula", "pcg", "finance", "time", "part", "without", "survey", "via", "market", "file", "week", "day", "month", "one", "system", "product", "user", "Product", "intranet", "information", "User", "role", "roles", "can", "projectidrequirementsclassid", "requirements","class", "ID", "within", "file", "let", "provide", "detail", "number", "will", "year")) docs <- tm_map(docs, removePunctuation) #Strip digits docs <- tm_map(docs, removeNumbers) #remove stopwords docs <- tm_map(docs, removeWords, stopwords("english")) #remove whitespace docs <- tm_map(docs, stripWhitespace) # pre stem replace words #docs <- tm_map(docs, toReplace, ' ffunctional ', " functional ") #Stem document docs <- tm_map(docs, toReplace, 'easy', "easily") docs <- tm_map(docs,stemDocument) #post stem replace # docs <- tm_map(docs, toReplace, 'implement', "build") # docs <- tm_map(docs, toReplace, 'implement', "creat") # docs <- tm_map(docs, toReplace, 'model', "user stori") # docs <- tm_map(docs, toReplace, 'model', "use case") docs <- tm_map(docs, removeWords, c("case", "year", "data", "day", "per", "customer", "hour", "busi", "store")) #Create document-term matrix dtm <- DocumentTermMatrix(docs) #convert rownames to filenames rownames(dtm) <- filenames #collapse matrix by summing over columns freq <- colSums(as.matrix(dtm)) #length should be total number of terms length(freq) #create sort order (descending) ord <- order(freq,decreasing=TRUE) #List all terms in decreasing order of freq and write to disk wordcloud(docs, max.words = 200, random.order = FALSE) write.csv(freq[ord],"word_freq.csv") library(topicmodels) #Set parameters for Gibbs sampling burnin <- 400 iter <- 200 thin <- 50 seed <-list(2003,7,63,100001,765, 500,500,500,500,500) nstart <- 10 best <- TRUE #Number of topics k <- 10 #Run LDA using Gibbs sampling ldaOut <-LDA(dtm,k, method="Gibbs", control=list(nstart=nstart, seed = seed, best=best, burnin = burnin, iter = iter, thin=thin)) #write out results #docs to topics ldaOut.topics <- as.matrix(topics(ldaOut)) write.csv(ldaOut.topics,file=paste('LDAGibbs',k,"DocsToTopics.csv")) #top 6 terms in each topic ldaOut.terms <- as.matrix(terms(ldaOut,5)) write.csv(ldaOut.terms,file=paste("LDAGibbs",k,"TopicsToTerms.csv")) #probabilities associated with each topic assignment topicProbabilities <- as.data.frame(ldaOut@gamma) write.csv(topicProbabilities,file=paste("LDAGibbs",k,"TopicProbabilities.csv")) #Find relative importance of top 2 topics topic1ToTopic2 <- lapply(1:nrow(dtm),function(x) sort(topicProbabilities[x,])[k]/sort(topicProbabilities[x,])[k-1]) #Find relative importance of second and third most important topics topic2ToTopic3 <- lapply(1:nrow(dtm),function(x) sort(topicProbabilities[x,])[k-1]/sort(topicProbabilities[x,])[k-2]) #write to file write.csv(topic1ToTopic2,file=paste("LDAGibbs",k,"Topic1ToTopic2.csv")) write.csv(topic2ToTopic3,file=paste("LDAGibbs",k,"Topic2ToTopic3.csv"))