#########################
#Nonparametric Statistics
#
#Cochran's Q-Test


library(RVAideMemoire)


#####################################################################
#Basketball Example

game.prediction=c(1,1,1,
                  1,1,1,
                  0,1,0,
                  1,1,0,
                  0,0,0,
                  1,1,1,
                  1,1,1,
                  1,1,0,
                  0,0,1,
                  0,1,0,
                  1,1,1,
                  1,1,1)

fan=gl(3,1,36,labels=1:3)
block=gl(12,3,labels=c(1:12))

cbind(game.prediction,fan,block)

cochran.qtest(game.prediction~fan|block)




###################################################################
#Machine Learning Example


#Iris Data
head(iris)
table(iris$Species)
plot(iris[,1:4],col=iris$Species)


#Folds for 10-fold CV
createfolds=function(n,K){
  reps=ceiling(n/K)
  folds=sample(rep(1:K,reps))
  return(folds[1:n])
}

set.seed(5301)
folds=createfolds(nrow(iris),10)

table(folds)


#############################################
#Predictions for Random Forest, SVM, and k-NN

#Loading libraries
library(randomForest)
library(e1071)
library(class)

#Setting up prediction vectors
pred.rf=iris$Species
pred.svm=iris$Species
pred.knn=iris$Species



#10-fold CV
for(i in 1:10){
  train=(folds!=i)
  test=(folds==i)
  
  rf.model=randomForest(Species~.,data=iris[train,])
  svm.model=svm(Species~.,data=iris[train,])
  
  pred.rf[test]=predict(rf.model,newdata=iris[test,])
  pred.svm[test]=predict(svm.model,newdata=iris[test,])
  pred.knn[test]=knn(iris[train,1:4],iris[test,1:4],cl=iris$Species[train])
}


#Confusion Matrices
confmatrix=function(y,predy){
  matrix=table(y,predy)
  accuracy=sum(diag(matrix))/sum(matrix)
  return(list(matrix=matrix,accuracy=accuracy,error=1-accuracy))
}

confmatrix(iris$Species,pred.rf)
confmatrix(iris$Species,pred.svm)
confmatrix(iris$Species,pred.knn)


#Accuracy Vectors

acc.rf=(pred.rf==iris$Species)
acc.svm=(pred.svm==iris$Species)
acc.knn=(pred.knn==iris$Species)

table(acc.rf)
table(acc.svm)
table(acc.knn)



#Cochran's Q-test

prediction=c(acc.rf,
             acc.svm,
             acc.knn)

algorithm=c(rep(1,150),
            rep(2,150),
            rep(3,150))

block=rep(1:150,3)

cbind(prediction,algorithm,block)  


cochran.qtest(prediction~algorithm|block)


#Pairwise McNemar Tests

#RF vs SVM
table(acc.rf,acc.svm)
mcnemar.test(table(acc.rf,acc.svm))


#RF vs knn
table(acc.rf,acc.knn)
mcnemar.test(table(acc.rf,acc.knn))


#SVM vs knn
table(acc.svm,acc.knn)
mcnemar.test(table(acc.svm,acc.knn))