
plot.single.effort.ranking        <- function(filename,  feature.group.names=c("B", "B+C"), model="lr.2phase", rule="aic", direction="forward", remove.flag=FALSE)
{
	filename <- basename(filename)
	if (!file.exists("plot")) { dir.create("plot") }
	data.CE <- data.ER <- NULL
	for (feature.group.name in feature.group.names) {
		fout <- sprintf("output/file(%s)_model(%s)_rule(%s)_direction(%s)_remove(%s)_%s.out", filename, model, rule, direction, remove.flag, feature.group.name)
		if (!file.exists(fout)) {
			cat(fout, " is not exists.\n")
			return(NULL)
		}

		out <- dget(file=fout)
		data.CE[[feature.group.name]] <- out[["divide.effort"]][["CE"]]
		data.ER[[feature.group.name]] <- out[["divide.effort"]][["ER"]]
	}

	T.name <- feature.group.names[1]
	N.name <- feature.group.names[2]

	T <- data.CE[[T.name]][, "CE.E.0.2"]
	N <- data.CE[[N.name]][, "CE.E.0.2"]
	data.all <- cbind(T, N)
	
	PV <- wilcox.test(T, N, paired=TRUE)$"p.value"
	CD <- effsize::cliff.delta(T, N)$"estimate"
	
	scols <- rep("black", 2)
	if ((PV <= 0.05) && (CD >=  0.147)) { scols[2] <- "red"  }
	if ((PV <= 0.05) && (CD <= -0.147)) { scols[2] <- "blue" }

	pout <- sprintf("plot/file(%s)_model(%s)_rule(%s)_direction(%s)_remove(%s)_effort.CE.png", filename, model, rule, direction, remove.flag)
	png(file=pout, width=0.6, height=0.8, units="in", res=300)
	par(mar=c(0, 0, 0, 0), oma=c(0.8, 1.0, 0.6, 0.1), cex=0.6)

	boxplot(data.all, xlab="", ylab="", xaxt="n", boxwex=0.4, frame=FALSE, outline=FALSE, border=scols, axes=FALSE); box(bty="L", mgp=c(1, 1, 0))
	axis(2, mgp=c(0.05, 0.12, 0), tck=-0.02, cex.axis=1)
	text(x=c(1, 2), y=par("usr")[3], labels=c("B", "B+C"), cex=0.8, adj=c(0.5, 1.5), xpd=NA)

	dev.off()
}

plot.single.effort.classification <- function(filename,  feature.group.names=c("B", "B+C"), model="lr.2phase", rule="aic", direction="forward", remove.flag=FALSE)
{
	filename <- basename(filename)
	if (!file.exists("plot")) { dir.create("plot") }
	
	data.CE <- data.ER <- data.AUC <- NULL
	for (feature.group.name in feature.group.names) {
		fout <- sprintf("output/file(%s)_model(%s)_rule(%s)_direction(%s)_remove(%s)_%s.out", filename, model, rule, direction, remove.flag, feature.group.name)
		if (!file.exists(fout)) {
			cat(fout, " is not exists.\n")
			return(NULL)
		}

		out <- dget(file=fout)
		data.CE[[feature.group.name]] <- out[["divide.effort"]][["CE"]]
		data.ER[[feature.group.name]] <- out[["divide.effort"]][["ER"]]
	}

	T.name <- feature.group.names[1]
	N.name <- feature.group.names[2]

	T <- data.ER[[T.name]][, "ER.P.0.2"]
	N <- data.ER[[N.name]][, "ER.P.0.2"]
	data.all <- cbind(T, N)
	PV <- wilcox.test(T, N, paired=TRUE)$"p.value"
	CD <- effsize::cliff.delta(T, N)$"estimate"
	
	scols <- rep("black", 2)
	if ((PV <= 0.05) && (CD >=  0.147)) { scols[2] <- "red"  }
	if ((PV <= 0.05) && (CD <= -0.147)) { scols[2] <- "blue" }

	pout <- sprintf("plot/file(%s)_model(%s)_rule(%s)_direction(%s)_remove(%s)_effort.ERA.png", filename, model, rule, direction, remove.flag)
	png(file=pout, width=0.6, height=0.8, units="in", res=300)
	par(mar=c(0, 0, 0, 0), oma=c(0.8, 1.0, 0.6, 0.1), cex=0.6)

	boxplot(data.all, xlab="", ylab="", xaxt="n", boxwex=0.4, frame=FALSE, outline=FALSE, border=scols, axes=FALSE); box(bty="L")

	axis(2, mgp=c(0.05, 0.12, 0), tck=-0.02, cex.axis=1)
	text(x=c(1, 2), y=par("usr")[3], labels=c("B", "B+C"), cex=0.8, adj=c(0.5, 1.5), xpd=NA)

	dev.off()
}

table.effort.ranking              <- function(filenames, feature.group.names=c("B", "B+C"), model="lr.2phase", rule="aic", direction="forward", remove.flag=FALSE)
{
	data.res <- NULL
	for (filename in filenames) {
		filename <- basename(filename)
		if (!file.exists("plot")) { dir.create("plot") }
		data <- NULL
		for (feature.group.name in feature.group.names) {
			fout <- sprintf("output/file(%s)_model(%s)_rule(%s)_direction(%s)_remove(%s)_%s.out", filename, model, rule, direction, remove.flag, feature.group.name)
			if (!file.exists(fout)) {
				cat(fout, " is not exists.\n")
				return(NULL)
			}

			out <- dget(file=fout)
			data[[feature.group.name]] <- out[["divide.effort"]][["CE"]]
		}

		T.name <- feature.group.names[1]
		N.name <- feature.group.names[2]

		T <- data[[T.name]][, "CE.E.0.2"]
		N <- data[[N.name]][, "CE.E.0.2"]
		data.all <- cbind(T, N)

		PV <- wilcox.test(T, N, paired=TRUE)$"p.value"
		CD <- effsize::cliff.delta(T, N)$"estimate"

		scols <- rep("black", 2)
		if ((PV <= 0.05) && (CD >=  0.147)) { scols[2] <- "red"  }
		if ((PV <= 0.05) && (CD <= -0.147)) { scols[2] <- "blue" }

		arow <- apply(data.all, 2, median)
		data.res <- rbind(data.res, c(arow, PV, CD))
	}

	data.res <- as.data.frame(data.res)
	colnames(data.res) <- c("T", "N", "wp", "CliffD")
	rownames(data.res) <- basename(filenames)

	data.res$improve <- (data.res[, "N"] - data.res[, "T"])/data.res[, "T"]

	data.res <- data.res[, c("T", "N", "improve","CliffD",  "wp")]
	data.res <- rbind(data.res, colMeans(data.res))
	write.csv(round(data.res, 3), file="CE-0.2.csv")
}

table.effort.classification       <- function(filenames, feature.group.names=c("B", "B+C"), model="lr.2phase", rule="aic", direction="forward", remove.flag=FALSE)
{
	data.res <- NULL
	for (filename in filenames) {
		filename <- basename(filename)
		if (!file.exists("plot")) { dir.create("plot") }
		data <- NULL
		for (feature.group.name in feature.group.names) {
			fout <- sprintf("output/file(%s)_model(%s)_rule(%s)_direction(%s)_remove(%s)_%s.out", filename, model, rule, direction, remove.flag, feature.group.name)
			if (!file.exists(fout)) {
				cat(fout, " is not exists.\n")
				return(NULL)
			}

			out <- dget(file=fout)
			data[[feature.group.name]] <- out[["divide.effort"]][["ER"]]
		}

		T.name <- feature.group.names[1]
		N.name <- feature.group.names[2]
		
		T <- data[[T.name]][, "ER.P.0.2"]
		N <- data[[N.name]][, "ER.P.0.2"]
		data.all <- cbind(T, N)
		
		PV <- wilcox.test(T, N, paired=TRUE)$"p.value"
		CD <- effsize::cliff.delta(T, N)$"estimate"
		
		scols <- rep("black", 2)
		if ((PV <= 0.05) && (CD >=  0.147)) { scols[2] <- "red"  }
		if ((PV <= 0.05) && (CD <= -0.147)) { scols[2] <- "blue" }

		arow <- apply(data.all, 2, mean)
		data.res <- rbind(data.res, c(arow, PV, CD))
	}
	
	data.res <- as.data.frame(data.res)
	colnames(data.res) <- c("T", "N", "wp", "CliffD")
	rownames(data.res) <- basename(filenames)
	
	data.res$improve <- (data.res[, "N"] - data.res[, "T"])/data.res[, "T"]
	data.res <- data.res[, c("T", "N", "improve", "CliffD", "wp")]
	data.res <- rbind(data.res, colMeans(data.res))
	write.csv(round(data.res, 3), file="ER-0.2.csv")
}