philipppro / defaults Goto Github PK

# Inner resampling: cv10
inner.rdesc = cv10

# On train datasets
defaults = computeDefaults() # defaults is a list of 10 configurations

# On test datasets
# Use Nested CV to select a single best default out of the 10 on each dataset:
#  Inner Loop:
# Select a best configuration out of the 10.
#  Outer Loop:
# Get an unbiased estimate of algorithm performance.

lrn.def = makeTuneWrapper(lrn, inner.rdesc, auc, par.set = ps, 
  makeTuneControlDesign(design = defaults))

res.def = evalParsOpenML(tasks[[i]], lrn.def)
    
# Search randomly (2x randomsearch)
lrn.rnd2 = makeTuneWrapper(lrn, inner.rdesc, auc, par.set = ps, 
  makeTuneControlRandom(maxit = 2 * nrow(defaults)))

res.rndx2 = evalParsOpenML(tasks[[i]], lrn.rnd2)

evalParsOpenML = function(task, lrn) {
  run = runTaskMlr(task, lrn, measures = list(auc, f1))
  perf = getBMRAggrPerformances(run$bmr, as.df = TRUE)
  return(perf)
}