hypelib / hype Goto Github PK

let func = fun w x -> 
                    l.Decode w
                    l.Run x
        let w0 = l.Encode()
        let w, loss, _, lhist = Optimize.Train(func, w0, dataset, valid, par)

type Activation = 
        |Sigm
        |Softmax
        |Linear
        |Tanh

        member this.funDM =
            match this with
            |Sigm -> DM.Sigmoid
            |Softmax -> DM.mapCols DV.SoftMax
            |Tanh -> DM.Tanh
            |Linear -> id

        member this.funDV =
            match this with
            |Sigm -> DV.Sigmoid
            |Softmax -> DV.SoftMax
            |Tanh -> DV.Tanh
            |Linear -> id

let inline ( *+) dv f = DV.Append(dv, toDV [|f|])

/// i -> number of inputs  h -> number of hidden units
type AutoEncoder(i,h,a:Activation) =

    // Hepler functions
    let removeBias (X:DM) = X.[0..X.Rows-2, *]
    let replaceBias (X:DM) = X.[0..X.Rows-2, *] |> DM.appendRow (DV.create X.Cols 1)
    let appendBiasDM (X:DM) = DM.appendRow (DV.create X.Cols 1) X

    /// Weights W:DM . Since it has tied weights both layers use the same weights one transpose of the other.
    member val W = Rnd.NormalDM(h+1, i+1, D 0.f, D 0.1f) with get, set

    /// Flattened weights W':DV
    member this.W' with get() = DM.toDV this.W and set W' = this.W <- DV.toDM (h+1) W'

    /// Forward propagate the data
    member this.RunDM (X:DM) = let h = X |> appendBiasDM |> (*) this.W |> replaceBias
                               (DM.Transpose this.W) * h |> removeBias

    /// Forward propagation when W' provided by optimization algorithm
    member this.Run (W':DV) (X:DM) = this.W' <- W'
                                     this.RunDM X

    /// Encode data and get hidden unit
    member this.Encode (X:DM) = X |> appendBiasDM |> (*) this.W |> replaceBias

// TEST
let ae = AutoEncoder(3,2, Activation.Sigm)
let p = {Params.Default with Regularization = NoReg; Loss = Loss.Quadratic }
let X' = ((toDM [[1.f;5.f;2.f];[8.f;2.f;2.f];[1.f;5.f;2.f];[8.f;2.f;2.f];
                [1.1f;5.2f;2.f];[8.1f;2.1f;2.f];[0.9f;4.9f;2.f];[7.9f;1.9f;2.f]])) / 10 |> DM.Transpose

let ds' = Dataset(X', X'.Copy())
let a,b,_,_ = Optimize.Train(ae.Run , ae.W', ds', p)

module Array2D =  
    let copyFast (array : 'T[,]) =  array.Clone() :?> 'T[,]

#time "on"

let test1() = 
    let mutable res  = Array2D.zeroCreate<float32> 100 100 
    for i in 0 .. 1000 do   
        for j in 0 .. 100 do
            res <- Array2D.copy res

let test2() = 
    let mutable res  = Array2D.zeroCreate<float32> 100 100 
    for i in 0 .. 1000 do   
        for j in 0 .. 100 do
            res <- Array2D.copyFast res

test1()  // 4.4s
test2() // 0.98s

match r with
        | L1Reg l -> fun (w:DV) -> l * (DV.l1norm w)
        | L2Reg l -> fun w -> l * (DV.l2normSq w)
        | NoReg -> fun _ -> D 0.f

| Constant    of D         // Constant
| Decay       of D * D     // 1 / t decay, a = a0 / (1 + kt). Initial value, decay rate
| ExpDecay    of D * D     // Exponential decay, a = a0 * Exp(-kt). Initial value, decay rate

/// Constant learning rate
| Constant    of D         

/// Learning rate of 1 / t decay, a = a0 / (1 + kt). Initial value, decay rate
| Decay       of D * D    

/// Exponential decay learning rate, a = a0 * Exp(-kt). Initial value, decay rate
| ExpDecay    of D * D