« home

Dropout

Creator: Petar Veličković (original)

machine learningneural networksregularizationcetztikz

Illustration of applying dropout with a rate of $p = 0.5$ to a multilayer perceptron.

Download

PNG PDF SVG

Code

LaTeX

dropout.tex (71 lines)

\documentclass[tikz]{standalone}

\usetikzlibrary{positioning}

\def\layersep{2}
\def\nodesep{1.5}

\begin{document}
\begin{tikzpicture}[
    node/.style={circle, draw, thick},
  ]

  \foreach \y in {1,...,5}{
      \node[node] (i\y) at (0,\nodesep*\y) {};
      \node[node, right=\layersep of i\y] (h1\y) {};
      \node[node, right=\layersep of h1\y] (h2\y) {};
    }

  \node[node, right=\layersep of h22] (o1) {};
  \node[node, right=\layersep of h24] (o2) {};

  \foreach \source in {1,...,5}
  \foreach \dest in {1,...,5}{
      \path[-stealth, thick] (i\source) edge (h1\dest);
      \path[-stealth, thick] (h1\source) edge (h2\dest);
    }
  \foreach \source in {1,...,5}
  \foreach \dest in {1,2}
  \draw[-stealth, thick] (h2\source) -- (o\dest);

  \draw[-stealth, thick] (7.5,3*\nodesep) -- node[above,font=\Large\bfseries] {dropout} (9.5, 3*\nodesep);

  % Boundary

  \foreach \y in {1,...,5}
  \node[node, right=15em of h2\y] (di\y) {};

  \node[red,font=\huge] at (di1) {$\times$};
  \node[red,font=\huge] at (di3) {$\times$};

  \foreach \y in {1,...,5}
  \node[node, right=\layersep of di\y] (dh1\y) {};

  \node[red,font=\huge] at (dh11) {$\times$};
  \node[red,font=\huge] at (dh13) {$\times$};
  \node[red,font=\huge] at (dh14) {$\times$};

  \foreach \y in {1,...,5}
  \node[node, right=\layersep of dh1\y] (dh2\y) {};

  \node[red,font=\huge] at (dh22) {$\times$};
  \node[red,font=\huge] at (dh24) {$\times$};

  \node[node, right=\layersep of dh22] (do1) {};
  \node[node, right=\layersep of dh24] (do2) {};

  \foreach \source in {2,4,5}
  \foreach \dest in {2,5}
  \draw[-stealth, thick] (di\source) -- (dh1\dest);

  \foreach \source in {2,5}
  \foreach \dest in {1,3,5}
  \draw[-stealth, thick] (dh1\source) -- (dh2\dest);

  \foreach \source in {1,3,5}
  \foreach \dest in {1,2}
  \draw[-stealth, thick] (dh2\source) -- (do\dest);

\end{tikzpicture}
\end{document}

Typst

dropout.typ (112 lines)

#import "@preview/cetz:0.3.2": canvas, draw

#set page(width: auto, height: auto, margin: 8pt)

#canvas({
  import draw: line, circle, content

  let node-style = (stroke: black + 1pt, fill: white)

  let layer-sep = 2.5 // Horizontal separation between layers
  let node-sep = 1.5 // Vertical separation between nodes
  let arrow-style = (stroke: black + 1pt, mark: (end: "stealth"), fill: black)

  // Helper function to draw a layer of nodes
  let draw-layer(x, nodes, prefix: "") = {
    for ii in range(nodes) {
      circle(
        (x, node-sep * (ii + 1)),
        radius: 0.3,
        name: prefix + str(ii + 1),
        ..node-style,
      )
    }
  }

  // Helper to connect all nodes between layers
  let connect-layers(from-prefix, to-prefix, from-nodes, to-nodes) = {
    for ii in range(from-nodes) {
      for jj in range(to-nodes) {
        line(
          (from-prefix + str(ii + 1)),
          (to-prefix + str(jj + 1)),
          ..arrow-style,
        )
      }
    }
  }

  // Left network (fully connected)
  // Draw all layers
  draw-layer(0, 5, prefix: "i") // Input layer
  draw-layer(layer-sep, 5, prefix: "h1") // First hidden layer
  draw-layer(2 * layer-sep, 5, prefix: "h2") // Second hidden layer

  // Draw output nodes
  circle((3 * layer-sep, 2 * node-sep), radius: 0.3, name: "o1", ..node-style)
  circle((3 * layer-sep, 4 * node-sep), radius: 0.3, name: "o2", ..node-style)

  // Connect all layers
  connect-layers("i", "h1", 5, 5)
  connect-layers("h1", "h2", 5, 5)

  // Connect to output nodes
  for ii in range(5) {
    line(("h2" + str(ii + 1)), "o1", ..arrow-style)
    line(("h2" + str(ii + 1)), "o2", ..arrow-style)
  }

  // Draw dropout arrow
  let mid-x = 4 * layer-sep
  line(
    (3.5 * layer-sep, 3 * node-sep),
    (4.5 * layer-sep, 3 * node-sep),
    ..arrow-style,
    name: "dropout-arrow",
  )
  content(
    "dropout-arrow.mid",
    text(weight: "bold", size: 1.2em)[dropout],
    anchor: "south",
    padding: 3pt,
  )

  // Right network (with dropout)
  // Draw all layers
  draw-layer(mid-x + layer-sep, 5, prefix: "di")
  draw-layer(mid-x + 2 * layer-sep, 5, prefix: "dh1")
  draw-layer(mid-x + 3 * layer-sep, 5, prefix: "dh2")

  // Draw output nodes
  circle((mid-x + 4 * layer-sep, 2 * node-sep), radius: 0.3, name: "do1", ..node-style)
  circle((mid-x + 4 * layer-sep, 4 * node-sep), radius: 0.3, name: "do2", ..node-style)

  // Add dropout X marks
  let x-style = (fill: red, weight: "bold", size: 4em, baseline: -4pt)
  content("di1", text(..x-style)[×])
  content("di3", text(..x-style)[×])
  content("dh11", text(..x-style)[×])
  content("dh13", text(..x-style)[×])
  content("dh14", text(..x-style)[×])
  content("dh22", text(..x-style)[×])
  content("dh24", text(..x-style)[×])

  // Connect remaining nodes (after dropout)
  for ii in (2, 4, 5) {
    for jj in (2, 5) {
      line(("di" + str(ii)), ("dh1" + str(jj)), ..arrow-style)
    }
  }

  for ii in (2, 5) {
    for jj in (1, 3, 5) {
      line(("dh1" + str(ii)), ("dh2" + str(jj)), ..arrow-style)
    }
  }

  for ii in (1, 3, 5) {
    line(("dh2" + str(ii)), "do1", ..arrow-style)
    line(("dh2" + str(ii)), "do2", ..arrow-style)
  }
})