« home

Skip Connection

machine learningneural networkscetztikz

Illustration of skip connection in a residual block. Inspired by the ResNet paper. arxiv:1512.03385


Skip Connection

  Download

PNGPDFSVG

  Code

  LaTeX

skip-connection.tex (23 lines)

\documentclass[tikz]{standalone}

\usetikzlibrary{positioning, calc, decorations.pathreplacing}

\begin{document}
\begin{tikzpicture}

  \node[fill=orange!50] (l1) {layer 1};
  \node[blue!50!black, right=of l1, label={below:activation}] (act1) {$a(\vec x)$};
  \node[fill=teal!50, right=of act1] (l2) {layer 2};
  \node[right=of l2, font=\Large, label={below:add}, inner sep=0, pin={60:$\mathcal F(\vec x) + \vec x$}] (add) {$\oplus$};
  \node[blue!50!black, right=of add, label={below:activation}] (act2) {$a(\vec x)$};

  \draw[->] (l1) -- (act1);
  \draw[->] (act1) -- (l2);
  \draw[<-] (l1) -- ++(-2,0) node[below, pos=0.8] {$\vec x$};
  \draw[->] (l2) -- (act2) node[above, pos=0.8] {};
  \draw[->] ($(l1)-(1.5,0)$) to[out=90, in=90] node[below=1ex, midway, align=center] {skip connection\\(identity)} node[above, midway] {$\vec x$} (add);
  \draw[decorate, decoration={brace, amplitude=1ex, raise=1cm}] (l2.east) -- node[midway, below=1.2cm] {$\mathcal F(\vec x)$} (l1.west);

\end{tikzpicture}
\end{document}

  Typst

skip-connection.typ (107 lines)

#import "@preview/cetz:0.3.2": canvas, draw

#set page(width: auto, height: auto, margin: 8pt)

#canvas({
  import draw: line, content, rect, hobby

  let node-sep = 2.5 // Horizontal separation between nodes
  let arrow-style = (mark: (end: "stealth", fill: black, scale: 0.5), stroke: black)


  // Draw main nodes
  content(
    (0, 0),
    [layer 1],
    fill: rgb("#ffd699"),
    name: "l1",
    frame: "rect",
    padding: (3pt, 6pt),
    stroke: none,
  ) // orange!50
  content(
    (node-sep, 0),
    $a(arrow(x))$,
    name: "act1",
    frame: "rect",
    padding: (0, 3pt),
    stroke: none,
  )
  content((rel: (0, -0.3), to: "act1.south"), "activation")
  content(
    (2 * node-sep, 0),
    [layer 2],
    fill: rgb("#7dc3c3"),
    name: "l2",
    frame: "rect",
    padding: (3pt, 6pt),
    stroke: none,
  ) // teal!50
  content(
    (3 * node-sep, 0),
    text(size: 1.8em)[$plus.circle$],
    name: "add",
  )
  content((rel: (0, -0.3), to: "add.south"), "add")
  content(
    (3.75 * node-sep, 0),
    $a(arrow(x))$,
    name: "act2",
    frame: "rect",
    padding: (0, 3pt),
    stroke: none,
  )
  content((rel: (0, -0.3), to: "act2.south"), "activation")

  // Draw main flow arrows
  line("l1", "act1", ..arrow-style)
  line("act1", "l2", ..arrow-style)
  line("l2", "add.west", ..arrow-style)
  line("add.east", "act2", ..arrow-style)

  // Draw input arrow
  line(
    (rel: (-2, 0), to: "l1"),
    "l1",
    stroke: black,
    name: "input",
    mark: (end: "stealth", fill: black, scale: 0.5),
  )
  content((rel: (0, -0.2), to: "input.10%"), $arrow(x)$)

  // Draw skip connection using hobby curve
  hobby(
    (rel: (-1.5, 0), to: "l1"),
    (rel: (0, 2.2), to: "act1"),
    "add.north",
    close: false,
    tension: 0.8,
    ..arrow-style,
    name: "skip",
  )
  content((rel: (0, 0.3), to: "skip.mid"), $arrow(x)$)
  content(
    (rel: (0, -0.3), to: "skip.mid"),
    align(center, text(size: 0.8em)[skip connection\ (identity)]),
    anchor: "north",
  )

  // Draw F(x) curly brace
  content(
    (rel: (0, -1.2), to: "act1"),
    [#math.underbrace(box(width: 17em), text(size: 1.4em)[$cal(F)(arrow(x))$])],
    name: "fx-brace",
  )

  // Add F(x) + x label
  content(
    (rel: (0.8, 0.8), to: "add"),
    $cal(F)(arrow(x)) + arrow(x)$,
    name: "fx-label",
    frame: "rect",
    stroke: none,
    padding: 1pt,
  )
  line("fx-label.south", "add.north-east", stroke: .2pt, name: "fx-arrow")
})