% Borrowed from MIT 6.987 Advanced Data Structures course (Prof Demaine, 2003)

\documentclass[11pt]{article}
\usepackage[utf8]{inputenc} 
\usepackage{amsmath, amssymb, amsthm, bm}
\usepackage[usenames,dvipsnames]{xcolor}
\usepackage[colorlinks,citecolor=blue,urlcolor=blue,linkcolor=blue]{hyperref}
\usepackage{hypernat}

\usepackage{tikz}
\usepackage{tkz-graph}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf STA4513: Statistical Models of Networks } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\usepackage{enumerate}

\renewcommand{\Pr}{\mathbb{P}}
\newcommand{\EE}{\mathbb{E}}
\newcommand\independent{\protect\mathpalette{\protect\independenT}{\perp}}
\def\independenT#1#2{\mathrel{\rlap{$#1#2$}\mkern2mu{#1#2}}}

\newcommand{\dist}{\sim}

\begin{document}

\lecture{1 --- September 10th, 2014}{Fall 2014}{Prof.\ Daniel M. Roy}{Alberto Camacho}

\section{Overview}
This lecture is an introduction to the typical setting in network data and how it is modeled. We introduce the \emph{Latent Variable Models} and the \emph{Stochastic Block Model}.

This lecture borrows heavily from Peter Hoff's 567 course at U. Washington, in particular, lectures 1, 2, and 17. 

\section{Typical Statistical Setting}

In a typical scenario, we are given datasets $X_1$, $X_2, \ldots$, and we want to:
\begin{enumerate}[(a)]
\item Identify hidden structure/patterns.
\item Predict future or missing data.
\end{enumerate}

\paragraph{Examples of datasets:}
Data can be represented in matrix form. In the following examples, the rows refer to \textbf{objects} (users, days, \dots) and columns refer to \textbf{attributes} of these objects.
\begin{enumerate}
\item $(height, weight, age)$ of students: 

$$X = 
  \begin{bmatrix}
    171 & 65 & 21\\
    182 & 70 & 19\\
    170 & 60 & 20
  \end{bmatrix}
  $$
  
 \item Daily average $(temp, humid., rainfall)$ in Toronto:
 
$$X = 
  \begin{bmatrix}
    21 & 40 & 1\\
    25 & 50 & 0\\
    22 & 20 & 0
  \end{bmatrix}
  $$
  \item Movie ratings by users:
  
  $$X = 
  \begin{bmatrix}
    5 & 5 & 5 & 22\\
    5 & 1 & 2 & 55\\
    5 & 2 & 5 & 23
  \end{bmatrix}
  $$
\end{enumerate}

We notice that data can have different \emph{nature}. E.g., it can represent:
\begin{itemize}
\item a \emph{set} of measurements
\item a \emph{sequence} of measurements in time (ordered!)
\item an \emph{array} of measurements, each associated with a pair of objects
\end{itemize}

\section{Network Data}

Relational Data are data that include:
\begin{itemize}
\item a set of object labels. We \emph{assume} that different rows correspond to different objects.
\item measurements associated with pairs of object labels.
\end{itemize}

\paragraph{Example: Symmetric Social Network}
The symmetric social network is a typical example of a relational dataset.
\begin{align*}
\lbrace y_{ij} | 1 \leq i < j \leq n \rbrace &= \text{binary indicators of friendship between individuals} \\
y_{ij} &= \left\lbrace
  \begin{array}{l}
     1 \text{ if } i,j \text{ friends} \\
     0 \text{ otherwise} \\
  \end{array}
  \right.
\end{align*}

\section{Components of Network Data}
A network dataset consists of:
\begin{itemize}
\item \textbf{Nodes:} set or multiple sets of object labels: objects, egos, \ldots
\item \textbf{Measurements} associated with individual/pairs/triplets of nodes
	\begin{description}
	\item[dyadic variables] (structural variables): measurements on pair of nodes (dyal)
	\item[nodal variables] (actor attributes): measured on nodes.
	\end{description}
\end{itemize}

The typical setting of statistics is to have nodes and nodal variables. On the other hand, triadic variables are rare. We distinguish among \emph{standard} and \emph{relational} data.
\begin{description}
\item[Standard Data] consists of nodes and nodal variables.
\item[Relational Data] consists of nodes, nodal variables and \textbf{dyadic} variables.
\end{description}


\section{Types of Relations}
Networks can be characterized according to different criteria:
\begin{enumerate}
\item According to symmetry of edges:
	\begin{description}
	\item[undirected networks] one measurement per pair ($y_{ij}, i<j$)
	\item[directed networks] two measurements ($y_{ij}, y_{ji}$)
	\end{description}

	\paragraph{Example:}
	\begin{itemize}
	\item undirected: facebook, distance between cities
	\item directed: emails sender/recipient
	\end{itemize}

\item Binary vs. valued:
	\begin{description}
	\item[Binary] (or \textbf{dichotomous}:) A binary relation takes only two values.
	\item[Valued:] A valued relation takes more than two values (that can be ordinal or categorical).
	\end{description}
\item More dichotomies, e.g. Bipartite Relations where groups of objects are disjoint (e.g. in movie ratings row and column objects are two partitions).


\end{enumerate}


\section{Structure of Course}
\noindent Probabilistic Symmetries lead to \textbf{models}. Network data may manifest:
\begin{itemize}
\item uniform structure
\item exchangeable (or \emph{partial} exchangeable)/conditional independence
\item \textbf{RCE} Row/Column Exchangeable
\item graph limits: $G_1, G_2, \ldots \rightarrow G$. The limit of a sequence of graphs with $|G_n| \rightarrow \infty$
\end{itemize}

While there is a whole theory in dense graphs, there is less work in sparse graphs and stationary network sampling.

\section{Conditionally Uniform Model}

Consider a graph $X = (X_{ij})_{\lbrace i,j \rbrace \in [n]}$, $[n] = \lbrace 1, \ldots, n \rbrace$, where:
$X_{ij} = \left\lbrace
  \begin{array}{l}
     1 \text{ if edge between } i, j\\
     0 \text{ otherwise} \\
  \end{array}
  \right.
  $

Let $T_1, T_2, \ldots$ be a collection of statistics, where:
\begin{itemize}
\item $T_1 = \#$ of edges
\item $T_2 = \#$ of triangles
\item \ldots
\end{itemize}

Then, conditional on $T_1(X), \ldots, T_k(X)$, the graph $X$ is uniformly distributed.

\subsection{Erd\H{o}s-Rényi Model}
Important special case is the Erd\H{o}s-Rényi model, when we consider the statistic $T_1$ alone. That is, we consider only the number of edges (or edge density) in the graph, so the graph is uniquely characterized by the probability $p$ of having an edge between two nodes.

\section{Latent Variable Models}
The idea is that:
\begin{itemize}
\item Nodes have attributes ($u_i$).
\item Edges are independent given these attributes. That is, given $u_i, u_j$, then $X_{ij} \independent X \setminus X_{ij}$.
\end{itemize}

The aim is to look for good attributes (E.g., $X_{ij} \dist F_\theta(u_i,u_j)$, where $u_i$ and $\theta$ are attributes) that allow us to make predictions over the edges.

\section{Block Model}
The Block Model consists of:
\begin{itemize}
\item A \textbf{partition} of the nodes into classes (classes/blocks/clusters/types).
The partition is represented by a classification function $c: [n] \rightarrow [K]$, where $K$ is the number of blocks.

\item An \textbf{estimation} of the rate of ties $X_{ij}$ between \& within blocks.
Between-group \textbf{density matrix}:
$$
\Theta=
  \begin{bmatrix}
    \theta_{11} & \cdots & \theta_{1k}\\
    \vdots & \ddots & \vdots\\
    \theta_{k1} & \cdots & \theta_{kk}
  \end{bmatrix}
$$
where $\theta _{c_i, c_j} = Pr(X_{ij} = 1)$.
\end{itemize}

Edges are independent in this model and nodes within the same block are \emph{stochastically equivalent}.

\begin{definition}
Two nodes $i,k$ are stochastically equivalent if, $\forall j$, $Pr(X_{ij} = 1) = Pr(X_{kj} = 1)$.
\end{definition}

In this case, $c_i = c_k$ and $\theta_{c_i, c_j} = \theta_{c_k, c_j}$.

\paragraph{Question: How would we identify stoc. equiv. nodes?}

\section{Stochastic Block Model}
\noindent We want to estimate:
\begin{itemize}
\item class function ($c_i$)
\item between-class rates
\end{itemize}
\noindent Under SBM:
\begin{align*}
Pr(X=x | c,\theta) &= \Pi_{i\neq j} \theta_{c_i,c_j}^{y_{ij}} (1 - \theta_{c_i,c_j})^{1-y_{ij}}  \\
& = \Pi_{k,l \in [K]} \theta_{kl}^{s_{kl}} (1-\theta_{kl})^{n_{kl}-s_{kl}}
\end{align*}
\noindent where:
\begin{align*}
n_{kl} &= \# \text{ pairs $i,j$ s.t. $c_i = k$, $c_j=l$} \\
s_{kl} &= \# \text{ pairs $i,j$ s.t. $c_i = k$, $c_j=l$ and $X_{ij} = 1$}
\end{align*}

\section{Approximate MLE of $c_1, \dots, c_n, \theta$}
Estimate graph model with:
\begin{itemize}
\item EM
\item Gibbs sampling/MLMC
\end{itemize}
We can order classes and (more easily, less costly) see relations \& structure. 

\paragraph{Example:}

\begin{figure}[htbg]
\center
\begin{tikzpicture}
\shade[top color=black!95!white,bottom color=black!1!white,shading angle=45] (0,0) rectangle (3,3);

\draw[thick] (0,0) -- (3,0) -- (3,3) -- (0,3) -- (0,0);
\draw[thick] (1.3,0) -- (1.3,3);
\draw[thick] (2.3,0) -- (2.3,3);
\draw[thick] (0,1.7) -- (3,1.7);
\draw[thick] (0,0.7) -- (3,0.7);

\draw (0.65 cm,3) -- (0.65 cm,3) node[anchor=south] {$2$};
\draw (1.8 cm,3) -- (1.8 cm,3) node[anchor=south] {$1$};
\draw (2.65 cm,3) -- (2.65 cm,3) node[anchor=south] {$3$};

\draw (1pt,0.35 cm) -- (1pt,0.35 cm) node[anchor=east] {$3$};
\draw (1pt,1.1 cm) -- (1pt,1.1 cm) node[anchor=east] {$1$};
\draw (1pt,2.35 cm) -- (1pt,2.35 cm) node[anchor=east] {$2$};

\end{tikzpicture}
\caption{Block Model}
\label{fig:block_model}
\end{figure}


In the block model of figure \ref{fig:block_model}, we see $\theta_{12} < \theta_{22}$.

The expected number of edges in block $ij$ is $\binom {n_{ij}} {2} \theta_{ij}$

In principle, more classes imply that $\theta_{ij}$ is less noisy.

\section{Novicki \& Snijders (2001)}
 The Stochastic Block Model has been extended in a number of directions:
\begin{itemize}
\item covariates
\item directed data ($X_{ij}, X_{ji}$)
\item valued data
\item unknown $K$ (number of blocks)
\item hierarchical block models
\end{itemize}

\paragraph{Ordinal Data:}
\begin{align*}
Z_{ij} &= \theta_{c_i,c_j} + \epsilon_{ij} \\
X_{ij} &= g(Z_{ij}) =  \left\lbrace
  \begin{array}{l}
     y_1 \text{ if } z_{ij} \in [c_1,c_2) \\
     \vdots \\
     y_m \text{ if } z_{ij} \in [c_m,c_{m+1})
  \end{array}
  \right.
\end{align*}


\paragraph{Directional Data:}
\begin{align*}
p(X_{ij}, X_{ji} | \mu , \gamma, a_i , b_i )  &\propto e^{\mu_{ij} x_{ij} + \mu{ji} x_{ji} + \gamma x_{ij} x_{ji}} \\
p_i \equiv \mu_{ij} &= \mu + a_i + b_j  \\
\text{[NS01] } \mu_{ij} &= \theta_{c_i,c_j} \\
\end{align*}

\section{Matrix Form of Block Model}
$\mathbb{E}X_{ij} = \theta_{c_i,c_j}$ ($c_i, c_j$ unobserved)

$\Theta$ matrix can be expressed as $\theta_{c_i, c_j} = u_i^T \Theta u_j$, where:
\begin{itemize}
\item $u: K\times 1$ vector, all zero except $u[c_i] = 1$
\item $\Theta: K \times K$ matrix of rates
\end{itemize}

\subsection{Limitations of SBMs}
\begin{description}
\item[stochastic equivalence] nodes in the same class are stoc. equiv. In reality, don't expect any two nodes to act identically.
\item[sender/receiver equivalence] latent variable ($c_i$'s) are the same regardless of direction of interaction.
\end{description}


\section{Latent Factor Models}
Generalize from $\theta_{c_i, c_j} = u_i^T \Theta u_j$.

$\mathbb{E} X_{ij} = u_i^T D v_j$, where:
\begin{itemize}
\item $u_i$ is a vector of latent \emph{factors} describing $i$ as a sender.
\item $v_j$ is a vector of latent \emph{factors} describing $i$ as a receiver.
\item $D$ is a diagonal matrix.
\end{itemize}

\noindent Continuous, binary, ordinal data:
\begin{align*}
Z_{ij} &= u_i^T D v_j + \epsilon_{ij} \\
X_{ij} &= g(Z_{ij})
\end{align*}

\section{Understanding Latent Factors}
We can write $Z = U^T D V + E$, where $U,V \in \mathbb{R}^{k\times n}, D \in \mathbb{R}^{k\times k}$.

Then, $Z_{ij} = \Sigma_{k \in [K]} d_k u_{ik} v_{jk}  + \epsilon_{ij}$

\paragraph{Interpretation:}
\begin{itemize}
\item $u_i \approx u_j \Rightarrow $ approx. stoc. equiv. as senders.
\item $u_i \approx v_j , D>0\Rightarrow $ sender $i$ approx. receiv. $j$.
\end{itemize}

\paragraph{Matrix Decomposition Interpretation:}

\begin{enumerate}
\item every $m \times n$ matrix $Z$ can be written $Z = UDV^T$, where 
  \begin{itemize}
  \item $D = diag(d_1, \ldots, d_{mn}$ ($d_1 > d_2 > \dots$).
  \item $U,V$ orthonormal
  \item $U \in \mathbb{R}^{n\times m}, V \in \mathbb{R}^{m\times n}, D \in \mathbb{R}^{m\times m}$
  \end{itemize}
\item if $UDV^T$ is the SVD of $Z$, then \\$\hat{Z} = U_{[1:k]} D_{[1:k,1:k]} V_{[1:k]}^T$ least square rank-k approx of $Z$.

\item $Z$ is symmetric
\item $Z = U \bigwedge U^T + \epsilon_{ij}$

eigenmodel (Hoff) $\bigwedge = diag(\lambda_1, \ldots, \lambda_k)$.
\item $\lambda_r > 0$: homophily.
\item $\lambda_r < 0$: anti-homophily.
\end{enumerate}


%\bibliography{mybib}
\bibliographystyle{alpha}

\end{document}
