% EngCguide.tex
% for the suite of standard Cambridge designs
% 2011/02/03, v1.10

\NeedsTeXFormat{LaTeX2e}[1996/06/01]

%TODO: Had to remove \abovecaptionskip and \belowcaptionskip lines from EngC for this to import correctly.
\RequirePackage{listings}


\documentclass{EngC}
% \documentclass{cambridge7A}
% \documentclass[multi]{EngC} % multi-contributor option
% \documentclass[prodtf]{EngC}% if you have helvetica neue condensed fonts

\lstset{
  basicstyle=\small\ttfamily,
  columns=fullflexible,
  breaklines=true,
  frame = single,
  numbers = left,
  numberstyle = \small,
  framexleftmargin=0pt,
  language=Python
}


\usepackage[rightcaption,raggedright]{sidecap}% for side captions
\usepackage{framed}         % for floatingboxes
\usepackage{soul}           % for letterspacing in theorem-style headings

% for the Harvard author-date referencing system
\usepackage[agsm]{harvard}

\usepackage{booktabs}

% if you are using either vancouver.bst or IEEEtran.bst and wish to remove
% square braces in the reference list, uncomment the line below
% \removesquarebraces

\usepackage{rotating}
\usepackage{floatpag}
\usepackage{mathtools}
\rotfloatpagestyle{empty}

\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}

\usepackage{siunitx}

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}

% \usepackage{txfonts}% times font (used to produce EngCguide.pdf)
                  % this package must be loaded after amsthm.sty
\usepackage{graphicx}

\usepackage{hyperref}[linkcolor=black]

\usepackage{cleveref}
\usepackage{todonotes}

\usepackage{tikz}
\usetikzlibrary{positioning,chains,fit,shapes,calc,arrows.meta}




% indexes
% uncomment the relevant set of commands

% for a single index
% \usepackage{makeidx}
% \makeindex

% for multiple indexes using multind.sty
% \usepackage{multind}
% \ProvidesPackage{multind}
% \makeindex{authors}
% \makeindex{subject}

% for multiple indexes using index.sty
\usepackage{index}
% \newindex{aut}{adx}{and}{Author index}
\makeindex

\newcommand\cambridge{EngC}

% see chapter 3 for details
\theoremstyle{plain}% default
\newtheorem{theorem}{Theorem}[chapter]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem*{corollary}{Corollary}

\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{condition}[theorem]{Condition}

\theoremstyle{remark}
\newtheorem{remark}{Remark}[chapter]
\newtheorem{notation}[remark]{Notation}
\newtheorem*{case}{Case}

\hyphenation{line-break line-breaks docu-ment triangle cambridge amsthdoc
cambridgemods baseline-skip author authors cambridgestyle en-vir-on-ment polar}

\setcounter{tocdepth}{2}% the toc normally lists sections;
% for the purposes of this document, this has been extended to subsections

%\newcommand\todo[1]{}

\newcommand{\empt}[2]{$#1^{\langle #2 \rangle}$}

\begin{document}
\title
%[Subtitle, if you have one]
{Personalized Machine Learning}

\author{Julian McAuley}

\frontmatter
\maketitle
\tableofcontents
\listoffigures
%   \listoftables
\listoffloatingboxes
%   \listofcontributors
% \editedlistofcontributors
% \include{notation}

\chapter*{Notation}

\begin{tabular}{@{}ll}
 $u \in U$ & users $u$ in user set $U$\\
 $i \in I$ & items $i$ in item set $I$\\
 $I_u$ & set of items rated by user $u$\\
 $U_i$ & set of users who have rated item $i$\\
 $|U|$ & number of users\\
 $|I|$ & number of items
\end{tabular}

\mainmatter

\chapter{Introduction}

\chapter{Personalization in Everyday Life}

\section{The Importance of Personalization}

\subsection{e-Commerce and Advertising}
\subsection{Social and News-Media}
\subsection{User Productivity}
\subsection{Personalized Health}
\subsection{Computational Social Science}
\subsection{Personalized Dialog and Interactive Agents}

\section{Techniques for Personalization}

\subsection{Contextual Personalization and Model-Based Personalization}

\subsection{User Representations as Manifolds}

\part{Fundamentals of Personalized Machine Learning}

\chapter{Machine Learning Primer}
\label{chap:basics}

In this 

\emph{Supervised Learning}\index{Supervised Learning} approaches assume that our datasets can be separated into two components:
\begin{description}
\item[labels] (denoted $y$) that we would like to predict, and
\item[features] (denoted $X$) which we believe will help us to predict those labels
\end{description}
For example, given a sentiment analysis\index{Sentiment Analysis} task (Chapter \ref{chap:text}), our data might be (the text of) reviews from \emph{Amazon}\index{Amazon} or \emph{Yelp}\index{Yelp}, and our labels would be the ratings associated with those reviews.

\begin{floatingbox}
\caption{Supervised, Semi-supervised, and Unsupervised Learning}
\emph{Supervised Learning} approaches are those that seek to directly learn the relationship between the observed data $X$ and the labels $y$. Examples include regression and classification, as covered in this chapter.
\emph{Unsupervised Learning} approaches seek to find patterns in the data $X$, but are not specifically concerned with predicting any label. Examples include techniques for clustering and dimensionality reduction. 
%\todo{Covered in some chapter?}
% \emph{Semi-supervised learning} approaches include those that model both the data $X$ and the labels $y$, but are not explicitly concerned with learning the mapping $f(X) \rightarrow y$.
%%\todo{Needs an example}
\label{box:supervised}
\end{floatingbox}

Throughout this chapter, we will assume that we are given \emph{labels} in the form of a vector, and features in the form of a matrix



%\section{Regression}
%\todo{Section structure? Regression could be its own chapter}

\section{Linear Regression}
\label{sec:linearReg}

Perhaps the simplest association we could assume between our features $X$ and our labels $y$ would be a \emph{linear}\index{Linear model} relationship, i.e.,~that the relationship between $X$ and $y$ was defined as
\begin{equation}
  y = X\theta.
  \label{eq:linearTheta}
\end{equation}
In the above, $X$ is a matrix describing features extracted from the data, $y$ is our vector of labels, and $\theta$ is a vector of unknowns which describe which features are \emph{relevant} to predicting the labels.

Ignoring strict notation for now, a trivial example might consist of predicting a review's rating as a function of its length. To do so, let's consider a small dataset of 100 $(\text{length}, \text{rating})$ pairs from \emph{Goodreads} Fantasy novels. \Cref{fig:ratingLength} plots the relationship between review length (in characters) and the rating.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLength.pdf}
\caption{Ratings compared to review length (in characters), based on 100 reviews of fantasy novels from \emph{Goodreads} \label{fig:ratingLength}}
\end{SCfigure}

From \Cref{fig:ratingLength}, there appears to be an association between ratings and review length, i.e.,~more positive reviews tend to be longer. A very simple model might attempt to describe that relationship with a line, i.e.,
\begin{equation}
  \text{rating} \simeq \theta_0 + \theta_1 \times (\text{review\ length}).
  \label{eq:sysOfEq0}
\end{equation}
Note that the above is just the standard equation for a line ($y = mx + b$), where $\theta_1$ is a slope and $\theta_0$ is an intercept.

If we can identify a line that approximately describes this relationship, we can use it to estimate a rating from a given review, even though we may never have seen a review of some specific length before. In this sense, the line is a simple \emph{model} of the data, as it allows us to predict labels from previously unseen features. To do so we formalize the problem of finding a line of best fit.

Specifically, we are interested in identifying the values of $\theta_0$ and $\theta_1$ that
%correspond to a \emph{line of best fit}, i.e.,~so that the line 
most closely matches the trend in \Cref{fig:ratingLength}.
%
To solve for $\theta = [\theta_0, \theta_1]$, we can write out the problem as a system of equations in matrix form:
\begin{equation}
  y = X \cdot \theta,
  \label{eq:sysOfEq1}
\end{equation}
where $y$ is our vector of observed ratings and $X$ is our matrix of observed features (in this case the reviews lengths). For the first few samples of our Goodreads data we have:
\begin{equation}
  \underbrace{\left[ 
  \begin{tabular}{c} 5 \\ 5 \\ 5 \\ 4 \\ 3 \\ 5 \\ \vdots \end{tabular}
  \right]}_{y}
  \simeq
  \underbrace{\left[
  \begin{tabular}{cc} 1 & 2086 \\ 1 & 1521 \\ 1 & 1519 \\ 1 & 1791 \\ 1 & 1762 \\ 1 & 470 \\ \multicolumn{2}{c}{\vdots} \end{tabular}
  \right]}_{X} \cdot \underbrace{\left[ \begin{tabular}{c} $\theta_0$ \\ $\theta_1$ \end{tabular} \right]}_{\theta}.
  \label{eq:sysOfEq2}
\end{equation}
It is useful to compare \Cref{eq:sysOfEq0,eq:sysOfEq2} to understand how the matrix expression above expands to include the slope ($\theta_1 \times (\text{review\ length})$) and intercept ($\theta_0$) terms.

\begin{floatingbox}
\caption{Why is there a column of `1's in the feature matrix?}
The first column of the feature matrix $X$ in \Cref{eq:sysOfEq2}, and in most feature matrices throughout this chapter, is a column of ones. To explain why we always have this feature, it is useful to expand the inner product $[1, \text{length}] \cdot [\theta_0, \theta_1]$ (e.g.~as in \Cref{eq:sysOfEq2}) to confirm that it expands to the equation for a line $\theta_0 + \theta_1 \times \text{length}$. Without the constant term in our feature matrix, we would be implicitly assuming that the fitted line passes through $(0,0)$.
%\todo{Only moderately useful explanation}
\label{box:feature1}
\end{floatingbox}

We would like to solve \Cref{eq:sysOfEq2} for $\theta$. Na\"ively, we might attempt to multiply both sides of the equation $y = \theta \cdot X$ by $X^{-1}$; however the inverse is not well-defined, since $X$ is not a square matrix.

To obtain a square matrix, we (left) multiply both sides by $X^T$:
\begin{equation}
    X^T y \simeq X^TX \theta,
\end{equation}
resulting in a square (in this case $2 \times 2$) matrix $X^TX$. We can now multiply both sides by the inverse of this matrix:
\begin{equation}
    (X^TX)^{-1}X^T y \simeq (X^TX)^{-1}(X^TX)\theta, \quad \text{or\ simply} \quad \theta = (X^TX)^{-1}X^T y.
    \label{eq:pseudoInv}
\end{equation}
The quantity $(X^TX)^{-1}X^T$ is known as the \emph{pseudoinverse} of $X$.

Computing $\theta = (X^TX)^{-1}X^Ty$ for our 100 ratings from Goodreads yields
%\todo{make sure numbers match those in figure}
\begin{equation}
    \theta = \left[ 
    \begin{tabular}{c} 
    $3.984$ \\ 
    $1.194 \times 10^{-4}$
    \end{tabular}
    \right],
    \label{eq:thetaTwo}
\end{equation}
corresponding to the line
\begin{equation}
    \text{rating} = 3.984 + 1.194 \times 10^{-4} (\text{review\ length}).
    \label{eq:reviewLength1}
\end{equation}
This line reflects a positive (albeit slight) trend between review length and ratings: for every additional character in a review, our estimate of the rating increases very slightly (by $1.194 \times 10^{-4}$ points). This line of best fit is depicted in \Cref{fig:ratingLengthBestFit}. We will revisit how to interpret the parameters of linear models in \Cref{sec:interpretLinear}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLengthBestFit.pdf}
\caption{Line of best fit between ratings and review length. \label{fig:ratingLengthBestFit}}
\end{SCfigure}

\paragraph{More Complex Models}

The above reasoning generalizes to fitting more complex models than a simple line, for example we could imagine that a rating could be related to both the length of the review and the number of comments the review received:
\begin{equation}
    \text{rating} \simeq \theta_0 + \theta_1 \times (\text{review\ length}) + \theta_2 \times (\text{n\_comments}).
    \label{eq:threeParameters}
\end{equation}
The above process---finding a line of best fit that best approximates the relationship between our observed features $X$ and labels $y$---describes the basic concept of \emph{linear regression}.

\paragraph{Adding More Dimensions}

Just as \Cref{eq:sysOfEq0} corresponded to fitting a line in two dimensions, \Cref{eq:threeParameters} now corresponds to fitting a plane in three. But ultimately the procedure for fitting this model remains the same. We simply have an additional column in our feature matrix:
\begin{equation}
    X = \left[
  \begin{tabular}{ccc} 1 & 2086 & 1 \\ 1 & 1521 & 1 \\ 1 & 1519 & 5 \\ 1 & 1791 & 1 \\ 1 & 1762 & 0 \\ \multicolumn{3}{c}{\vdots} \end{tabular}
  \right].
\end{equation}
Solving $\theta = (X^TX)^{-1}X^Ty$ yields
\begin{equation}
    \theta = \left[
    \begin{tabular}{c}
         3.954 \\
         $7.243 \times 10^{-5}$\\
         0.108
    \end{tabular} \right]
    \begin{tabular}{l}
      intercept\\
      slope for length\\
      slope for number of comments
    \end{tabular}.
    \label{eq:thetaThree}
\end{equation}
Interestingly, when we add this additional parameter $\theta_2$, the values of $\theta_1$ and $\theta_0$ are different from those of the model we previously fit (compare \Cref{eq:thetaThree,eq:thetaTwo}). Critically, the slope associated with the length term ($\theta_1$) is reduced in our new model. We discuss how to interpret these parameters in \Cref{sec:interpretLinear}.







\section{Evaluating Regression Models}



When developing the linear models above, we were somewhat imprecise about what is meant by a `line of best fit' (or generally a model of best fit). Indeed, the pseudoinverse is not a `solution' to the system of equations given in \Cref{eq:sysOfEq2}, but is merely an approximation (naturally, the line of best fit does not pass through all points exactly).

Here, we would like to be more precise about what it means for a model to be `good'. This is a key issue when fitting and evaluating any machine learning model. One needs a way of quantifying how closely a model fits the given data. Given a desired measure of success, we can compare alternative models against this measure, and design optimization schemes that optimize the desired measure directly.

\subsection{The Mean Squared Error}
\label{sec:MSE}

A commonly used evaluation criterion when evaluating regression algorithms is called the Mean Squared Error, or MSE. The Mean Squared Error between a model $f_\theta(X)$ and a set of labels $y$ is defined as 
%\todo{make sure notation is consistent with rest of chapter}
\begin{equation}
    \text{MSE}(y, f_\theta(X)) = \frac{1}{|y|} \sum_{i=1}^{|y|} (f_\theta(x_i) - y_i)^2,
\end{equation}
in other words, the average squared difference between the model's predictions and the labels. Often reported is also the \emph{Root} Mean Squared Error (RMSE), i.e.,~$\sqrt{\text{MSE}(y, f_\theta(X))}$; the RMSE is sometimes preferable as it is consistent in scale with the original labels.
%\todo{check this}

With some effort, it can be shown that the linear model $f_\theta(X)$ that minimizes the MSE compared to the labels $y$ is given by using the pseudoinverse as in \Cref{eq:pseudoInv}. We leave this as an exercise.
%\todo{write this exercise and reference}

\subsection{Why the Mean Squared Error?}
\label{sec:whyMSE}

Although the Mean Squared Error has a convenient relationship with the pseudoinverse, it may otherwise seem a somewhat arbitrary choice of error measure. For instance, it may seem more obvious at first to compute an error measure such as the Mean Absolute Error (or MAE):
\begin{equation}
    \text{MAE}(y, f_\theta(X)) = \frac{1}{|y|} \sum_{i=1}^{|y|} | f_\theta(x_i) - y_i |
    \label{eq:MAE}
\end{equation}
Or, why not count the number of times the model is wrong by more than one star? Or why not measure the mean \emph{cubed} error?

To defend the mean squared error as a reasonable choice, we need to characterize what types of errors are more `likely' than others. Essentially, the Mean Squared Error is assigning small penalties to small errors, and \emph{very} large penalties to large errors. This is in contrast to, say, the Mean Absolute Error, which assigns penalties exactly proportional to how large the error is. What the Mean Squared Error therefore seems to be assuming is that small errors are \emph{common} and large errors are very uncommon.

What we are talking about informally above is a notion of how errors are \emph{distributed} under some model. Formally, we say that the labels are equal to our model's predictions, plus some error:
\begin{equation}
    y = \underbrace{f_\theta(X)}_{\text{prediction}} + \underbrace{\epsilon}_{\text{error}},
\end{equation}
and that our error follows some probability distribution. Our argument above said that small errors are common and large errors are very rare. This suggests that errors may be distributed following a \emph{bell curve}, which we could capture with a Gaussian (or `Normal') distribution:
\begin{equation}
    \epsilon \sim \mathcal N(0, \sigma^2).
\end{equation}
The density function for a (zero mean) Gaussian distribution is given by
\begin{equation}
    f'(x') = \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{x'}{\sigma}\right)^2}
\end{equation}
(we use the notation $f'$ and $x'$ to avoid confusion with $f$ and $x$ elsewhere). So, the probability density of an error as large as $y_i - f_\theta(x)$ is given by
\begin{equation}
    \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y - f_\theta(x)}{\sigma}\right)^2}.
    \label{eq:errorDensity}
\end{equation}
This density function is depicted in \Cref{fig:gaussianDensity}.
%\todo{squash plot so that it's same as others?}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/gaussianError.pdf}
\caption{Gaussian error density. \label{fig:gaussianDensity}}
\end{SCfigure}

\subsection{Maximum Likelihood Estimation of Model Parameters}
%\todo{subsection or subsubsection? organization could be better. or could add some gluing text above}

%Now that we have this type of density function in mind, we can begin to reason more formally about what it means for a particular model to be a `good' fit to the data.
Having defined the density function above, we can now reason more formally about about what it means for a particular model to be a `good' fit to the data. In other words, we would like to ask how `likely' a particular model is in terms of a given error distribution. %For example, are the errors we observe consistent with a bell curve shape? 
%\todo{cut this text up a lot, check}

Specifically, the density function in \Cref{eq:errorDensity} gives us a means of assigning a probability (or likelihood) to a particular set of labels $y$, given features $X$ and a model $\theta$, under some particular error distribution (in this case a Gaussian):
%\todo{mouthful}
\begin{equation}
    \mathcal L_\theta(X | y) = \prod_{i = 1}^{|y|} \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}.
    \label{eq:likelihood1}
\end{equation}
Essentially, we want to choose $\theta$ so as to maximize this likelihood.
% Something else here?
Our goal intuitively is to choose a value of $\theta$ that is consistent with this error distribution, i.e.,~a model that makes many small errors and few large ones.

Precisely, we would like to find
%\begin{equation}
    $\argmax_{\theta} \mathcal L_\theta(X | y)$.
%\end{equation}
This procedure (finding a model $\theta$ that maximizes the likelihood under some error distribution is known as \emph{Maximum Likelihood Estimation}.
We solve by taking logarithms and removing irrelevant terms ($\pi$, $\sigma$):
\begin{align}
  \argmax_\theta \mathcal L_\theta(X | y) & = \argmax_\theta \ell_\theta(X | y)\\
  & = \argmax_\theta\log \prod_{i = 1}^{|y|} \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}\\
  & = \argmax_\theta \sum_i \log e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}\\
  & = \argmax_\theta -\sum_i (y_i - f_\theta(x_i))^2\\
  & = \argmin_\theta \sum_i (y_i - f_\theta(x_i))^2\\
  & = \argmin_\theta \frac{1}{|y|} \sum_i (y_i - f_\theta(x_i))^2
  \label{eq:MSEMLEderiv}
\end{align}
Note crucially in the above equation that \emph{the maximum likelihood solution for $\theta$ under our Gaussian error model is precisely the Mean Squared Error}. This demonstrates the relationship between the Mean Squared Error and Maximum Likelihood Estimation (which we summarize in 
%\Cref{box:MSEMLE}).
Box~\ref{box:MSEMLE}).
%\todo{text here, and in the box, is a bit clunky, try rewriting}

\begin{floatingbox}
\caption{The MSE and the MLE}
The argument we made in \Cref{sec:whyMSE} explained our motivation behind the choice of the Mean Squared Error (MSE): by choosing the MSE as our error metric, we are implicitly assuming that our model's errors follow a Gaussian distribution. This assumption is explained by the fact that minimizing the mean squared error maximizes the likelihood of the observed errors under a Gaussian error model.
\label{box:MSEMLE}
\end{floatingbox}

The above arguments may seem like just a mathematical curiosity, and indeed in practice we will often minimize the Mean Squared Error without scrutinizing the decision to do so. But this relationship between error functions and probabilities will come up regularly when we develop models for classification (\Cref{sec:classification}), sequence mining (\Cref{chap:temporalSequential}), and recommender systems (\Cref{chap:recsys}). To summarize a few key points:
\begin{enumerate}
    \item When we optimize a certain error criterion, we are often making implicit assumptions about how errors are distributed.
    \item Sometimes, a model will poorly fit a dataset because these assumptions are violated. Understanding the underlying assumptions gives us a chance to diagnose them and attempt to correct it (see below).
    %\todo{below is paragraph, so can't be referenced}
    \item In many of the models we fit later, 
    %\todo{which?} 
    we will use this style of probabilistic language, i.e., we will talk about some observed data having high \emph{likelihood} under some model. Fitting such models will use this same strategy of selecting a model which maximizes this likelihood.
\end{enumerate}

\subsection{The $R^2$ Coefficient}

Having motivated our choice of the Mean Squared Error at some length, it is worth asking how low the MSE should be before we consider our model to be `good enough'?

This quantity turns out not to be well-defined: the Mean Squared Error will depend on the scale and variability of our data, and the difficulty of our task. For example, predicted ratings on a 5-point scale would likely have lower MSEs than predicted ratings on a 100-point scale; on the other hand, this might not be the case if ratings on a 100-point scale were highly concentrated (e.g.~nearly all ratings were in the 92-95 range). Finally, the MSE in either setting could be higher simply due to a lack of available features that allow us to predict ratings accurately.

As such, we would like a calibrated measurement of model error. As we just argued, the MSE is related to the \emph{variance} of the data: this relationship is easy to see as follows:
%\todo{Why is this MSE(y|X) and not MSE(f|y,X)?}
\begin{align}
    \bar{y} & = \frac{1}{|y|} \sum_i y_i\\
    \mathrm{var}(y) & = \frac{1}{|y|} \sum_i (y_i - \bar{y})^2\\
    \mathrm{MSE}(y|X) & = \frac{1}{N} \sum_i (y_i - f(x_i))^2
\end{align}
In other words, the Mean Squared Error would be equal to the variance if we had a trivial predictor that always estimated $f(x_i) = \bar{y}$.\footnote{Note that this is the best we could do if using a trivial predictor of the form $f(x_i) = \theta_0$ (\Cref{ex:?}).} Thus the variance might be used as a way of normalizing the Mean Squared Error:
\begin{equation}
    \mathrm{FVU}(y|X) = \frac{\mathrm{MSE}(y|X)}{\mathrm{var}(y)}
\end{equation}
This quantity, known as the \emph{Fraction of Variance Unexplained} essentially measures the extent to which the model \emph{explains variability} in the data, as compared to a predictor which always predicts the mean (i.e.,~one which explains no variability at all).

This quantity will now take a value between 0 and 1: 0 being a perfect classifier (MSE of zero), and 1 being a trivial classifier.\footnote{The FVU \emph{could} be greater than 1, if our classifier were \emph{worse} than a trivial one.}

Often, one reports the \emph{$R^2$ coefficient}, which is simply one minus the FVU:
\begin{equation}
    R^2 = 1 - \frac{\mathrm{MSE}(y|X)}{\mathrm{var}(y)},
\end{equation}
which now takes a value of 1 for a perfect predictor, and 0 for a trivial predictor. The name `$R^2$' comes from a different way of deriving the same quantity, in terms of the correlation between the predictions and the labels.\footnote{We omit this alternate derivation for now, but revisit the idea of correlation briefly in \Cref{sec:pearsonSim}.}



\subsection{What to do if Errors \emph{Aren't} Normally Distributed?}

The arguments we made above characterized the relationship between the Mean Squared Error and the normal (Gaussian) distribution. In summary, the MSE 
%\todo{consistently abbreviate} 
is a reasonable choice so long as our model errors are expected to be centered around zero, and not to have large outliers.

But what can we do if these assumptions do not hold? First, we consider how to validate the assumptions in the first place. Recall that our assumptions assume that the residuals,
\begin{equation}
    r_i = (y_i - f_\theta(x_i))
    \label{eq:qq}
\end{equation}
follow a normal distribution. To begin with, a simple plot may reveal whether the residuals follow the desired overall trend.

\Cref{fig:qq} (left) shows a histogram of residuals $r_i$ for a simple prediction task, in which we estimate review lengths as a function of user gender (covered later in \Cref{sec:binaryFeat}).
Although the plot has a slight bell shape, it deviates from the normal distribution in several key ways, for instance:
\begin{itemize}
    \item The residuals do not appear to be centered around zero. In fact the \emph{average} residual is zero,\footnote{In fact, the average residual of this type of linear regression model is \emph{always} zero (see \Cref{exercise:?}).} though largest bins in the histogram are somewhat below zero.
    \item The are some large outliers (i.e.,~extremely long reviews whose length was underpredicted).
    \item There are no small outliers, and there is almost no `left tail,' i.e.,~the model never significantly overpredicts.
\end{itemize}

\begin{figure}
\begin{center}
\includegraphics[scale=0.75]{fig/qqhist.pdf}
\includegraphics[scale=0.75]{fig/qq.pdf}
\end{center}
\caption{Histogram of observed residuals (left), and residuals compared to theoretical quantiles under a normal distribution. \label{fig:qq}}
\end{figure}

Although the histogram in \Cref{fig:qq} allows us to quickly assess whether the residuals follow a normal distribution, this can be visualized more precisely by comparing the theoretical quantiles of a normal distribution to the observed residuals, as in \Cref{fig:qq}, right.\footnote{This type of diagnostic plot can be generated easily with a library function, e.g.~this one was generated with \texttt{scipy.stats.probplot}.} The plot essentially compares the (sorted) residuals to those we would expect if we were to sample the same number of values from a normal distribution: if our residuals followed a normal distribution, plotting these quantities against each other would result in a straight line. Again, the plot basically reveals that there is an unusual outlier, and that residuals are missing the left tail (i.e., overpredictions) that would be expected. Note that this same type of diagnostic tool can be used to compare our residuals against any distribution in the same way.

While the above is merely a diagnostic for determining \emph{whether} the residuals followed a normal distribution, the more difficult question is how these discrepancies can be corrected. Some general guidelines are as follows:
\begin{description}
    \item[Remove outliers] The normal distribution (and thus the MSE) is especially sensitive to outliers due to how it penalizes large errors. To the extent that extremely long reviews do not conform to the usual behavior of the data, we could simply discard them before training.
    \item[Choose an error distribution less sensitive to outliers] For example, the Mean Absolute Error assigns a smaller penalty to large mispredictions, so outliers will have a smaller effect on the model.
    \item[Choose a skewed distribution] In this example we are predicting lengths, which by definition are bounded below (at length zero) but not above. Thus there will be a long tail of underpredctions, but not large overpredictions. We might account for this by modeling the data using a skewed probability distribution (such as a Gamma distribution.
    \item[Fit a better model] Note that the diagnostic in \Cref{fig:qq} is a function of the \emph{errors}, rather than the original data. Thus, for example, if we had a feature that allowed us to correctly predict the length of the unusually long review, the errors may again become more consistent with a normal distribution.
\end{description}

Again, most of the time the Mean Squared Error is a safe and reasonable choice, and can be used without too much scrutiny.
%\todo{Feel like I wrote this text elsewhere}
Nevertheless it is useful to have a sense of its underlying assumptions so that one can detect when they have been violated.

\section{Feature Engineering}

\subsection{Simple Feature Transformations}
\label{sec:simpleTransforms}

The first model we fit in \Cref{eq:sysOfEq0} revealed a positive association between review length and ratings. However, fitting the data with a line (\Cref{fig:ratingLengthBestFit}) does not seem to fit the data very accurately. Fitting the data with a line seems limiting, given that the trend may be better captured by a polynomial or asymptotic function (since the rating can not grow above five stars).

Na\"ively, we might think that this is a fundamental limitation of linear models. Note however that the assumption of linearity in $\theta$ (\Cref{eq:linearModel}) does not prevent us from fitting (for example) a polynomial function. The polynomial equation
\begin{equation}
  \text{rating} \simeq \theta_0 + \theta_1 \times (\text{review\ length}) + \theta_2 \times (\text{review\ length})^2
  \label{eq:sysOfEqPoly}
\end{equation}
is linear in $\theta$, even though we have transformed the input features in $X$.

This idea can be applied straightforwardly to fit polynomial functions, as shown in \Cref{fig:cubicFit}.\footnote{Actually, these curves were generated using the feature $\frac{\text{length}}{1000}$, as the matrix inverse $(X^TX)^{-1}$ becomes numerically unstable given large values of $\text{length}^3$.}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLengthBestFitQuad.pdf}
\caption{Quadratic and cubic polynomials of best fit. \label{fig:cubicFit}}
\end{SCfigure}

\subsection{Binary and Categorical Features: One-Hot Encodings}
\label{sec:binaryFeat}

So far we have dealt with regression problems where we have both real-valued inputs (features $X$), and real-valued outputs (labels $y$). What can we do in cases where features are binary or categorical?

As an example, let's consider whether the length of a user's review can be predicted by (or more simply, is related to) their gender. To do so, we'll look at a different dataset (of a few hundred beer reviews) that includes the gender of its users.

That is, we'd like a model of the form:
\begin{equation}
    \text{length} \simeq \theta_0 + \theta_1 \times \text{gender}.
    \label{eq:genderModel}
\end{equation}
Obviously, gender (represented in this dataset as a string) is not a numerical quantity, so we need some appropriate \emph{encoding} of the gender variable.

For the moment, let's treat gender as a binary variable. We'll relax this assumption in a moment to allow for a non-binary gender variable (and allow for the possibility that the gender is missing, as it can be in this dataset), but for the moment let's encode the gender variable as:
\begin{equation}
    \text{Male} = 0; \quad \text{Female} = 1.
    \label{eq:genderEncoding1}
\end{equation}
Alternately, this is just a binary indicator specifying whether this user is female. This encoding, although only one of a few we might have used, allows us to fit a linear model and estimate the values of $\theta_0$ and $\theta_1$. The model we fit (after removing users who did not specify a gender) is
\begin{equation}
    \text{length (in words)} \simeq 127.07 + 8.76 \times ( \text{user is female} ).
\end{equation}
With a little thought, we can interpret the model parameters as indicating that, on average, females write slightly longer reviews (by 8.76 words) compared to males. Note that 127.07 is not the population average, but rather the average for males (whose gender feature is zero); we will revisit how to interpret the parameters of these models in \Cref{sec:interpretLinear}.

A scatter plot of the data (i.e., the encoded gender attribute and the review lengths), as well as the line of best fit above is depicted in \Cref{fig:genderLength}. Note that although we have fit the data with a line (\Cref{fig:genderLength}, left), the actual feature values only occupy two points (0 and 1); thus the fit is perhaps better represented with a bar plot (\Cref{fig:genderLength}, right). 
%Note the fact that the model would predict (for example) a length of 1,000 words if the gender feature took a value of 100 does not invalidate it, as such values are never observed.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/genderVlength.pdf}
\includegraphics[scale=0.75]{fig/genderVlengthBar.pdf}
\caption{Gender versus review length (beer data). Visualized via a line of best fit (left) and a bar plot (right). \label{fig:genderLength}}
\end{SCfigure}

\subsubsection{Categorical Features}

%\todo{Probably this should be a subsection}
In practice, the gender attribute may assume more than binary labels in some datasets. To accommodate this, we might na\"ively imagine extending our encoding from \Cref{eq:genderEncoding1} to include additional values:
%\todo{Four category gender figure shouldn't have axis label `gender?'}
\begin{equation}
\begin{tabular}{ll}
    Male = & 0;\\
    Female = & 1;\\
    Other = & 2;\\
    Not specified = &3;\\
    etc.
    \end{tabular}
    \label{eq:oneHot1}
\end{equation}
Again we fit the same model as in \Cref{eq:genderModel}. Doing so we might obtain a fitted model like the one in \Cref{fig:genderFit4} (left).

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/genderVlengthBar4.pdf}
\includegraphics[scale=0.75]{fig/genderVlengthBar4oneHot.pdf}
\caption{Categorical features with a na\"ive sequential encoding (left), and a one-hot encoding (right). \label{fig:genderFit4}}
\end{SCfigure}

Note that the model fit in \Cref{fig:genderFit4} (left) implicitly makes some dubious assumptions. For example, because the model is linear, it assumes that the difference between `male' and `female' lengths is \emph{the same as} the difference between `female' and `other' lengths.
\footnote{That is, males receive the prediction $\theta_0$; females receive $\theta_0 + \theta_1$; other receives $\theta_0 + 2\theta_1$, etc.}

This assumption is not supported by the data, and in fact would be different if we simply reordered our indices in \Cref{eq:oneHot1}. Rather, we would like to associate different predictions to members of each group, as in \Cref{fig:genderFit4} (right). This can be achieved via a different encoding:
\begin{equation}
    \begin{tabular}{lll}
      Male = & $[0,0,0]$\\
      Female = & $[0,0,1]$\\
      Other = & $[0,1,0]$\\
      Not specified = & $[1,0,0]$\\
      etc.
    \end{tabular}
    \label{eq:oneHot2}
\end{equation}
%\todo{finish}
Using the above encoding, we can quickly confirm that the model would make predcitions as follows:
\begin{equation}
    \begin{tabular}{ll}
    Male: & $y = \theta_0$\\
    Female: & $y = \theta_0 + \theta_1$\\
    Other & $y = \theta_0 + \theta_2$\\
    Not specified & $y = \theta_0 + \theta_3$\\
    etc.
    \end{tabular}
\end{equation}
That is, $\theta_0$ is the prediction for males, $\theta_1$ is the \emph{difference} between females and males, etc. Note that we now have four parameters to estimate four values, as opposed to two parameters as in \Cref{eq:oneHot1}. As such, the model has sufficient flexibility to make different estimates for each group, as in \Cref{fig:genderFit4} (right).

This type of encoding, in which we have a separate feature dimension for each category, is called a \emph{one-hot} encoding.
%\todo{is this the first time I mention?}

Note that to represent four categories in \Cref{eq:oneHot2} we only used three-dimensional features (or in general, for $N$ categories, we could use an $(N-1)$-dimensional encoding. Possibly this seems slightly confusing compared to using a four-dimensional feature vector (e.g.~$\text{Male} = [0,0,0,1]$, etc.). Two reasons for using an $(N-1)$ dimensional feature vector are as follows:
\begin{itemize}
  \item Doing so is not necessary; together with $\theta_0$, the representation in \Cref{eq:oneHot2} uses four parameters to predict four values, so adding an additional dimension would add no more expressive power to the model and would be redundant.
  \item Doing so could possibly be harmful. While adding a redundant feature seems harmless, in practice it means the system of equations in \Cref{eq:?} would no longer have a unique solution, as the matrix $X^TX$ would be uninvertable (\Cref{exercise:?}).
\end{itemize}

Similarly, a \emph{multi-hot} encoding can be used in cases where an instance can belong to multiple categories simultaneously, for example for a `race' feature, a user may associate with multiple racial groups.
%\todo{crappy example}
Note that this is equivalent to a concatenation of several binary features.

\subsection{Missing Features}
\label{sec:missingFeat}

Often datasets will have features that are missing, for example the underlying data used for the example in \Cref{sec:binaryFeat} consisted of a gender attribute that many users may leave unspecified.

When dealing with binary or categorical features we dealt with these missing values quite straightforwardly---we simply treated `missing' as an additional category.

But if a continuous feature, such as a user's age or income, were missing, we must think harder about how to handle it. Trivially, we might simply discard instances with missing features, though this strategy will harm model performance if it means discarding a substantial fraction of our data.

Alternately we might replace the missing entries by the average (or mode) value for that feature; this strategy is known as `feature imputation.' This may be more effective than discarding the feature, but may also introduce some bias, as (for example) users who choose to leave a feature unspecified may be quite different from the average or mode.

To avoid the above issues, we would like a strategy that uses features when they are available, but makes separate predictions for those users when they are not. This can be achieved via the following strategy: for any feature $x$ which is sometimes missing, replace it by two features $x'$ and $x''$ as follows:
\begin{equation}
    x' = \left \lbrace
    \begin{tabular}{ll}
      1 & if feature is missing\\
      0 & otherwise
    \end{tabular} \right. , \quad
    x'' = \left\lbrace
    \begin{tabular}{ll}
      0 & if feature is missing\\
      $x$ & otherwise
    \end{tabular} \right. .
\end{equation}
Following this parameters can be fit within a model as usual:
\begin{equation}
    y = \theta_0 + \theta_1 x' + \theta_2 x''.
\end{equation}
The above representation may seem somewhat arbitrary, but makes sense once we expand the expression for missing and non-missing features. E.g.~when a feature is available predictions are made according to
\begin{equation}
    y = \theta_0 + \theta_2 x,
\end{equation}
whereas when a feature is missing predictions are made according to
\begin{equation}
    y = \theta_0 + \theta_1.
\end{equation}
This achieves the desired effect: when the feature is available we predict as normal, and when the feature is unavailable we predict using a learned value ($\theta_1$). Note that this strategy is very similar to feature imputation, but rather than using a heuristic imputation strategy, the model will directly learn what is the best prediction to impute.

\subsection{Temporal Features}
\label{sec:temporalFeatures}

Temporal features may make excellent predictors in various settings. Outcomes such as ratings, clicks, purchases (etc.)~are often influenced by factors such as the day of the week, the season, or long-term trends that span several years.

Let's explore an example in which we try to predict the rating of a book on Goodreads based on the day of the week that it was entered. Average ratings for each weekday\footnote{Again based on a small sample of reviews from the Fantasy genre.} are shown in \Cref{fig:temporalFit}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/timePlot.pdf}
\caption{Ratings as a function of the weekday, and line of best fit. \label{fig:temporalFit}}
\end{SCfigure}

As before, we might try to describe this relationship using a line, i.e.,~to fit a model of the form 
%\todo{make sure use of $=$ and $\simeq$ is consistent throughout this section}
\begin{equation}
    \text{rating} = \theta_0 + \theta_1 \times (\text{day of week}).
    \label{eq:fitDoW}
\end{equation}
For this equation to make sense, we need to map the day of the week to a numeric quantity. A trivial encoding might assign numbers sequentially, e.g.
\begin{equation}
    \text{Sunday} = 1; \quad \text{Monday} = 2; \quad \text{Wednesday} = 3; \quad \text{etc.}.
\end{equation}
Fitting \Cref{eq:fitDoW} using this representation yields the line of best fit depicted in \Cref{fig:temporalFit}, which reveals a slight upward trend as the days of the week progress.

The linear trend in \Cref{fig:temporalFit} seems a fairly poor fit to the data; we might think about fitting more complex functions like polynomials to better capture the observed data. But consider that our model is essentially periodic: Sunday (represented by a 1) follows Saturday (represented by a 7), though we could just as easily have represented Wednesday as 1 and Tuesday as 7. These choices seem arbitrary, but they impact our model in unexpected ways.

The above point is perhaps clearer if we visualize our model's predictions over a period of two weeks, as in \Cref{fig:periodic}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/timePlotPeriodic.pdf}
\caption{If we consider that our weekly measurements are periodic, we realize that fitting periodic data with a linear trend seems unrealistic. \label{fig:periodic}}
\end{SCfigure}

We'll revisit the critical role of temporal features in the context of recommendation in \Cref{sec:caseStudyNetflix}, and in 
%\todo{check this, add others}

\subsection{Transformation of Output Variables}

Finally, just as we saw how to transform features in \Cref{sec:simpleTransforms}, we can also transform our \emph{output} variables.

For example, let's consider fitting a model to determine whether resubmitted posts on \emph{reddit}
%\todo{Cite datasets somehow?}
receive lower numbers of upvotes, i.e.,
%long reviews on \emph{Steam} tend to be associated with games with a long play time, i.e.,
% \begin{equation}
%     \text{time played (hours)} = \theta_0 + \theta_1 (\text{review length}).
% \end{equation}
\begin{equation}
    \text{upvotes} = \theta_0 + \theta_1 \times (\text{submission number})
    \label{eq:reddit1}
\end{equation}
(where the `submission number' is `1' for an original resubmission, `2' for the first resubmission, etc.)
This model, along with the observations on which it is based, are shown in \Cref{fig:redditResub} (left).

% \begin{figure}
%     \centering
%     \includegraphics[scale=0.75]{fig/lengthPlayed.pdf}
%     \includegraphics[scale=0.75]{fig/lengthPlayedLog.pdf}
%     \caption{Review length versus hours played (left), and logarithm of hours played (right), from \emph{Steam} video game reviews. Lines of best fit for both plots are included.}
%     \label{fig:lengthPlayed}
% \end{figure}

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/redditUp.pdf}
    \includegraphics[scale=0.75]{fig/redditUpLog.pdf}
    \caption{Number of upvotes versus submission number on \emph{reddit}. The left plot shows the original data (with averaged upvote counts), the right plot shows the logarithm of the number of upvotes. Lines of best fit for both plots are included.}
    \label{fig:redditResub}
\end{figure}

Although the line of best fit indicates a slight downward trend, it does not appear to correspond closely to the overall shape of the data. 
%Part of the issue is that the data contains numerous \emph{outliers} (i.e.,~games with extremely large `time played' values), that the model is unable to reliably predict; as we discussed in \Cref{sec:whyMSE}, such large outliers do not fit the assumptions made when taking our line of best fit, and ultimately spoil the model's ability to capture a meaningful trend.
%\todo{Write this, or maybe not}
Possibly we might imagine that the data follows an exponentially decreasing trend, e.g.~every time you resubmit a post, you can expect to receive half as many upvotes.

Again, one might assume that this type of trend is something that cannot be captured by a linear model. But in fact we can possibly address this by transforming the output variable $y$. For example, consider fitting
\begin{equation}
    \log_2(\text{upvotes}) = \theta'_0 + \theta'_1 (\text{submission number})
    \label{eq:reddit2}
\end{equation}
%(note that we add a constant to ensure that the argument of the logarithm is always positive).
Now, a unit change in the prediction corresponds to a post receiving \emph{twice} as many upvotes.
%a game being played \emph{twice} as long. 
While this is still a linear model, the model corresponds to fitting
\begin{equation}
    \text{upvotes} = 2^{\theta'_0 + \theta'_1 (\text{submission number})}.
    \label{eq:reddit3}
\end{equation}
The transformed data and line of best fit are shown in \Cref{fig:redditResub} (right). 

Arguably, this second line better captures the overall trend, and does not have the same issues with outliers. 
%
In fact, if we transform the fitted values from \Cref{eq:reddit2} back to their original scale via \Cref{eq:reddit3}, the transformed values actually have a Mean Squared Error about 10\% \emph{lower} than the model from \Cref{eq:reddit1}, indicating that the transformed data more closely follows a linear trend compared to the untransformed data.
%\todo{mouthful}
%compute the Mean Squared Error of the models in \Cref{eq:reddit1} and \Cref{eq:reddit2}, the model from \Cref{eq:reddit3} has 



\section{Interpreting the Parameters of Linear Models}
\label{sec:interpretLinear}

When analyzing the linear models developed so far, we have already talked about interpreting their parameters in terms of general trends, correlation, differences between groups, etc.

While is tempting to casually interpret the meaning of various features, we must be careful and precise about doing so.

First, we should be precise about the interpretation of our slope and intercept terms. For example, when we modeled rating as a function of review length (\Cref{eq:reviewLength1}), we stated that under our model, ratings increased fractionally ($1.194\times 10^{-4}$) for every character of a review.

This interpretation makes sense given a model containing only a single features, but as soon as we incorporate multiple features we must be more careful. Consider e.g.~the model from \Cref{eq:thetaThree}, in which we included both the length and number of comments as predictors. We could no longer state that under this model, that the rating increases (by $7.243 \times 10^{-5}$) for every character in the review. Precisely, we must interpret the parameters as follows: \emph{Our prediction of the rating increases by $7.243\times 10^{-5}$ for every character in the review, assuming the other features remain unchanged}. This definition is stated precisely in \Cref{box:interpretLinear}.

\begin{floatingbox}
\caption{Interpreting the parameters of linear models \label{box:interpretLinear}}
%\todo-{Capitalize captions in boxes or not?}
Given a linear model $y = X\theta$ we should interpret a parameter $\theta_k$ as follows:

\emph{For every unit change in $x_{ik}$, our prediction of the output $y_i$ would increase by $\theta_k$ \emph{if all other features remain fixed}}

It is important to note that we are talking about the model's \emph{prediction} (rather than an actual change in the label), which could change if different features were included. And we must include the condition that other features remain constant. Without which we would fail to account for the correlations among different features.
\end{floatingbox}

Critically, features like review length and number of comments may be highly \emph{correlated}, such that we would rarely see longer reviews without also seeing more comments. For example, when incorporating features based on polynomial function (as in \Cref{eq:sysOfEqPoly}), or when dealing with one-hot encodings (as in \Cref{eq:oneHot1}), a feature generally \emph{cannot} change without the other features changing.

%\todo{clean up writing a little}
Second, we should be clear when interpreting parameters that we are talking about \emph{predictions} under a particular model rather than actual changes in the output variable $y_i$. These predictions can change as we include additional features, and a feature that had previously been predictive may be less predictive in the presence of another (as we saw in \Cref{eq:thetaThree}); likewise, we should be careful not to conclude that (e.g.)~length is \emph{not} correlated with the output, simply because another feature is \emph{better} correlated.

Finally, we should be careful not to make statements about the \emph{causal} effect of features on the output variable. Our line of best fit does not state that long reviews `cause' positive opinions any more than it states that positive opinions cause long reviews.

% Second, we should note that the 
% Consider a trivial example in which we predict a student's GPA as a function of their commute time:
% \begin{equation}
%     \mathrm{GPA} = 3.1 - 0.15 \times (\text{commute time to school in hours}).
%     %\todo{Could try a real example}
% \end{equation}
% From this one might casually conclude that long commutes contribute to lower GPAs. However if we fit a model that includes a third feature, hours employed, we might instead fit
% \begin{equation}
%     \mathrm{GPA} = 3.2 - 0.02 \times (\text{commute time}) - 0.08 \times (\text{hours worked})
% \end{equation}


\section{Fitting Models with Gradient Descent}
\label{sec:gradient}

When solving regression problems in the previous section, we looked for \emph{closed form} solutions. That is, we set up a system of equations (\Cref{eq:linearTheta}) in $\theta$, and attempted to solve them for $\theta$ (albeit approximately via the pseudoinverse).

As we begin to fit more complex models
%\todo{where does regularization go?} 
including in \Cref{sec:classification,sec:regularization}, a closed-form solution may no longer be available.

\emph{Gradient descent} is an approach to search for the minimum value of a function, by iteratively finding better solutions based on an initial starting point. The process (depicted in \Cref{fig:gradient}) operates as follows: 
%\todo{algorithmic environment}
\begin{enumerate}
    \item Start with an initial guess for $\theta$
    \item Compute the derivative $\frac{\partial}{\partial \theta}f(\theta)$. Here $f(\theta)$ is the MSE (or whatever criterion we are optimizing) under our model $\theta$.
    \label{step2}
    \item Update our estimate of $\theta \coloneqq \theta - \alpha \cdot f'(\theta)$
    \label{step3}
    \item Repeat Steps \ref{step2} and \ref{step3} until convergence.
    \label{step4}
\end{enumerate}
During each iteration, the process now follows the path of steepest descent, and will gradually arrive at a minimum of the function $f_\theta$.\footnote{Assuming the function is `well-behaved,' e.g.~the objective is bounded below, and the function is differentiable everywhere; though these are rarely issues when dealing with error functions specifically.}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/gradient.pdf}
%\todo{Figure has problems (e.g. f(heta)), and annotation looks funny}
\caption{Gradient descent demonstration. \label{fig:gradient}}
\end{SCfigure}

The above is a simple description of the procedure that omits many details. We omit some of the specifics for now and save them until we implement the procedure in practice, 
%\todo{pick some way to reference code from the text}
though some of the main issues include:
\begin{itemize}
    \item Given the starting point in \Cref{fig:gradient}, the algorithm would only achieve a \emph{local} rather than a \emph{global} optimum. To address this we could investigate ways to come up with a better initial `guess' of $\theta$, or later,
    %\todo{mentioned later?} 
    we could investigate alternative optimization approaches less susceptible to local minima.
    \item The step size $\alpha$ (Step~\ref{step3} above) must be chosen carefully. If $\alpha$ is too small, the procedure will converge very slowly; if $\alpha$ is too large, the procedure may `overshoot' the minimum value and obtain a \emph{worse} solution during the next iteration. Again, other than carefully tuning this parameter, we could investigate optimization methods not dependent on choosing this rate (see e.g.~quasi-Newton methods such as L-BFGS \cite{liu1989limited}).
    \item `Convergence' as defined in Step~\ref{step4} is not well-defined. We might define convergence in terms of the change in $\theta$ (or $f_\theta(X)$) during two successive iterations, or alternately we may terminate the algorithm once we stop making progress on held-out (validation) data, as in \cref{sec:validation}.
\end{itemize}

\subsection{Linear Regression via Gradient Descent}

To solidify the ideas above, let's consider the specific example of minimizing the Mean Squared Error of a linear model, i.e.,
%\todo{Consistent way of writing $(y - x_i \cdot \theta)$, e.g. does $y$ go first. Compare to $R^2$ equation}
\begin{equation}
    f_\theta(y;X) = \frac{1}{|y|} \sum_{i = 1}^{|y|} (x_i \cdot \theta - y_i)^2.
\end{equation}
The
%\todo{equation above has both $X$ and $y$. This should be corrected everywhere}
derivative $f'(\theta)$ can be computed as follows:
%\todo{Check that I'm writing $X$ when I mean $X$ and $x_i$ when I mean $x_i$}
\begin{equation}
   \frac{\partial f}{\partial \theta_k} = \frac{1}{|y|} \sum_{i = 1}^{|y|} 2 x_{ik} (x_i \cdot \theta - y_i).
\end{equation}
Note that the above is a \emph{partial} derivative in $\theta_k$, which must be computed for each feature dimension $k = \lbrace 1 \ldots K \rbrace$.
%\todo{put in notation table}
The above derivative is more obvious after expanding $x_i \cdot \theta = \sum_{k=1}^K x_{i,k}\theta_k$.



\section{Non-linear Regression}
\label{sec:nonlinear}

So far, we have limited our discussion to models of the form $y = X\theta$, mostly because these offered us a convenient (closed form) solution to finding lines of best fit in terms of $\theta$.

However, this type of model has several limitations that we might wish to overcome, such as:
\begin{itemize}
    \item We cannot incorporate simple constraints on our parameters, such as that a certain parameter should be positive, or that one parameter is larger than another (which might be based on domain knowledge of a certain problem).
    \item Although we can manually engineer non-linear transforms of our features (as we did in \Cref{sec:simpleTransforms}, we cannot have the model learn these non-linear relationships automatically.
    \item The model cannot learn complex \emph{interactions} among features, e.g.~that length is correlated with ratings, but only if the user is female.\footnote{To be precise, the linear model does consider relationships among features in the limited sense that parameters for one feature will change in the presence of other correlated features (\Cref{sec:interpretLinear}); and, the model \emph{could} capture relationships between (e.g.)~gender and length if we were to manually engineer a feature describing this relationship. Our point here is about whether the model can learn these relationships automatically.}
\end{itemize}

The above goals can potentially be realized if we are allowed to transform model \emph{parameters}: for instance, we could ensure that a particular parameter was always positive by fitting
\begin{equation}
    \theta_k = \log(1 + e^{\theta_k'})
\end{equation}
(this is known as a `softplus' function; note that this function smoothly maps $\theta_k' \in \mathbb R$ to $\theta_k \in (0,\infty)$); or if we wanted one feature to be larger than another (e.g.~$\theta_k > \theta_j$) we could simply add the positive quantity above to another feature:
\begin{equation}
    \theta_k = \theta_j + \log(1 + e^{\theta_k'}).
\end{equation}

Although it goes beyond our discussion here, fitting these types of non-linear parameterizations (and especially transformations that deal with complex combinations of parameters) is largely the goal of \emph{deep learning}.

We will see examples of non-linear models in later chapters (e.g.~\Cref{sec:rnn},\Cref{sec:modelbased}) though for the moment we briefly discuss

%\todo{double check code}
%\todo{only useful if there's comparible code elsewhere}
\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
y = tf.constant(y, shape=[len(y),1]) # Convert to column vector
M = len(X[0]) # Feature dimensionality

# Regularized MSE, described using tensorflow matrix operations
def MSE(X, y, theta, lambda):
  return tf.reduce_mean((tf.matmul(X, theta) - y)**2) +
         lambda * tf.reduce_sum(theta**2)

# Initialize theta
theta = tf.Variable(tf.constant([0.0]*M, shape=[M,1]))

# Adam Optimizer
optimizer = tf.train.AdamOptimizer(0.01)
objective = optimizer.minimize(MSE(X,y,theta,1.0))
\end{lstlisting}
\end{minipage}

\subsection{Case Study: Image Popularity on Reddit}

\section{The Learning Pipeline}

\subsection{Generalization and Overfitting}

So far, when discussing model evaluation in \Cref{sec:evaluateRegression}, we've considered training a model to predict labels $y$ from a dataset $X$; we've then evaluated the model by comparing the predictions $f(x_i)$ to the labels $y_i$. Critically, we're using the \emph{same} data to train the model as we're using to evaluate it.

The risk in doing so is that our model may not \emph{generalize} well to new data. For example, when fitting a model relating review length to ratings (as in \Cref{fig:ratingLengthBestFit?}), increasing the degree of the polynomial would continue to lower the errors of the predictor; alternately, we could have modeled review length using a one-hot encoding (so that there was a different predicted value for every length). In spite of their low MSEs (or high $R^2$ coefficients), these models would not seem to meaningfully capture the \emph{trends} in the data; rather, they are effectively `memorizing' meaningless details in the observed data.

To consider an extreme case, imagine fitting a vector $y$ using only \emph{random} features. The code below fits a vector of 50 observations using 1, 10, 25, and 50 random features, and then prints the $R^2$ coefficient of each model:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
y = numpy.random.rand(50)
mod = linear_model.LinearRegression()
for n in [1,10,25,50]:
    X = numpy.random.rand(50,n)
    mod.fit(X,y)
    print(mod.score(X,y))
\end{lstlisting}
\end{minipage}
Here, the $R^2$ coefficients take values of 
% 0.07321723526307
% 0.24827931383412216
% 0.3478616991354071
% 1.0
0.07, 0.25, 0.35, and 1.0---once we include 50 random features, we can fit the data perfectly. Of course, given that our features were \emph{random}, this `fit' is not meaningful, and the model has merely discovered random correlations between the observed data and labels.

The above points to two issues that need to be addressed when training a model:
\begin{enumerate}
    \item We should not evaluate a model on the same data that was used to train it. Rather we should use a held-out dataset (i.e.,~a \emph{test} set).
    \item Features that improve performance on the training data will not necessarily improve performance on the held-out data. 
\end{enumerate}
%\todo{Do I ever use enumerates or always itemize?}
Evaluating a model on held-out data gives us a sense of how well we can expect that model to work `in the wild.' Thus we are measuring how well our model will \emph{generalize} to new data.

\paragraph{Overfitting}
%\todo{Don't like the paragraph name since it matches the subsection name}

Fundamentally, if our model works well on a training dataset but not on held-out data, it must mean that the training data is somehow not representative of the held-out data. For instance, if we had held-out sales data from the most recent month, and trained on data from the previous eleven, it is possible that the most recent month of observations follow a different trend, or occurs during a different season, etc.

In principle, one can might address the above simply by ensuring that the training and test sets
%\todo{have I defined test set?}
are (non-overlapping) \emph{random samples} of the data, such that both the training and test data will be drawn from the same distribution.

%Addressing the second point above, if a model works well on the training data but not on held-out data, 
we are said to be \emph{overfitting}.

\begin{figure}
    \centering
\begin{tikzpicture}
\draw (0,1.6) rectangle (10,1.1) node[pos=.5] {Dataset $\lbrace X; y \rbrace$};
\draw (0,1) rectangle (5,0.5) node[pos=.5] (train) {Training Set};
\node [below=of train,yshift=3em] {Estimate $\theta$};
\draw (5.125,1) rectangle (7.375,0.5) node[pos=.5] (valid) {Validation Set};
\node [below=of valid,yshift=3em] {Choose $\lambda$};
\draw (10,1) rectangle (7.5,0.5) node[pos=.5] (test) {Test Set};
\node [below=of test,yshift=3em] {Evaluation only};

\end{tikzpicture}
    \caption{Basic roles of training, validation, and test sets.}
    \label{fig:trainValidTest}
\end{figure}




\subsection{Model Complexity and Regularization}

So far we have talked vaguely about what it means for a model to be `too complex' and suggested that we should choose a model that is complex enough to fit the data, but simple enough not to overfit.
%\todo{have we actually discussed those things?}
This idea is often referred to as \emph{Occam's Razor}, a philosophical principle which states that among several alternate hypotheses that explain some phenomenon, one should favor the simplest.

However for these notions to be useful we must be precise about what it means for a model to be `complex.' We would like to define complexity in terms of the parameters $\theta$, such that given a fixed set of features and labels, we could select the `simplest' $\theta$ that adequately explains (or models) the data.

We'll discuss two candidate notions of `simplicity' as follows:
\begin{itemize}
    \item A simple model is one in which all terms are about equally important, i.e.,~one in which all terms $\theta_k$ are about equally large.
    \item A simple model is one that includes only a few terms, i.e.,~in which only a few values $\theta_k$ are non-zero.
\end{itemize}
Below we'll argue that these two notions of `complexity' are captured by the following expressions:
\begin{eqnarray}
    \Omega_2(\theta) & = \| \theta \|_2^2 & = \sum_k \theta^2_k\\
    \Omega_1(\theta) & = \| \theta \|_1 & = \sum_k | \theta_k |,
\end{eqnarray}
i.e.,~the sum of squares and sum of absolute values.
%\todo{Finish}
%Explanation:
%https://stats.stackexchange.com/questions/45643/why-l1-norm-for-sparse-models

In order to fit a model which simultaneously explains the data but is not overly complex (corresponding to our goal above), we write down a new objective that combines our original accuracy objective from \Cref{eq:?} with one of the complexity expressions above:
\begin{equation}
    \underbrace{\frac{1}{|y|}\sum_{i = 1}^{|y|} (x_i \cdot \theta - y_i)^2}_{\text{accuracy}} + \lambda \underbrace{\sum_k \theta^2}_{\mathclap{\text{model complexity}}}
\end{equation}

Note not generally speaking the term $\theta_0$, i.e.,~the \emph{offset} term, should not be included in the regularizer, i.e.,~our regularizer should be $\sum_{k=1}^K \theta_k^2$ or $\sum_{k=1}^K |\theta_k|$. That is, our underlying assumption that few parameters are non-zero, or that parameters are small, should not apply to the offset term. If we were to include the offset term when regularizing, we would generally select a model which made systematically smaller (in magnitude) predictions.
%\todo{cite exercise}

\begin{floatingbox}
\caption{Guidelines for building training, validation, and tests sets}
\begin{itemize}
    \item Training, validation, and test sets should (generally speaking) be non-overlapping, random samples of a dataset. Some exceptions apply, for example when fitting temporal models in \Cref{chap:temporal}, we might build a test set out of the \emph{most recent} observations in order to get a sense of how well a model would work \emph{now} rather than how well it would work \emph{on average}.
    \item The size of our training set may be driven by modeling as well as practical concerns. Our training set should be large enough that we can reasonably expect to fit our model on the data (as a guideline, we might hope to have an order of magnitude more training examples than model parameters); likewise if we have a simple model with just a few parameters we need not train on millions of observations.
    \item Likewise the size of our validation and test sets should be large enough that we can be reasonably confident of our results.
    %\todo{Reference some section on this}
\end{itemize}
\label{box:validationGuidelines}
\end{floatingbox}
%\todo{finish}


\paragraph{`Theorems' Regarding Training, Testing, and Validation Sets}



\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/validation.pdf}
    \caption{Example train, validation, and test curves, demonstrating the relationships between each type of error.}
    \label{fig:trainValidCurves}
\end{figure}

\begin{itemize}
    \item The training error increases as $\lambda$ increases; typically for large $\lambda$ it will asymptote to some value, e.g.~a linear model might asymptote to the error of a trivial predictor (i.e.,~the variance of the label).
    \item The validation and test errors will be at least as high as the training error; intuitively, the algorithm will not work better on `unseen' data than it did on training data.
    \item When $\lambda$ is too small, a too-complex model achieves low training error, but high validation/test errors. In this case, the model is said to be \emph{overfitting}.
    \item When $\lambda$ is too large, a too-simple model has high training, validation, and test error. In this case, the model is said to be \emph{underfitting}.
    \item Generally, there should be a `sweet spot' between under and overfitting, which is determined using our validation set. This point (marked in \Cref{fig:validation} with an `x') corresponds to the model we expect to yield the best generalization performance on the test set.
\end{itemize}


\subsection{Guidelines for Model Pipelines}



\paragraph{Uses of a Validation Set}
%\todo{Mention using the validation set to terminate gradient updates}
The core use of our validation set is to estimate model hyperparameters such as $\lambda$ above. Beyond regularization coefficients, `hyperparameters' more broadly refer to any tunable model components that do not get optimized during the training phase. For example, when building models from text in \Cref{chap:text}, the number of words in our dictionary (from which we build features) would be an example of a hyperparameter, and could be chosen using our validation set to select the model that will generalize the best.

Our validation set can also be used when training a model. For instance, when we fit models via gradient descent (as in \Cref{sec:gradient}), we might periodically evaluate the model's validation performance every few gradient iterations; rather than waiting until the model reaches a particular convergence threshold, we can simply terminate once performance is no longer improving on the validation set.

\subsection{Significance Testing}

\section{Classification}
\label{sec:classification}

So far, we have considered supervised learning tasks in which the output variable $y$ is a real number, i.e.,~$y \in \mathbb R$. Often, we will deal with problems with binary or categorical output variables, for example we might be interested in problems such as:
\begin{itemize}
  \item Will a user click on a product or advertisement? (binary outcome)
  \item What category of object does an image contain? (multiclass)
  \item What product is a user most likely to purchase next? (multiclass)
  \item Which of two products would a user prefer? (binary)
\end{itemize}

\subsection{Logistic Regression}
\label{sec:logisticRegression}

We will focus on a form of classifier that extends the ideas behind regression to classification problems.
%\todo{Move the figure below}


Logistic Regression sets up classification using a probabilistic framework; the predictions $X\theta$ that we used when building regressors are transformed into \emph{probabilities} of observing a particular label.

Ultimately, logistic regression is just one of dozens of classification schemes; we describe it here rather than other popular alternatives (such as Support Vector Machines \cite{cortes1995support}, or Random Forest Classifiers \cite{ho1995random}) mainly because logistic regression more closely matches the approaches we will develop in later chapters.

In short, logistic regression associates a probability of the dataset under some model, which is fully differentiable and can be optimized using gradient-based approaches (as we saw in \Cref{sec:whyMSE,sec:gradient}); 
%\todo{check}
this same type of modeling will be used when building Recommender Systems in \Cref{chap:recsys}, or ? in \Cref{sec:}.
%\todo{finish}

When fitting regular linear regressors, we wanted a model $f_\theta$ whose estimates $f_\theta(X_i)$ were as close as possible to the (real-valued) labels $y_i$. When adapting a linear regression algorithm  to classification, we might instead seek models that associate positive values of $X_i \cdot \theta$ with positive labels ($y_i = 1$), and negative values of $X_i \cdot \theta$ with negative labels ($y_i = 0$).

If we could do so, we could write down the \emph{accuracy} associated with a particular model:
%\todo{renders ugly}
\begin{equation}
\frac{1}{|y|}
    \sum_{i=1}^{|y|} \underbrace{\delta(y_i = 0) \delta(X_i\cdot \theta \leq 0)}_{\hspace{-15mm}\mathrlap{\text{label is positive and prediction is positive}}} + \overbrace{\delta(y_i = 1) \delta(X_i\cdot \theta > 0)}^{\mathllap{\text{label is negative and prediction is negative}}\hspace{-15mm}}
    \label{eq:accuracySummation}
\end{equation}
(here $\delta$ is an indicator function that returns $1$ if the argument is true, $0$ otherwise). The equation above, in spite of slightly confusing notation, is merely counting the number of times we correctly predict a positive score for positively labeled data, and a negative score for negatively labeled data.

We now simply desire from our classifier $\theta$ that it maximizes the accuracy in \Cref{eq:accuracySummation}.
Unfortunately, directly optimizing the accuracy in \Cref{eq:accuracySummation} is NP-hard (see e.g.~\cite{nguyen2013algorithms}). To get a sense for why it is difficult, consider that the function in \Cref{eq:accuracySummation} is essentially a \emph{step} function (\Cref{fig:sigmoid}, left), i.e.,~it is flat (deravitive zero) almost everywhere; it is therefore not amenable to techniques like gradient ascent.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/step.pdf}
\includegraphics[scale=0.75]{fig/sigmoid.pdf}
\caption{The sigmoid function (right), and the step function (left) that it approximates. \label{fig:sigmoid}}
\end{SCfigure}

So, to optimize the accuracy approximately, we would like a function that is \emph{similar to} \Cref{eq:accuracySummation}, but is more straightforward to optimize.

\emph{Logistic Regression} achieves this goal by converting the outputs of a linear function $X_i \cdot \theta$ to \emph{probabilities} via a smooth function. Our intuition is that large values of $X_i \cdot \theta$ should correspond to high probabilities, and small (i.e.,~large negative) values of $X_i \cdot \theta$ should correspond to small probabilities.

This goal can be achieved via the \emph{sigmoid function}:
\begin{equation}
    \sigma(x) = \frac{1}{1 + e^{-x}}.
\end{equation}
This function, depicted in \Cref{fig:sigmoid}, maps a real value to the interval $(0,1)$, and passes through 0.5 when $x=0$. Thus it can be interpreted as a probability:
\begin{equation}
p_\theta(y_i = 1|X_i) = \sigma(X_i \cdot \theta) = \frac{1}{1 + e^{-X_i\cdot \theta}}
\end{equation}
Now, as a smooth surrogate for the expression in \Cref{eq:accuracySummation}, we can instead optimize
%TODO: Rewrite the previous function so that it uses this notation rather than delta functions
\begin{align}
    \mathcal{L}_\theta(y|X) & = \prod_{y_i = 1} p_\theta(y_i = 1|X_i) \times \prod_{y_i = 0} \left(1 - p_\theta(y_i = 0|X_i)\right)\\
    & = \prod_{y_i = 1} \frac{1}{1 + e^{-X_i\cdot \theta}} \times \prod_{y_i = 0} \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_i\cdot \theta}}.
\end{align}
% Our goal is to maximize this expression for $\theta$, i.e., to find
% \begin{equation}
%     \argmax_\theta L_\theta(y|X)
% \end{equation}
The above expression is a \emph{likelihood function}, much like we saw in \Cref{eq:likelihood1}.
Intuitively, for this expression to be maximized we want positive instances ($y_i = 1$) to be associated with high probabilities, and negative instances ($y_i = 0$) with low probabilities.

Our goal is to maximize the above function, i.e.,~to find $\argmax_\theta L_\theta(y|X)$. Short of a closed form solution, our approach is as follows:
\begin{itemize}
    \item Take the logarithm $\ell_\theta(y|X)$, since $\argmax_\theta \mathcal{L}_\theta(y|X) = \argmax_\theta \log(\mathcal{L}_\theta(y|X))$.
    \item \emph{Subtract} a regularizer $\lambda \| \theta \|_2^2$ (or similar).\footnote{Note that we subtract the regularizer rather than adding it as we did in \Cref{?} since we are maximizing rather than minimizing our objective; since we still want low complexity, we seek to maximize $-\lambda \| \theta \|_2^2$.}
    \item Compute the gradient of $\ell_\theta(y|X) - \lambda \| \theta \|_2^2$, and optimize via gradient ascent.
\end{itemize}
We compute the gradient below as follows:
\begin{align}
\ell_\theta(y|X) & = \sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-X_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right) - \lambda \| \theta \|_2^2 \label{eq:logisticDerive1}\\
 & = \sum_{i} - \log(1 + e^{-X_i\cdot \theta}) + \sum_{y_i = 0} -X_i \cdot \theta - \lambda \| \theta \|_2^2 \label{eq:logisticDerive2}\\
\frac{\partial \ell}{\partial \theta_k} & = \sum_i x_{ik} \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_i \cdot \theta}} - \sum_{y_i = 0} x_{ik} - \lambda\theta_k\\
& = \sum_i x_{ik} \left( 1 - \sigma(X_i \cdot \theta) \right) - \sum_{y_i = 0} x_{ik} - \lambda \theta_k
\end{align}
(note the summation indices change between
%\todo{Why do the references below point to the wrong thing?}
\Cref{eq:logisticDerive1,eq:logisticDerive2}, since both terms in \Cref{eq:logisticDerive1} have the same denominator).
%\todo{Finish this section}

%The basic idea behind logistic regression is that each observation $X_i$ 
%%\todo{notation?} that has a positive label ($y_i = 1$) should be associated

\section{Evaluating Classification Models}
\label{sec:evaluatingClassifiers}

So far, when developing classifiers, we have focused on maximizing the alignment between the labels and the model's outputs. E.g.~in the case of logistic regression, we want the predicted probability $p_\theta(y_i = 1 | x_i)$ to be as close as possible to the label $y_i$. Implicitly, when doing so, we are trying to maximize the model's \emph{accuracy}:
\begin{equation}
%\todo{f_\theta|y,X is not the notation I use elsewhere
    \text{accuracy}(f_\theta|y,X) = \frac{1}{N} \sum_{i=1}^{|y|} \delta ( f_\theta(x_i) = y_i ),
\end{equation}
where $\delta$ is an indicator function,\footnote{
%\todo{put this wherever delta is first used, or just in notation page}
$\delta(x) = 1$ if the argument $x$ is true, $0$ otherwise.}
and $f_\theta(x_i)$ is the binarized output of the model (e.g.~in the case of logistic regression, $f_\theta(x_i) = \delta(x_i \cdot \theta > 0)$). Equivalently we are minimizing the \emph{error}, i.e.,
\begin{equation}
    \text{error}(f_\theta|y,X) = 1 - \text{accuracy}(f_\theta|y,X).
\end{equation}
%\todo{could drop or inline error definition}

To motivate the difficulty of properly evaluating classifiers, consider the following classification task. We saw in \Cref{fig:genderLength} that there was a slight relationship between gender and review length; now, let's see if we can develop a simple classifier that attempts to predict gender based on review length:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = [[1, len(d['review/text'])] for d in data]
y = [d['user/gender'] == 'Female' for d in data]

mod = sklearn.linear_model.LogisticRegression()
mod.fit(X,y)
predictions = mod.predict(X) # Binary vector of predictions
correct = predictions == y # Binary vector indicating which predictions were correct
print(sum(correct) / len(correct))
\end{lstlisting}
\end{minipage}
%\todo{This probably shouldn't be the first time code is used in this chapter}

Surprisingly, the classifier produced by this code is 98.5\% accurate!
%\todo{Don't know if I want exclamation marks}
This result might seem implausible, but turns out to be an issue with the error measure itself. Counting the number of negative labels in the dataset

% \noindent
% \begin{minipage}{\linewidth}
% \begin{lstlisting}
% 1 - (sum(y) / len(y))
% \end{lstlisting}
% \end{minipage}
reveals that the data is 98.5\% male (i.e.,~98.5\% negative labels). Not only does this reveal that the accuracy is unlikely to be an informative metric in this case, but it reveals that \emph{our goal of optimizing the accuracy} caused us to learn a trivial classifier---the model simply predicts zero everywhere.

The above example demonstrates the problem with na\"ively computing (or optimizing) the accuracy. Several situations where we might need more nuanced evaluation measures include:
\begin{itemize}
    \item Datasets whose labels are highly imbalanced, such as the example above.
    \item Situations where different types of errors have different associated costs. E.g.~failing to detect dangerous luggage is a more severe mistake than an erroneous positive identification.
    \item When we use classifiers for search or retrieval (as we will often do when developing recommender systems), we often care about the ability of the model to confidently identify a few positive instances (e.g.~those surfaced on a results page), and are not interested in its overall accuracy.
\end{itemize}

Below we will develop error measures designed to handle each of these scenarios.

%\subsection{Type I and Type II Errors}
\subsection{Balanced Metrics for Classification}
\label{sec:type1type2}

The basic issue with the example presented above was that we allowed one of the two labels to dominate the classifier's objective. Although in some cases we may justifiably want a classifier that focuses more on the dominant label, in the example above we would likely prefer a solution that had reasonable accuracy \emph{per class}.

To achieve this we need evaluation metrics that consider the two classes (positive and negative, or female and male in our example) separately. To do so, we consider each of the four possible outcomes in terms of our prediction and label:
\begin{alignat}{3}
  \mathit{TP} & =\text{True Positives} &&= |\lbrace i | y_i \land f_\theta(x_i) \rbrace| \\
  \mathit{TN} & =\text{True Negatives} &&= |\lbrace i | \lnot y_i \land \lnot f_\theta(x_i) \rbrace| \\
  \mathit{FP} & =\text{False Positives} &&= |\lbrace i | \lnot y_i \land f_\theta(x_i) \rbrace| \\
  \mathit{FN} & =\text{False Negatives} &&= |\lbrace i | y_i \land \lnot f_\theta(x_i) \rbrace|.
  %\todo{No understanding of alignat above, and should fix equation below too}
\end{alignat}
From these, we can define errors (or accuracies) that consider each of the two classes in isolation:
%\todo{Replace by array environment and add more &'s before the last equal, and in equation above to properly align}
\begin{eqnarray}
 \mathit{TPR} & = \text{True Positive Rate} & = \frac{|\text{true positives}|}{|\text{labeled positive}|} = \frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}\\
 \mathit{TNR} & = \text{True Negative Rate} & = \frac{|\text{true negatives}|}{|\text{labeled negative}|} = \frac{\mathit{TN}}{\mathit{TN} + \mathit{FP}}\\
 \mathit{FPR} & = \text{False Positive Rate} & = \frac{|\text{false positives}|}{|\text{labeled negative}|} = \frac{\mathit{FP}}{\mathit{FP} + \mathit{TN}}\\
 \mathit{FNR} & = \text{False Negative Rate} & = \frac{|\text{false negatives}|}{|\text{labeled positive}|} = \frac{\mathit{FN}}{\mathit{FN} + \mathit{TP}}.
 %\todo{Probably change the order of the expressions in the denominators to be more consistent}
\end{eqnarray}
Note that it is trivial to optimize any one of these criteria in isolation (e.g.~we can achieve a True Positive Rate of 1.0 simply by always predicting positive). As such, we would normally optimize a criterion which considers both positive and negative labels together. One such measure is the \emph{Balanced Error Rate}, which simply takes the average of the False Positive and False Negative rates:
\begin{equation}
    \mathrm{BER}(f_\theta | y,X) =
    \frac{1}{2}(\mathit{FPR} + \mathit{FNR}) = 1 - \frac{1}{2}(\mathit{TPR} + \mathit{TNR}).
    % \frac{1}{2} \Bigl(
    %     \underbrace{\frac{\mathit{FN}}{\mathit{TP} + \mathit{FN}}}_{\mathit{FNR}} +
    %     \underbrace{\frac{\mathit{FP}}{\mathit{TN} + \mathit{FP}}}_{\mathit{FPR}}
    %     \Bigr)
    %\todo{Don't use Bigl, just use \smash and \vphantom}
    % =
    % 1 - \frac{1}{2} \Bigl(
    %     \underbrace{\frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}}_{\mathit{TPR}} +
    %     \underbrace{\frac{\mathit{TN}}{\mathit{TN} + \mathit{FP}}}_{\mathit{TNR}}
    %     \Bigr)
\end{equation}
In our motivating example, this now attributes half of the error to the `Female' (positive) class and half of the error to the `Male' (negative) class.

Note that an appealing quality of the Balanced Error Rate is that (unlike the accuracy), it can no longer be minimized via trivial solutions: always predicting `True,' or always predicting `False,' or predicting at random, will result in a BER of 0.5.

\subsection{Optimizing the Balanced Error Rate}

Having argued
%(\Cref{sec:type1type2})
that the Balanced Error Rate may be preferable to the accuracy if we wish to avoid trivial solutions, we next ask how to train a classifier to avoid producing trivial solutions in the first place.

Intuitively, the degenerate solutions we saw in 
\Cref{sec:}
%\todo{this}
arose due to an imbalance in our training data (i.e.,~a high ratio of positive or negative labels). Trivially, we might correct this by \emph{re-sampling} our training data: i.e.,~sampling either a fraction of our negative negative instance, or sampling negative instances (with replacement) until we have an equal number of positive and negative instances.

While the above is a common and reasonably effective strategy, the same goal can be achieved more directly simply by \emph{weighting} the positive and negative instances. Note that in our objective for logistic regression:
\begin{equation}
    \sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-X_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-X_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right),
\end{equation}
the two summations (over $y_i = 1$ and $y_i = 0$) essentially reward the model for correctly predicting positive instances (i.e.,~True Positive Rate) and negative instances (i.e.,~True Negative Rate). The issue with the above objective is that it will be imbalanced, and one of the two terms will dominate the expression, in the event that positive or negative instances are over-represented in our dataset.

To address this, we can normalize the two expressions by the number of samples in the positive and negative classes:
\begin{equation}
    \frac{|y|}{2|\lbrace i | y_i = 1 \rbrace|}\sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-X_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-X_i\cdot \theta}}{1 + \frac{|y|}{2|\lbrace y | y_i = 0 \rbrace|} e^{-X_I\cdot \theta}} \right).
\end{equation}
%\todo{kinda sloppy notation}
%\todo{finish}
Note that in addition to normalizing, both sides are multiplied by $\frac{|y|}{2}$; this is not strictly necessary but is done by convention such that the total `weight' of all instances is still $|y|$.

The above can be accomplished in sklearn with the \texttt{class\_weight='balanced'}
%\todo{do I ever use texttt anywhere?}
option as follows:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = [[1, len(d['review/text'])] for d in data]
y = [d['user/gender'] == 'Female' for d in data]

mod = sklearn.linear_model.LogisticRegression(class_weight='balanced')
mod.fit(X,y)
\end{lstlisting}
\end{minipage}

Note that the same idea can be applied to problems including more than two categories, and that one can choose different weighting schemes, e.g.~to assign any desired relative weights to true positives versus true negatives.

\subsection{Error Metrics for Ranking}

% Precision and recall, precision@k

Often, the goal of training a classifier is not merely to generate exhaustive sets of `true' and `false' instances. For example, if we wanted to identify relevant webpages in response to a query, or to recommend items that a user is likely to purchase, in practice it may not matter whether we can identify \emph{all} relevant webpages or products; rather, we might care more about whether we can surface \emph{some} relevant items among the first page of results returned to a user.

Note that the type of classifiers we've developed so far can straightforwardly be used for \emph{ranking}. That is, in addition to outputting a label ($y_i = x_i \cdot \theta > 0$ in the case of logistic regression), they can also output \emph{confidence scores} (i.e.,~$x_i \cdot \theta$, or equivalently $p_\theta(y_i = 1 | x_i) > 0.5$).
%\todo{check notation}
Thus, in the context of finding relevant webpages or products above, our goal might be to maximize the number of relevant items returned among the few most confident predictions. Furthermore, we might be interested in how the model's accuracy changes as a function of confidence; e.g.~even if the model's accuracy is low overall, is it accurate for the top 1\%, 5\%, or 10\% of most confident predictions?

\paragraph{Precision and Recall}
%\todo{Don't like subsubsections}

\emph{Precision} and \emph{Recall} assess the quality of a set of retrieved results

\begin{eqnarray}
    \text{Precision} & = & \frac{| \lbrace \text{relevant items} \rbrace \cap \lbrace \text{retrieved items} \rbrace |}{|\lbrace \text{retrieved items} \rbrace|}\\
    \text{Recall} & = & \frac{| \lbrace \text{relevant items} \rbrace \cap \lbrace \text{retrieved items} \rbrace |}{|\lbrace \text{relevant items} \rbrace|}
    \label{eq:precRecDefn1}
\end{eqnarray}

\begin{eqnarray}
  \text{Precision} & = & \frac{\mathit{TP}}{\mathit{TP} + \mathit{FP}}\\
  \text{Recall} & = & \frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}
\end{eqnarray}


\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
predictions = mod.predict(X) # binary vector of predictions

numerator = sum([(a and b) for (a,b) in zip(predictions,labels)])
nRetrieved = sum(predictions)
nRelevant = sum(y)

precision = numerator / nRetrieved
recall = numerator / nRelevant
\end{lstlisting}
\end{minipage}
%\todo{better way to write without using zip?}
%\todo{Test code}

\paragraph{$F_\beta$ Score}

Note that neither precision nor recall are particularly meaningful if reported in isolation. For instance, it is trivial to achieve a recall of 1.0 simply by using a classifier that returns `true' for every item (in which case, all relevant documents are returned); such a classifier would of course have low precision. Likewise, a precision of close to 1.0 can often be achieved by returning `true' only for a few items about which we are extremely confident; such a classifier would have low recall.

As such, to evaluate a classifier in terms of precision and recall, we likely want a metric which considers both, or otherwise to place additional constraints on our classifier (as we see below).

The $F_\beta$ score achieves this by taking a weighted average of the two quantities:
\begin{equation}
    F_\beta = (1 + \beta^2) \cdot \frac{\mathrm{precision}\cdot \mathrm{recall}}{\beta^2\text{precision} + \text{recall}}.
    \label{eq:fbeta}
\end{equation}
In the case of $\beta=1$ (which is normally called simply the `$F$-score'), \Cref{eq:fbeta} simply computes the harmonic mean of precision and recall, which is low if either of precision or recall are low.

Otherwise, if $\beta \not= 1$, the $F_\beta$ score reflects a situation where one cares about recall over precision by a factor of $\beta$.\footnote{Without going into detail, this motivation leads to the specific formulation in \Cref{eq:fbeta} \cite{van1979information}.}

There are several situations where one might care about recall more than precision, or vice versa. For instance, considering the motivating examples from the start of this section, in a baggage-handling scenario we would likely care primarily about recall, and would be willing to sacrifice precision to achieve it; or, in a search or recommendation setting, we may be happy retrieving only a few items, so long as some are relevant (i.e.,~high precision but low recall).

\paragraph{Precision and Recall @ $K$}

%\todo{Rewrite below so it sounds more like precision @ K}
%\todo{Also, argument below is ugly and doesn't follow from previous paragraph}
%Often, we are interested in evaluating the precision and recall as a function of the model's confidence. 
%We motivated precision and recall by 
One of our motivating examples when defining precision and recall
considered cases where we may only have a fixed budget of results that can be returned to a user.
%That is, what is the model's precision and recall among the top $K$ predictions? 
In particular, we might be interested in evaluating the precision and recall when our classifier returns only its $K$ most confident predictions.
To do so, we begin by sorting the labels $y_i$ according to their associated confidence scores (i.e.,~$x_i \cdot \theta$):
\begin{equation}
%  (0.04961637143573233, True),
%  (0.04265739426913856, True),
%  (0.038081609220486146, True),
%  (0.01647932932266133, True),
%  (0.0027684382608090807, False),
%  (-0.0024453115029425265, True),
%  (-0.005299363587960293, False),
%  (-0.005508568447381734, False),
%  (-0.008051803313750272, True),
%  (-0.010969509446894954, True),
%
{
\setlength{\tabcolsep}{2.2pt}
\small
\begin{tabular}{lcccccccccccc}
confidence $X_i \cdot \theta$: & $\cdots$ & 0.49 & 0.42 & 0.38 & 0.16 & 0.02 & -0.02 & -0.05 & -0.05 & -0.08 & -0.10 & $\cdots$ \\
label $y_i$: & $\cdots$ & True & True & True & True & False & True & False & False & True & True & $\cdots$
\label{eq:sortedByConfidence}
%\todo{Why are the cdots not vertically aligned?}
\end{tabular}
}
\end{equation}
Such tuples of confidence scores and labels can be generated as follows (in this case for a logistic regressor as in \Cref{sec:evaluatingClassificationModels}):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
confidences = mod.decision_function(X) # real vector of confidences

sortedByConfidence = list(zip(confidences,y))
sortedByConfidence.sort(reverse=True) # sorted as in %\Cref{eq:sortedByConfidence}%
\end{lstlisting}
\end{minipage}
Note that when evaluating our model's $K$ most confident predictions, we are no longer interested in whether the actual scores are greater or less than zero (i.e.,~whether the classifier would output `true' or `false'): we are only interested in the \emph{labels} among the top-K predictions.

The \emph{Precision @ $K$} and \emph{Recall @ $K$} now simply measure the precision and recall for a classifier which returns only the $K$ most confident predictions. That is, Precision @ $K$ measures what fraction of the top-$K$ predictions are actually labeled `true'; Recall @ $K$ measures the fraction of all relevant documents that are returned among the top $K$. The main difference to note (compared to the definition in \Cref{eq:precRecDefn1}) is that the number of `retrieved' documents is always $K$; that is, the `retrieved' documents are always the $K$ most confident, whether or not the classifier actually predicts a positive label (i.e.,~$x_i \cdot \theta > 0$).

%\todo{Add code for precision and recall @ k}

Unlike precision and recall,
%\todo{consistent capitalization}
precision@$K$
%\todo{Should write it like this everywhere?}
and recall@$K$ can be reported in isolation as they cannot be optimized by trivial solutions. Precision@10, for example, is an effective measure of a classifier's ability to return reasonable results among a page of 10 retrieved items. 

\paragraph{ROC and Precision/Recall Curves}

Another holisitc measure of a classifier's performance is to report the \emph{relationship} between precision and recall, or true between and false positives.

For example, the relationship between the number of True Positives and False Positives is known as the \emph{receiver operating} characteristic. It is so named because of its use in evaluating the performance of radar receiver operators

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
for i in range(1,len(sortedByConfidence)+1):
    retrievedLabels = [x[1] for x in sortedByConfidence[:i]]
    precision = sum(retrievedLabels) / len(retrievedLabels)
    recall = sum(retrievedLabels) / sum(y)
    xPlot.append(recall)
    yPlot.append(precision)
\end{lstlisting}
\end{minipage}

% AUC? Hit rate? Or leave until recommender systems?

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/ROC.pdf}
    \includegraphics[scale=0.75]{fig/PRcurve.pdf}
    \caption{Examples of Receiver-Operating Characteristic (left) and Precision Recall (right) curves.}
    %\todo{Why do the plots look funny? Seem to have big fonts}
    \label{fig:PR_ROC}
\end{figure}

% 

\section{Problems}
%\todo{Typesets ugly, with problem "section" followed by problems heading. Figure out how to avoid or add some text before the actual problems}

\begin{problemlist}
 \item A few examples of regression problems and computing MSE to get started. Copy from class.
 %\todo{this}
 \item Show that $\theta_0 = \bar{y}$ is the best possible solution for a trivial predictor (i.e.,~$y = \theta_0$) in terms of the Mean Squared Error (hint: write down the MSE of this trivial predictor and take its derivative).
 \label{ex:MSEtrivial}
 \item Repeat \Cref{eq:MSEtrivial}, but this time show that the best trivial predictor in terms of the Mean \emph{Absolute} Error (\Cref{eq:MAE}) is given by taking the \emph{median} value of $y$.
 \item In \Cref{eq:MSEMLEderiv} we motivated the choice of the MSE by explaining its relationship to a Gaussian error model. Likewise, show that minimizing the MAE is equivalent to maximizing the likelihood if errors follow a Laplace distribution (the Laplace distribution has probability density function $\frac{1}{2b} \exp\left(-\frac{|x - \mu|}{b}\right)$).
 \item (Hard) In \Cref{eq:pseudoInv} we saw how to compute a line of best fit via the pseudoinverse, $\theta = (X^TX)^{-1}X^Ty$; Show that the parameters which minimize the Mean Squared Error are found by taking the pseudoinverse, i.e.,~that $\argmin_\theta \frac{1}{|y|}\sum_{i=1}^{|y|} (x_i\cdot \theta - y_i)^2 = (X^TX)^{-1}X^Ty$ (that is, find the stationary point where $\frac{\partial \text{MSE}}{\partial \theta} = 0$).
 \label{ex:hardPseudoInv}
 \item (Hard) When minimizing the Mean Squared Error with a linear model as in \Cref{ex:hardPseudoInv}, show that the residuals $r_i = (y_i - x_i \cdot \theta)$ have average $\bar{r} = 0$.
\end{problemlist}

%-- Converting real-world datasets to 

%-- Show relationship between pseudoinverse and derivative-based solution



% -- Show that MSE residuals have average zero

% -- Show that Laplace MAE is MLE for Laplace distribution

% -- Exercise about MSE = mean and MAE = Median

-- Show that solving a classification problem by treating it as regression on 0/1 variables isn't effective, and reason about why

-- Some question about generating precision / recall, and ROC curves

-- Compare a model that includes $\theta_0$ in the regularizer to one that doesn't

% Exercise:
% At the beginning of \Cref{sec:classification} 

\chapter{Recommender Systems}
\label{chap:recsys}

\section{Basic Setup and Problem Definition}
\label{sec:setup}

The typical modality of the data we are trying to model might consist of sequences of historical interactions between users and items, for example, we might have a collection of movie ratings such as:
\begin{equation}
    \begin{tabular}{llrr}
      (Julian, & \emph{The Godfather}, & 4, & Jan 4 2019)\\
      (Julian, & \emph{Pulp Fiction}, & 3, & Jan 6 2019)\\
      (Laura, & \emph{Seven Samurai}, & 5, & Jan 8 2019)\\
      (Laura, & \emph{The Godfather}, & 4, & Jan 11 2019)\\
      \multicolumn{4}{c}{$\vdots$}
    \end{tabular}
\end{equation}
which might further be anonymized in terms of user IDs, item IDs, and sequential timestamps:\footnote{The timestamp shown here is known as the \emph{unix time}, representing the number of seconds since January 1970 (in UTC); such a representation is often useful as it allows straightforward comparison between timestamps.}
\begin{equation}
    \begin{tabular}{llrr}
      (264, & 547, & 4, & 1546588800)\\
      (264, & 82, & 3, & 1546761600)\\
      (3473, & 231, & 5, & 1546934400)\\
      (3473, & 547, & 3, & 1547193600)\\
      \multicolumn{4}{c}{$\vdots$}
    \end{tabular}
\end{equation}
Such a data format is ubiquitous across many popular recommendation datasets and tasks, including (for example) the popular \emph{Netflix Prize} dataset (\Cref{sec:netflixPrize}). Such data may include `side information,' such as reviews, demographic information about the users, or metadata about the movies, but often it may not. In fact, in the simplest form it may not even include ratings or timestamps.

Thus in essence the data we are trying to work with simply describes \emph{interactions} among users and content. Such interactions could describe clicks, purchases, ratings, likes, (etc.). In other settings the interactions could describe social connections among users, or even `interactions' among compatible clothing items (\Cref{sec:caseStudyClothing}).

Given interaction data such as that above, we would now like to ask questions such as:
\begin{itemize}
    \item How will Laura rate Pulp Fiction?
    \item Given that Laura liked the The Godfather, what other movies will she like?
    \item What movie is Laura likely to rate next?
\end{itemize}
Answering these questions seems difficult, as we seemingly know very little about the users and items involved. However we do know, for instance, that both Laura and Julian recently watched The Godfather, and gave it similar ratings; from this we could begin to reason that may exhibit similar preferences with regard to other movies also.

Reasoning about these types of questions, and modeling these types of interaction data, are the main goals of recommender systems.

\paragraph{How is Recommendation Different from Regression or Classification?}

In \Cref{chap:basics}, we saw several techniques that seem like they could already be used to predict outcomes like ratings and purchases. For example, predicting a rating (e.g.~of a movie) seems like a traditional regression task, and we can imagine various user and movie features that might be associated with ratings. As such, na\"ively we might try to extract user and movie features and fit a linear model of the form
\begin{equation}
    \text{rating}(\text{user},\text{movie}) = \langle \underbrace{\phi(\text{user}, \text{movie})}_{\mathclap{\text{user and movie features}}}, \theta \rangle.
\end{equation}
User features might include attributes like the user's age, gender, location, or other demographic features that might be associated with rating patterns; movie features could capture the length, MPAA rating, budget, or presence of certain actors (etc.). Assuming user and movie features can be collected independently, and since the model is linear, this could be rewritten as
\begin{equation}
    \text{rating}(\text{user}, \text{movie}) = \langle \underbrace{\phi^{(u)}(\text{user})}_{\mathclap{\text{user features}}}, \overbrace{\theta^{(u)}}^{\mathclap{\text{user parameters}}} \rangle + \langle \underbrace{\phi^{(i)}(\text{movie})}_{\mathclap{\text{movie (item) features}}}, \overbrace{\theta^{(i)}}^{\mathclap{\text{movie parameters}}} \rangle.
\end{equation}
When written this way, we can see that the prediction of the rating is the sum of two \emph{independent} predictions: one for the user (say $f(u)$) and one for the item (say $f(i)$). If we were to make recommendations based on these predictions, for example by recommending whichever unseen movie a user would give the highest rating to, i.e., \begin{equation}
  \argmax_{\mathclap{i \in {\text{unseen movies}}}} f(u) + f(i),
\end{equation}
our recommendation \emph{for every user} would simply be whichever movie had the highest predicted rating $f(i)$. In other words, every user would simply be recommended movies which had features associated with high ratings.

Critically, such a model does not \emph{personalize} its recommendations to individual users. Even if the model achieved a reasonable Mean Squared Error in terms of predicting ratings, it would not be an effective recommender system.

To overcome this limitation, a model must in some fashion capture \emph{interactions} between users and items, e.g.~how compatible is a user with a particular movie. We discussed this idea a little in \Cref{chap:nonparametric}, 
%\todo{rewrite once that chapter exists}
but investigate the idea in more detail here. 

In summary, modeling \emph{interections} between users and items is the main goal of recommender systems and is the main characteristic that differentiates them from other types of machine learning (\Cref{box:interactions}).

\begin{floatingbox}
\caption{Recommender Systems Compared to Other Types of ML}
The main distinguishing feature of recommender systems compared to other types of machine learning is their goal of explicitly modeling \emph{interactions} between users and consumed items based on historical patterns. This feature allows the models to understand which items are \emph{compatible} with which users, and thus to make different recommendations to each user in a personalized way.
\label{box:interactions}
\end{floatingbox}

\section{Notation and Representation}

Basic notation is established in \Cref{tab:recsysNotation}. There are several ways we can represent the interaction data described above. Formally, we might simply describe the dataset as a collection of tuples $(u,i,r,t)$, or $r_{u,i,t} \in \mathbb R$ indicating that a user $u$ entered the rating $r$ for item $i$ at time $t$.

But conceptually it is easier to think about these data in terms of sets or matrices. Set representations will be useful when establishing similarity between users in terms of sets of items they have consumed (or likewise similarity between items in terms of sets of users who have consumed them); matrix representations will be useful when developing models based on the concept of matrix factorization (or dimensionality reduction).

% \begin{table}
%     \centering
%     \begin{tabular}{ll}
%         \toprule
%          $u \in U$ & users $u$ in user set $U$\\
%          $i \in I$ & items $i$ in item set $I$\\
%          $I_u$ & set of items rated by user $u$\\
%          $U_i$ & set of users who have rated item $i$\\
%         \bottomrule
%     \end{tabular}
%     \caption{Common Recommender Systems Notation}
%     \label{tab:recsysNotation}
% \end{table}

\paragraph{Activities as Sets}

For our simplest recommendation model, we can describe users in terms of the sets of items they have interacted with, e.g.~for a user $u$:
\begin{equation}
    I_u = \text{set of items consumed by $u$};
\end{equation}
likewise, we can describe items in terms of the sets of users who have interacted with them:
\begin{equation}
    U_i = \text{set of users who consumed item $i$}
\end{equation}

\paragraph{Activities as Matrices}

%\todo{Explain that sets can't handle ratings}

Alternately, we can represent a dataset of user/item interactions via matrices. 
%\todo{rewrite with both a binary matrix and a rating matrix, then can write the set notations in terms of these}
%E.g.~a matrix encoding which i:
\begin{equation}
    R = \underbrace{\left[ \begin{tabular}{ccccc}
    5 & $\cdot$ & $\cdot$ & 2 & 3\\
    $\cdot$ & 4 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 5 & 5 & 3 & $\cdot$\\
    5 & $\cdot$ & 4 & $\cdot$ & 4\\
    1 & 1 & $\cdot$ & 4 & 5\\
    \end{tabular} \right]}_{\text{items}} \left. \mathclap{\begin{tabular}{c} \\ \\ \\ \\ \\ \end{tabular}} \right\rbrace \text{users}.
    \label{eq:interactionMatrix}
\end{equation}
Each row of $R$ represents a single user, and each column represents a single item. Then a particular entry $r_{ui}$ indicates the rating a particular user $u$ gave to an item $i$. Note that the vast majority of entries in such a matrix would typically be missing (most users do not rate most items); indeed, the missing entries are exactly the quantities that we would like to predict.

Naturally, the set and matrix representations can be written in terms of each other, for example, our set representation is equivalent to:
\begin{equation}
    I_u = \lbrace{ i | R_{u,i} \neq 0 }
\end{equation}

Both our set and matrix representations of interactions possibly seem limited---neither conveys the timestamps associated with the ratings (or any other side-information), and the set-based representation doesn't even encode users' ratings. Nevertheless they are useful for reasoning about the basic principles behind recommender systems (which will become the building blocks behind more sophisticated approaches), and in fact these simple techniques are among the most widely deployed.
%\todo{Not beautiful sentenc, lacks evidence}

\section{Neighborhood-based Approaches to Recommendation}
\label{sec:neighborhood}

%\todo{Maybe call this section memory based rather than neighborhood based} 
Perhaps the simplest (and most ubiquitous) approaches to recommendation are based on some notion of `similarity' among items. That is, an item is recommended to a user because it is similar to one that they they have recently clicked, liked, or consumed.

`People who viewed X also viewed Y' (or `people who bought X also bought Y', etc.)~features (\Cref{fig:alsoviewed}) are familiar examples of such similarity-based recommenders. Items are recommended to a user on the basis of how \emph{similar} they are to item the user is currently browsing.

\begin{SCfigure}
    \fbox{\includegraphics[width=0.7\textwidth]{fig/alsoviewed.png}}
    \caption{An example of neighborhood-based recommendation (`People who viewed X also viewed Y').
    \label{fig:alsoviewed}}
\end{SCfigure}

For such a recommender to be effective depends on choosing an appropriate similarity function. The similarity function in \Cref{fig:alsoviewed} is based on view data (presumably clicks); but even then, by what metric should we consider patterns of clicks to be `similar'? Do we literally just count the number of users who have clicked on both items? Or do we need some kind of normalization? Or should we consider temporal recency?

Appropriately designing such similarity functions, and recommending on the basis of such similarity, is the task of so-called \emph{neighborhood-based} recommender systems---items are recommended due to being in the \emph{neighborhood} of (i.e.,~similar to) other items.

\begin{floatingbox}
\caption{Memory-based and model-based recommender systems \label{box:modelBased}}
The neighborhood-based recommendation approaches we study in \Cref{sec:neighborhood} make recommendations by writing down similarity functions that take as input user and item interaction histories. There is no \emph{model} (i.e.,~no parameters) associated with each user or item; the original data is used as an input to the similarity functions. Such systems are referred to as \emph{memory-based} recommenders.
In contrast, the techniques in \Cref{sec:modelBased} learn \emph{representations} of each user and item, such that the original data isn't used directly when making a prediction at test time. Such systems are referred to as \emph{model-based} recommenders.
%TODO: Confusing, not written great
\end{floatingbox}

\subsection{Defining a Similarity Function}

Defining a recommender like the one in \Cref{fig:alsoviewed} essentially requires that we define a similarity function among items:
\begin{equation}
    \mathrm{sim}(i,j),
\end{equation}
and then given a query item $i$, recommend a set of items $j$ that maximize the given similarity.

Let's consider a small toy example, with four items, and sets of users (or rather user IDs) who have consumed each:
\begin{equation}
    \begin{tabular}{l}
    $U_1 = \lbrace 1, 3, 4, 8, 12, 15, 17, 24, 35, 39, 41, 43 \rbrace$\\
    $U_2 = \lbrace 2, 3, 4, 5, 9, 12, 13, 16, 19, 24, 27, 31 \rbrace$\\
    $U_3 = \lbrace 4, 5, 9, 12 \rbrace$\\
    $U_4 = \lbrace 4, 9 \rbrace$
    \end{tabular}
    \label{eq:simToy}
\end{equation}
(recall that in our notation $U_1$ represents the set of users who have bought item 1).
%\todo{check this}
Na\"ively, we might assume that a recommender like the one depicted in \Cref{fig:alsoviewed} is simply counting the number of users who purchased both items in common. In our set notation this would be:
\begin{equation}
    \mathrm{sim}(i,j) = | U_i \cap U_j |.
\end{equation}
Computing some similarities under this model we would find:
\begin{equation}
    \mathrm{sim}(1, 2) = 4; \quad \mathrm{sim}(2, 3) = 3; \quad \mathrm{sim}(3, 4) = 2;  \quad \text{etc.}
\end{equation}
i.e.,~we would rate items 1 and 2, or items 2 and 3, as being \emph{more similar} than items 3 and 4.

We should examine whether these relative scores seem reasonable. Items 1 and 2 are \emph{popular} items, which most users did \emph{not} purchase in common; whereas items 3 has \emph{half} of its users in common with item 4. If we were to build recommenders on this basis, we might recommend (for example) a popular album as being highly similar to a popular pair of jeans, simply on the basis that they have many users in common. In general, such a system would tend to identify popular items (such as items 1 and 2 in \Cref{eq:simToy}) as being similar. `Niche' items with fewer associated purchases (such as items 3 and 4 in in \Cref{eq:simToy}) would rarely be recommended.

In most cases, this is not the outcome we want; such a system would make generic recommendations of popular items, that likely would not seem specific to the context of a given query item. Presumably, we should improve our similarity function so that it has some appropriate normalization to account for item popularity, as we will see below.

This toy example is intended to demonstrate that `similarity' is not something easy to define, and that different definitions have implicit assumptions with non-obvious consequences.

\subsection{Jaccard Similarity}
\label{sec:jaccard}

Our first attempt at correcting the above issue is to normalize similarity scores in a way that considers the popularity of each item. The \emph{Jaccard Similarity}, or `intersection over union,' does so by computing
\begin{equation}
    \mathit{Jaccard}(i,j) = \frac{| U_i \cap U_j |}{| U_i \cup U_j |}.
    \label{eq:jaccard}
\end{equation}
This similarity function is perhaps best visualized by a Venn diagram such as that in \Cref{fig:venn}. The Jaccard similarity takes a value between 0 (when $U_i$ and $U_j$ do not overlap at all, and thus have no intersection) and 1 (when the intersection is equal to the union, i.e.,~the items were consumed by \emph{exactly} the same set of people).

%TODO: Make this look better
\begin{SCfigure}
%\fbox{\includegraphics[width=0.3\textwidth]{fig/venn1.pdf}}
\begin{tikzpicture}[fill=gray]
% left hand
\scope
\clip (1,0) circle (1);
\fill (0,0) circle (1);
\endscope
% right hand
\scope
\clip (0,0) circle (1);
\fill (1,0) circle (1);
\endscope
% outline
\draw (0,0) circle (1) (0,1)  node [text=black,above] {$U_i$}
      (1,0) circle (1) (1,1)  node [text=black,above] {$U_j$};
\end{tikzpicture}\hspace{1mm}
\begin{tikzpicture}[fill=gray]
% left hand
\scope
% \clip (1,0) circle (1);
\fill (0,0) circle (1);
\endscope
% right hand
\scope
% \clip (0,0) circle (1);
\fill (1,0) circle (1);
\endscope
% outline
\draw (0,0) circle (1) (0,1)  node [text=black,above] {$U_i$}
      (1,0) circle (1) (1,1)  node [text=black,above] {$U_j$};
\end{tikzpicture}
\caption{The similarity between two items $i$ and $j$ can be computed in terms of the intersection (left) and union (right) between the sets of users $U_i$ and $U_j$ who have consumed each.}
\end{SCfigure}

To demonstrate the Jaccard similarity in action, let's consider computing the similarity among items in terms of past purchases from \emph{Amazon.com}. 
%\todo{Make sure company names and stuff are italicized consistently}
We'll consider Amazon's publicly-available dataset of around 900,000 reviews from the \emph{Musical Instrument} category.\footnote{Available from \url{https://s3.amazonaws.com/amazon-reviews-pds/tsv/index.txt}}

We first build some data structures to store the sets of items consumed by each user (or the sets of users who have consumed each item), i.e.,~$I_u$ and $U_i$:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
usersPerItem = defaultdict(set)
itemsPerUser = defaultdict(set)

for d in dataset:
    user,item = d['customer_id'], d['product_id']
    usersPerItem[item].add(user)
    itemsPerUser[user].add(item)
\end{lstlisting}
\end{minipage}
We can also implement the Jaccard similarity straightforwardly:
%\todo{fix indentation level for all code}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
def Jaccard(s1, s2):
    numerator = len(s1.intersection(s2))
    denominator = len(s1.union(s2))
    return numerator / denominator
\end{lstlisting}
\end{minipage}
Now, \emph{recommendation} consists of finding the items with the highest Jaccard similarity compared to some given query (i.e.,~``people who bought X also bought Y''):
%\todo{comment code throughout}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
def mostSimilar(i, K): # Query item i, and number of results K to return
    similarities = []
    users = usersPerItem[i] # Users who have purchased i
    for j in usersPerItem: # Compute similarity against each item
        if j == i: continue
        sim = Jaccard(users, usersPerItem[j])
        similarities.append((sim,j))
    similarities.sort(reverse=True) # Sort to find the most similar
    return similarities[:K]
\end{lstlisting}
\end{minipage}

Finally, let's examine some recommendations, e.g.~of the \emph{`AudioQuest LP record clean brush'} (product ID \emph{B0006VMBHI}). The 5 most similar items (i.e., \texttt{mostSimilar('B0006VMBHI', 5)}) are:
\begin{eqnarray*}
\begin{tabular}{l}
\emph{Shure SFG-2 Stylus Tracking Force Gauge} \\
\emph{Shure M97xE High-Performance Magnetic Phono Cartridge} \\
\emph{ART Pro Audio DJPRE II Phono Turntable Preamplifier} \\
\emph{Signstek [...] Long-Playing LP Turntable Stylus Force Scale Gauge Tester} \\
\emph{Audio Technica AT120E/T Standard Mount Phono Cartridge}
% \emph{Technics: 45 Adaptor for Technics 1200 (SFWE010)} \\
% \emph{GruvGlide GRUVGLIDE DJ Package} \\
% \emph{STANTON MAGNETICS Record Cleaner Kit} \\
% \emph{Shure M97xE High-Performance Magnetic Phono Cartridge} \\
% \emph{Behringer PP400 Ultra Compact Phono Preamplifier}
\end{tabular}
\end{eqnarray*}
All of the recommended items are also related to record players (which make up only a fraction of items in the category), which seem semantically reasonable given the query.

And in just a few lines of code, and using a (reasonably large) real-world dataset, we have implemented our first recommender system! Our solution is simple (and our implementation is \emph{very} inefficient\footnote{In particular it is not necessary to iterate over \emph{all} items, but rather one can quickly compute a candidate set of only those items that could potentially have a non-zero Jaccard coefficient; see Exercise \ref{?}.}), but nevertheless quickly produced reasonable recommendations. Perhaps not surprisingly, these simple types of similarity-based recommendations drive many of the most high-profile recommender systems on the Web (see \Cref{sec:caseStudyAmazonRec}).
%\todo{hastily written, weak prose}

%TODO: Above code is not efficient

\subsection{Cosine Similarity}
\label{sec:cosine}

The Jaccard similarity captures our intuition about what items ought to be `similar' to each other, but is only defined if interactions are represented as \emph{sets}. We would like more nuanced similarity measures for data where feedback is associated with each interaction; for example we might not regard two users as `similar' if both had watched the \emph{Harry Potter} movies, in the event that one of them had liked the series and the other had disliked it.

The \emph{Cosine Similarity} achieves this by representing users' (or items') interaction histories in terms of \emph{vectors} rather than in terms of \emph{sets}. An example is shown in \Cref{fig:cosine} in which we have three items ($i_1$, $i_2$, and $i_3$) and two users ($u_1$ and $u_2$) who have each interacted with two of them. 

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/cosine.pdf}
\caption{The Cosine Similarity is defined in terms of the angle between two vectors, here describing users $u_1$ and $u_2$. \label{fig:cosine}}
\end{SCfigure}

In our previous (set) representation we would write $I_{u_1} = \lbrace i_2, i_3 \rbrace$ and $I_{u_2} = \lbrace i_1, i_3 \rbrace$ to describe the sets of items that $u_1$ and $u_2$ have interacted with. In vector representation we simply represent the users as $u_1 = (0,1,1)$ and $u_2 = (1,0,1)$; note that these vectors are equivalent to rows of our interaction matrix $R$ (\Cref{eq:interactionMatrix}), i.e.,~$R_{u_1}$ and $R_{u_2}$.
%\todo{Sloppy notation. Using a vector to index a matrix}

The Cosine Similarity (in this case between two users $u_1$ and $u_2$) is now defined in terms of the \emph{angle} between the vectors $u_1$ and $u_2$. Recall that the angle between two vectors $a$ and $b$ is defined as:
\begin{equation}
    \theta = \mathrm{Cos}^{-1} \left( \frac{a \cdot b}{|a|\cdot |b|} \right); \quad \text{or} \quad \mathrm{Cos}(\theta) = \frac{a \cdot b}{|a|\cdot |b|}.
\end{equation}
The angle $\theta$ measures the extent to which the two vectors point in the same direction; in the case of interaction data, the angle will range between \ang{0} (if the two users have interacted with exactly the same set of items) and \ang{90} (if the interaction vectors are orthogonal, i.e.,~if the users have interacted with non-overlapping sets of items). The actual cosine similarity 
%\todo{consistent capitalization}
is the cosine of this angle, e.g.~between two users $u$ and $v$ is:\footnote{Again it can be straightforwardly defined for \emph{item} similarity by interchanging users and items.}
\begin{equation}
    \text{Cosine Similarity}(u,v) = \frac{I_u \cdot I_v}{|I_u| \cdot |I_v|}.
    \label{eq:cosineSim}
\end{equation}
For (binary) interaction data the cosine of the angle is now between 1 (when the angle is zero, and the interactions are identical) and 0 (when the interactions are orthogonal).

It is instructive to compare \Cref{eq:jaccard,eq:cosineSim}. 
In the case of binary interaction data, both expressions take values of 1 when the interactions are identical, and 0 when the interactions are non-overlapping.
The numerator $I_u \cdot I_v$ in \Cref{eq:cosineSim} is equivalent to $|I_u \cap I_v|$, as in \Cref{eq:jaccard}. The two differ only in their denominators, both of which are essentially forms of normalization based on the size of the sets $I_u$ and $I_v$ (and both demoninators will take the same value when $I_u$ and $I_v$ are equal).

Of course the cosine similarity is more interesting once we consider numerical interactions, i.e.,~interactions associated with \emph{feedback}, rather than just binary (0/1) data. For example, consider data where each interaction is associated with a `thumbs-up' or `thumbs-down' rating. We might represent this via an interaction matrix $R$ such that $R_{u,i} \in \lbrace -1, 0, 1 \rbrace$ (where $-1/1$ indicates thumbs-up/thumbs-down, and $0$ indicates an item the user hasn't interacted with).

In this case, the Jaccard similarity would not be well-defined. But the cosine similarity can still be computed on rows (or columns) of $R$. An example is shown in \Cref{fig:cosine2}. Here, user $u_1$ has interacted with two items ($i_2$ and $i_3$), and liked both; $u_2$ has interacted with the \emph{same} items, but disliked both. The two user vectors now point in opposite directions, i.e.,~they have an angle of \ang{180} and a cosine similarity of -1.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/cosine2.pdf}
\caption{The Cosine Similarity for two users who rated the same items, but with opposite sentiment polarity. \label{fig:cosine2}}
\end{SCfigure}

Our code above for the Jaccard similarity can be straightforwardly modified to implement the cosine similarity (\Cref{exercise:cosine}). Doing so (for the same query item), the top recommendation remains the same (Shure SFG-2 Stylus Tracking Force Gauge). Among the next few recommendations, there are many ties (i.e.,~identical cosine similarities); upon inspection these turn out to be items with only a single (overlapping) interaction. In this case such items are preferred by the cosine similarity (and not the Jaccard) since the denominator grows quickly for items with many associated interactions (whereas the union term in \Cref{eq:jaccard} grows more slowly, assuming many of the interactions overlap).
%\todo{Results look a bit funny, check.}

\paragraph{Which similarity metric is `better'?}
%The above argument about the difference between the Jaccard and cosine similarity is somewhat imprecise. 
In the above example the Jaccard similarity seemed to work `better' than the Jaccard similarity, but our argument about the difference between the two is somewhat imprecise.
Note that this argument largely applies to this specific dataset (or even the specific query item we chose). Ultimately, these similarity measures are essentially \emph{heuristics}; whether one is `better' than another depends on our own assumptions and intuition about what similarity ought to mean. This is in contrast to the machine learning approaches we saw in previous chapters in which we had a specific objective (i.e.,~a measure of success) that we were trying to optimize. We'll revisit this question when we examine model-based recommender systems in \Cref{sec:modelbased}.

\subsection{Pearson Similarity}
\label{sec:pearsonSim}

We motivated the Jaccard similarity by considering binary interaction data (i.e.,~sets), and the cosine similarity by considering polarized interactions like `thumbs-up's and `thumbs-down's. But how would these similarity measurements operate on numerical feedback scores such as star ratings (as in \Cref{eq:interactionMatrix})?

Consider the users represented in \Cref{fig:pearson1} (left).
%\todo{Most of these 3d plots would look better with shorter $i_1$ axis.}
Here two users have rated the same items ($i_2$ and $i_3$); user $u_1$ rated them 3 and 5 stars (respectively); user $u_2$ rated them 5 and 3.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/pearson1.pdf}%
\includegraphics[scale=0.75]{fig/pearson2.pdf}
\caption{Pearson Correlation. Two users have rating vectors that point roughly in the same direction (left); after subtracting the average from each, they point in opposite directions (right). \label{fig:pearson}}
\end{SCfigure}

According to the Jaccard similarity (based only on interactions), we would consider the two users to be identical (Jaccard similarity of 1); according to the cosine similarity, we would regard them as being very similar, since the angle between the two vectors is small. However one could argue that these users are polar opposites of each other: if we consider 5 stars to be a \emph{positive} rating and 3 stars is a \emph{negative} rating, then these two users indeed have opposite opinion polarity.

Our definition of the cosine similarity does not account for this interpretation, essentially because it depends on the interactions already having explicitly positive or negative polarity. To might correct this by appropriately normalizing our ratings: if we subtract the average for each user (4 stars for both $u_1$ and $u_2$), we find that their ratings are each 1 star above or below their personal average. After doing so, the example becomes very similar to the one from \Cref{sec:cosine} (see \Cref{fig:pearson}, right).

This is essentially the idea captured by the Pearson Correlation. %\todo{again, check capitalization} 
The \emph{Pearson Correlation Coefficient} is a classical measurement for assessing the relationship between two variables, i.e.,~whether they trend in the same direction, regardless of scale and constant differences between them. The Pearson Correlation between two vectors $x$ and $y$ is defined as
\begin{equation}
    \text{Pearson Correlation}(x,y) = \frac{\sum_{i=1}^{|x|} (x_i - \bar{x})(y_i - \bar{y})}{\sqrt{\sum_{i=1}^{|x|}  (x_i - \bar{x})^2}\sqrt{\sum_{i=1}^{|y|} (y_i - \bar{y})^2}}.
    \label{eq:pearsonCC}
\end{equation}
Compare this definition to that of the cosine similarity in \Cref{eq:cosineSim}: the only difference is that from each measurement we subtract the mean ($\bar{x}$ or $\bar{y}$) of the corresponding vector.

When applying this concept to rating data, we should be careful not to regard unobserved ratings (i.e.,~missing values of $R_{u,i}$ in \cref{eq:interactionMatrix}) as zeros---doing so would distort our estimate of the user mean. Thus we might define the similarity between two users $u$ and $v$ (or similarly, items) only in terms of items they have 
both 
interacted with:
\begin{equation}
    \text{Pearson Similarity}(u,v) =
    \frac
    {
      \sum_{i \in I_u \cap I_v}
        (R_{u,i} - \bar{R_u})
        (R_{v,i} - \bar{R_v})
    }
    {
      \sqrt{\sum_{i \in I_u \cap I_v} (R_{u,i} - \bar{R_u})^2}
      \sqrt{\sum_{i \in I_u \cap I_v} (R_{v,i} - \bar{R_v})^2}
      %\sqrt{\sum_{i \in I_u} (R_{u,i} - \bar{R_u})^2}
      %\sqrt{\sum_{i \in I_v} (R_{v,i} - \bar{R_v})^2}
    }.
    \label{eq:pearsonSim}
\end{equation}
%TODO: Think about putting this argument back in
Note that our choice to define the Pearson Similarity by considering only \emph{shared} items is somewhat arbitrary; we could instead have considered \emph{all} items rated by each user in the denominator. Using our definition (which appears in e.g.~\cite{sarwar2001item}), we regard users as maximally similar if they have rated shared items in the same way; if we considered all items rated by each user in the denominator, we would regard users as less similar if they had also rated some different items. Remember that these similarity functions are merely heuristics---neither option should be considered more `correct,' but rather we should choose the definition that suits our intuition (or generates the most satisfactory results) in a particular situation.
%\todo{long argument to make a not very important point! Think whether to include this.}

%%\todo{Finish with code examples and results}

%TODO: Bullets don't look great, maybe use a {description} environment or similar
\begin{floatingbox}
\caption{Summary of Similarity Measures. \label{box:sumSim}}
Our comparison of the Jaccard, Cosine, and Pearson similarity can be summarized as follows:
\begin{itemize}
    \item The \emph{Jaccard Similarity} computes the similarity between \emph{sets}. The basic idea is to find items that have been purchased in common by many users (or similarly, users who have purchased many items in common); the set union is used to normalize the quantity, so that the measure does not overly favor large sets.
    \item The \emph{Cosine Similarity} instead represents interactions as vectors (basically, rows or columns of our interaction matrix $R$). Similarity is then computed in terms of the angle between vectors for two items (or users). This definition allows similarity to be computed for numerical interaction data, especially if a polarity (i.e.,~positive or negative) is associated with each interaction.
    \item Finally, the \emph{Pearson Similarity} was motivated by the idea that numerical feedback may need to be properly calibrated in order to associate a polarity to each score. For instance, a rating of `3.5' might be positive for one user but negative for another. The Pearson Similarity calibrates this polarity simply by subtracting the average for each item (or user); after this calibration the definition is similar to that of the cosine similarity.
\end{itemize}
\end{floatingbox}

Fitting the Pearson Similarity given the query item from \Cref{sec:jaccard} does not produce particularly satisfactory results; 
%the similar items come from a broad range of categories that do not seem to be closely related to the query item.
using $U_i \cap U_j$ in the denominator of \Cref{eq:personSim} results in many items with a similarity of 1.0 (usually just due to a single overlapping interaction); using $U_i$ and $U_j$ separately in the denominator of \Cref{eq:personSim} generates more meaningful results, though they come from a broad category of items that do not seem closely related.

Possibly these results are unsatisfactory simply because ratings of (e.g.)~a record cleaning brush are not due to factors that meaningfully transfer to other items. One likely purchses a record cleaning brush for its utility, rather than because of their personal preference toward such items. If variability in ratings is primarily due to build quality, or effectiveness (for example), then the Pearson Similarity might identify other items with `similar' build quality or effectiviness, but those may not be semantically similar items. In this particular example, the Jaccard similarity---which defines similarity in terms of \emph{what} was purchased---seems more appropriate than the Pearson Similarity, which defines similarity in terms of preferences.

Again though, possibly this measure is simply not suitable for this dataset or this query item. Let's try again on another dataset, this time from the Amazon \emph{Video Games} category. Given the query \emph{One Piece: Pirate Warriors}, the five most similar items in terms of Pearson Similarity\footnote{Again using $U_i$ and $U_j$ separately in the denominator} are:
\begin{eqnarray*}
\begin{tabular}{l}
\emph{Full Metal Alchemist: The Broken Angel}\\
\emph{Monster Rancher 4}\\
\emph{FINAL FANTASY X X-2 HD Remaster}\\
\emph{BlazBlue: Continuum Shift EXTEND Limited Edition}\\
\emph{Killzone 3}
(etc.)
\end{tabular}
\end{eqnarray*}
These recommendations look more reasonable. In addition to being for similar platforms (e.g.~PlayStation) most are reasonably similar in terms of genre and style (e.g.~Japanese, based on anime, etc.). Seemingly, for this type of time, features like style and genre better explain variation in ratings, making the Pearson Correlation more effective.

Finally, these similarity measures needn't be used so directly for recommendations as we have done here (i.e.,~simply retrieving the most similar item given a query). In practice they might simply be subroutines that guide more complex algorithms. For example, to recommend items to a user we might first find similar users, and recommend items that many of those users liked, rather than simply relying on item-to-item similarity directly (see e.g.~\Cref{sec:caseStudyAmazonRecs}).
%\todo{weak prose above}

%%\todo{Put this under some section heading?}
%Moved above
%Note that the memory-based recommendation approaches above are essentially \emph{heuristics} for determining whether items (or users) are similar to each other; each makes different assumptions about what `similarity' might mean, each of which might be appropriate given different datasets or goals.

\subsection{Using Similarity Measurements for Rating Prediction}

In \Cref{sec:modelbased}, we will contrast the similarity-based recommendation approaches above with machine learning (or `model-based') approaches which directly seek to predict ratings as accurately as possible.

However these two goals (measuring similarity versus predicting ratings) are not at odds with each other, and indeed one can use a measure of similarity as a means of predicting ratings.

The essence of such an approach is that the rating a user will give to an item can be estimated from ratings that user has given to \emph{similar} items (again, for some appropriate definition of `similarity'). One such definition (from \cite{sarwar2001item}) predicts the rating as a weighted sum of other items the user has rated:
\begin{equation}
    r(u,i) = \frac{\sum_{j \in I_u} R_{u,i} \cdot \mathrm{Sim}(i,j)}{\sum_{j \in I_u} \mathrm{Sim}(i,j)},
    \label{eq:simPredict}
\end{equation}
where $\mathrm{Sim}(i,j)$ could be any item-to-item similarity function such as those above. Note here that $r(u,i)$ is a prediction whereas $R_{u,i}$ is a historical rating.

The intuition behind the above equation is simply that the most similar items should be the most relevant when predicting future ratings, so the user's past ratings of those items are given the highest weights. Again though this is just a heuristic for predicting ratings and could be defined differently. For example we could write the same definition in terms of user similarity:
\begin{equation}
    r(u,i) = \frac{\sum_{v \in U_i} R_{u,i} \cdot \mathrm{Sim}(u,v)}{\sum_{v \in U_i} \mathrm{Sim}(u,v)},
    \label{eq:simPredict2}
\end{equation}
or, we, could possibly improve performance slightly by weighting deviations from the average rating, rather than ratings directly:
\begin{equation}
    r(u,i) = \bar{R_i} + \frac{\sum_{j \in I_u} (R_{u,i} - \bar{R_j}) \cdot \mathrm{Sim}(i,j)}{\sum_{j \in I_u} \mathrm{Sim}(i,j)},
    \label{eq:simPredict3}
\end{equation}

Using our video game data from \Cref{sec:pearsonSim}, and following the prediction function from \Cref{eq:simPredict3} (with the Jaccard Similarity as our similarity function), the Mean Squared Error of predicted ratings compared to the true labels is 1.786,
%1.9150345376505316 - simPredict
%
%1.7861436807136897 - simPredict3
compared to 1.838
%1.838224481118372
when always predicting the mean.

\section{Case Study: Amazon.com Recommendations}

In a 2003 paper \cite{linden2003amazon}, researchers described the techniques underlying \emph{Amazon.com}'s recommendation technology. The paper described systems similar to that in \Cref{fig:alsoviewed}, e.g.~``Customers who bought items in your shopping card also bought.''

The first recommendation method the paper describes is based on the Cosine Similarity (\Cref{sec:cosine}). Interestingly, cosine similarity is defined between \emph{users}, rather than between items as we did in \Cref{sec:jaccard}; the goal is then to recommend items that have previously been purchased by similar customers. The paper discusses the issues of scaling this type of similarity computation to the large number of \emph{Amazon} users, and discusses an alternative strategy to \emph{cluster} users

Cite the old amazon recommendation paper, and `Two Decades of Recommender Systems at Amazon.com'


\section{Model-based Approaches to Recommendation}
\label{sec:modelbased}

In contrast to `memory-based' recommendation approaches, \emph{model-based} approaches seek to learn parameterized representations of users and items, so that recommendations can be made in terms in terms of the learned parameters. Model-based approaches are typically cast in terms of supervised learning, so that the goal is to predict ratings, purchases, clicks (etc.)~as accurately as possible.

\begin{floatingbox}
\caption{Memory-based vs.~Model-based Approaches}
There are a variety of reasons why one might choose a model-based or memory-based approach. We summarize a few of the advatages and disadvantages as follows:
\begin{description}
  \item[Training and Inference Complexity] Model-based approaches often require (expensive) offline training; on the other hand, once trained, recommendations can potentially be retrieved quickly, e.g.~by retrieving a nearest neighbor or a maximum inner product in parameter space. In contrast, 
  %TODO: Finish
  \item[Interpretability] Often, simple recommendations such as those in \Cref{fig:alsoviewed} may be preferable simply because they are easy to explain to a user. In contrast, machine learning-based recommendations may make users uncomfortable due to their `black box' nature (see \Cref{sec:interpretability}).
  \item[Accuracy] Model-based systems are appealing because they directly optimize a desired error measure. On the other hand, error measures that are tractible may not be those that relate meaningfully to user satisfaction, and may distract from qualitative improvements.
\end{description}
\end{floatingbox}

%TODO: Other methods
%SLIM
%FISM
%Some sort of deep learning based model


\subsection{The Netflix Prize}
%\todo{Put this subsection somewhere, or make it a case study}

In 2006, \emph{Netflix} released a dataset of 100,000,000 movie ratings (across 17,770 users and around 480,000 users). Their dataset took exactly the form described in \Cref{sec:setup}, i.e.,~it consisted purely of (user, item, rating, timestamp) tuples. Associated with the dataset was a competition \cite{bennett2007netflix} to reduce the RMSE (on a test set with withheld ratings) by 10\% compared to Netflix's existing solution. The first team to do so would win a \$1,000,000 prize. 


The history of the competition is itself fascinating. Early leaders joined forces to develop ensemble approaches, and the winning teams were nearly tied in a nail-biting finish. The competition also led to a broader discussion around the value of such high-profile competitions, as well as the question of whether narrowly reducing a mean-squared error actually improves recommendations. Finally it also led to a lawsuit against Netflix following de-anonymization of the competition data \cite{narayanan2006break}.\footnote{This too is an interesting story, as the competition data, with anonymized user and item IDs, at first glance appears to be sufficiently anonymized.}
%\todo{Could discuss further. Also include e.g. whether monolithic models can really be implemented}

But other than the history surrounding the Netflix Prize, the dataset and high-profile competition spawned a great deal of research on recommender systems in general, and in particular the specific problem of rating prediction.
%\todo{rest of this sentence}
and in particular, solutions based on matrix factorization.

\begin{floatingbox}
\caption{Lessons from the Netflix Prize}
%TODO
Add text on ``Lessons from the Netflix Prize Challenge'', maybe in box.
\end{floatingbox}




\subsection{Matrix Factorization}

The basic assumption made by model-based recommenders is that there is some underlying \emph{structure} among the interactions we are trying to predict. Put differently, model-based recommender systems are essentially a form of dimensionality reduction.
%\todo{This and below could be in the start of the section, or the subsection heading could be below}

In simple terms, we assume that users' opinions, or the properties of the items they consume, can be efficently summarized. Do you tend to like action movies (and is this an action movie)? Do you tend to enjoy movies with a high budget, certain actors, or a long runtime? To the extent that purchases, clicks, or ratings can be explained by such factors, the goal of model-based recommendation is to discover them.

Considering data of the form in \Cref{sec:setup}, this seems a difficult process: we have no knowledge of the necessary \emph{features} that would be needed to discover these important factors (i.e.,~we don't know which movies are action movies, which movies have a long runtime, etc.). But surprisingly one can still uncover these underlying dimensions. Consider the interaction (e.g.~click) data depicted below:
\begin{equation}
    R = 
    \begin{tabular}{cc}
    $\left[ \begin{tabular}{ccccc}
    1 & $\cdot$ & 1 & $\cdot$ & $\cdot$\\
    1 & 1 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 1 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & $\cdot$ & $\cdot$ & 1 & 1\\
    $\cdot$ & $\cdot$ & $\cdot$ & 1 & $\cdot$\\
    \end{tabular} \right]$
    &
    \hspace{-6mm}
        \begin{tabular}{r}
        $u_1$\\
        $u_2$\\
        $u_3$\\
        $u_4$\\
        $u_5$
        \end{tabular}\\
        
        \begin{tabular}{ccccc}
        \hphantom{1}$i_1$ &
        \clap{$i_2$}\hphantom{1} &
        \clap{$i_3$}\hphantom{1} &
        \clap{$i_4$}\hphantom{1} &
        \clap{$i_5$}\hphantom{1}
        \end{tabular}
    \end{tabular}
        \label{eq:interactionMatrix2}
\end{equation}
The matrix appears to decompose roughly into two `blocks': if we wrote
\begin{equation}
    \begin{tabular}{l}
      $\gamma_{i_1} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_2} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_3} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_4} = \lbrack 0,1 \rbrack$\\
      $\gamma_{i_5} = \lbrack 0,1 \rbrack$
    \end{tabular}; \quad
    \begin{tabular}{l}
      $\gamma_{u_1} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_2} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_3} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_4} = \lbrack 0,1 \rbrack$\\
      $\gamma_{u_5} = \lbrack 0,1 \rbrack$
    \end{tabular},
    \label{eq:interactionFactors}
\end{equation}
Then we could summarize the matrix $R$ in \Cref{eq:interactionMatrix2} by writing
\begin{equation}
    R_{u,i} = \gamma_u \cdot \gamma_i.
    \label{eq:matfac1}
\end{equation}
The two blocks in $R$ might conceivably correspond to some feature in the data, e.g.~male and female users who buy men's and women's clothing. If so, the values in \Cref{eq:interactionFactors} would correspond to genders, though we discovered these factors automatically simply because they \emph{summarized the structure of the matrix}, rather than needing to rely on observed features.
%\todo{Kind of hate the above example}.

\emph{Matrix Factorization} follows this same idea, again by looking for underlying structure that explains observed interactions.

Essentially, our goal is to describe a matrix in terms of lower-dimensional factors, i.e.,
\begin{equation}
    \underbrace{\left[ \begin{tabular}{ccc}
    \ & \ & \ \\
    \ & R & \ \\
    \ & \ & \ 
    \end{tabular} \right]}_{N_u \times N_i} = 
    \underbrace{\left[ \begin{tabular}{c}
    \ \ \ \\
    $\gamma_U$\\
    \  
    \end{tabular}
    \right]}_{N_u \times K} \times
    \underbrace{\left[
    \begin{tabular}{ccc}
    \parbox{1mm}{\vspace{5mm}} & $\gamma_I^T$ & \parbox{1mm}{\vspace{5mm}}\\
    \end{tabular} \right]}_{K \times N_i}.
\end{equation}
%\todo{Kind of hate the notation above. Could just use U and I for number of users and items?}
%\todo{Probably should use |U| and |I| for the number of users and items}
That is, we are assuming that the matrix $R$, of dimension $N_u \times N_i$ (the number of users times the number of items), can be approximated by a `tall' matrix $\gamma_U$ and a `wide' matrix $\gamma_I$. 
%$\gamma_U$ and $\gamma_I$ essentially summarize the users and items (respectively)
Now, a single entry $R_{u,i}$ can be estimated by taking the corresponding row of $\gamma_U$ and column of $\gamma_I$:
\begin{equation}
    R_{u,i} = \gamma_u \cdot \gamma_i,
    \label{eq:matfac2}
\end{equation}
as in \Cref{eq:matfac1}. 
%\todo{Kinda ugly that this is *exactly* the same as the previous equation}
$\gamma_u \in \mathbb R^K$ is now a latent vector that describes a user, and $\gamma_i \in \mathbb R^K$ describes an item.

Examples of such vectors are depicted in \Cref{fig:matfac1}. Intuitively, $\gamma_u$ might be thought of as describing the `preferences' of the user $u$, whereas $\gamma_i$ describes the `properties' of item $i$. Then, the user $u$ will like (e.g.~give a high rating or interact with) the item $i$ if their preferences are \emph{compatible} with the item's properties (i.e.,~they have a high inner product). The latent dimensions, $\gamma_{\cdot,1}$, $\gamma_{\cdot,2}$ (etc.)~now describe those latent factors that best explain variability in $R$. For example, if such a model were trained on the Netflix dataset they might measure the extent to which a movie is a comedy or a romance, or the quality of its special effects. Again though, these factors are \emph{latent} and are discovered purely so as to maximally explain the observed interactions; models based on the principle of matrix factorization are commonly referred to as \emph{latent factor models}.

%\todo{Mention SVD and cite Koren paper}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/matfac1.pdf}
\caption{Representation of a user $u$ and item $i$ in a latent-factor model. \label{fig:matfac1}}
\end{SCfigure}

\subsection{Fitting the Latent Factor Model}

So far we have described the intuition behind modeling interactions in terms of latent user and item factors, but have not yet described how to fit a model based on this principle. That is, we would like to choose $\gamma_U$ and $\gamma_I$ so as to fit the interaction data most closely, e.g.~by minimizing some loss such as the Mean Squared Error, following the setting of the Netflix Prize:
%\todo{Don't think I mentioned that the Netflix Prize used the MSE yet? Maybe not necessary}
\begin{equation}
    \argmin_{\gamma} \frac{1}{|R|} \sum_{(u,i) \in R} (f(u,i) - R_{u,i})^2,
\end{equation}
where $f(u,i)$ is our prediction function $f(u,i) = \gamma_u \cdot \gamma_i$.

Note that this would be straightforward if the matrix $R$ were fully observed (i.e.,~every user rated every item). In this case, we could rely on the Singular Value Decomposition (SVD). Briefly, this decomposition allows us to factorize a real $M\times N$ matrix according to
\begin{equation}
    R = U\Sigma V^T,
\end{equation}
where $U$, $V$, and $\Sigma$ eigenvectors and eigenvalues of $R^TR$ and $RR^T$. Critically, if we keep just the $K$ eigenvectors corresponding to the largest eigenvalues, this corresponds to the \emph{best possible rank $K$ factorization in terms of the MSE} \cite{golub1971singular}.

However, the Singular Value Decomposition is defined only for \emph{fully observed} matrices, rather than partially observed interactions as in $R$. Even if this could be addressed (e.g.~via a data imputation strategy on the missing values), it would not be practical to compute the SVD on a matrix that could potentially have millions of rows and columns.

Instead, we seek a solution based on gradient descent. When minimizing the Mean Squared Error, the solution is similar to the one we saw in \Cref{sec:gradient}. Note that as usual we should be careful to split interactions $(u,i) \in R$ into training, validation, and test sets, and to include a regularizer to avoid overfitting, as we describe below.

\subsubsection{User and Item Biases}

Before describing the gradient-descent-based solution in detail, we first suggest some steps to improve the solution.

Although a simple solution of the form $r(u,i) = \gamma_u \cdot \gamma_i$ seems to capture the types of interactions we want, it is difficult to regularize. Consider adding a simple $\ell_2$ regularizer such as
\begin{equation}
    \Omega(\gamma) = \lambda \left( \sum_{u = 1}^{|U|} \sum_{k = 1}^K \gamma_{u,k}^2 +
                                    \sum_{i = 1}^{|I|} \sum_{k = 1}^K \gamma_{i,k}^2
                            \right)
    \label{eq:latentreg1}
\end{equation}
%\todo{Never use the notation Omega elsewhere. Maybe use other places or drop here.
%\todo{Write |I| / |U| here but is N_i / N_u above. Like |I| better but I forget why I did N_i in the first place}
This regularizer (for large $\lambda$) will encourage the parameters to be close to zero; as such the predictions $\gamma_u \cdot \gamma_i$ will also be pushed toward zero, and the system will systematically underpredict ratings.

There are several ways this could be avoided. Trivially, we might simply subtract the mean ($\bar{R}$) from all ratings before training so that they are centered around zero. Alternately, recall as in \Cref{sec:?} that we were careful to exclude the intercept term $\theta_0$ from our regularizer.
%\todo{That section doesn't exist yet}
Although our current model lacks such an intercept term, we can straightforwardly add one:
\begin{equation}
    r(u,i) = \alpha + \gamma_u \cdot \gamma_i.
\end{equation}
Note that we still regularize as in \Cref{eq:latentreg1}.

Although the offset term $\alpha$ corrects the problem of systematically underpredicting ratings, it retains a similar issue at the level of individual users or items. Again, the regularizer pushes $\gamma_u$ and $\gamma_i$ toward zero, and therefore pushes predictions toward $\alpha$. But individual users or items may tend to systematically give much higher (or lower) ratings than $\alpha$, meaning that our regularizer again encourages us to systematically under (or over) predict.

Again we correct for this by adding additional bias terms, this time at the level of individual users or items:
\begin{equation}
    r(u,i) = \alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i.
    \label{eq:MFscore}
\end{equation}
$\beta_u$ now encodes the extent to which user $u$'s ratings trend higher or lower than $\alpha$, and $\beta_i$ encodes the extent to which item $i$ tends to receive higher or lower ratings than $\alpha$.\footnote{Note that we are careful not to refer to $\alpha$ as an `average' rating, and indeed in general once we fit the model $\alpha \not= \bar{R}$, just as the offset $\theta_0$ is not the average $\bar{y}$ in a linear regression model.}

Adding these bias terms introduces an additional $|U| + |I|$ parameters to the model. Whether these terms should be included in the regularizer $\Omega$ is arguable: on the one hand they are similar to offset terms, which we would normally not regularize (for the same reason we do not regularize $\alpha$); on the other hand, failing to regularize them may lead to overfitting. In practice, the terms may simply be included in the regularizer:
\begin{equation}
    \lambda\Omega(\beta, \gamma) = \left(
    \sum_{u = 1}^{|U|} \beta_u^2 + \sum_{k = 1}^K \gamma_{u,k}^2 + 
    \sum_{i = 1}^{|I|} \beta_i^2 + \sum_{k = 1}^K \gamma_{i,k}^2
                      \right).
    \label{eq:latentreg2}
\end{equation}
Alternately one could regularize $\beta$ and $\gamma$ with different stregths (since we generally expect $\beta$ to have larger values:
\begin{equation}
    \lambda_1\Omega(\beta) + \lambda_2\Omega(\gamma) = \lambda_1 \left(
    \sum_{u = 1}^{|U|} \beta_u^2 + \sum_{i = 1}^{|I|} \beta_i^2 \right) + 
    \lambda_2 \left(
    \sum_{u = 1}^{|U|} \sum_{k = 1}^K \gamma_{u,k}^2 + 
    \sum_{i = 1}^{|I|} \sum_{k = 1}^K \gamma_{i,k}^2
                      \right),
    \label{eq:latentreg3}
\end{equation}
though doing so is difficult as it results in multiple regularization constants to tune.

\subsubsection{Gradient Update Equations}

Finally, the objective we wish to minimize (on a training set of interactions $\mathcal T$) is
\begin{eqnarray}
\text{obj}(\alpha; \beta; \gamma | \mathcal{T}) & = & \frac{1}{\mathcal T} \sum_{(u,i) \in \mathcal{T}} (r(u,i) - R_{u,i})^2 + \lambda \Omega(\beta, \gamma)\\
%\todo{check above}
    & = & \frac{1}{\mathcal{T}} \sum_{(u,i) \in \mathcal{T}} (\alpha + \beta_i + \beta_u + \gamma_i + \gamma_u - R_{u,i})^2 + \lambda \Omega(\beta, \gamma).
    \label{eq:MFObjective}
\end{eqnarray}
%\todo{Consistent notation for R_{(u,i)} in \mathcal{T} vs just (u,i) \in R or whatever.
Assuming the regularizer takes the form given in \Cref{eq:latentreg2}, the partial derivatives (for $\alpha$, $\beta_u$, and $\gamma_{u,k}$) are given by:
\begin{eqnarray*}
\frac{\partial \text{obj}}{\partial \alpha} & = & \sum_{(u,i) \in \mathcal{T}} 2(r(u,i) - R_{u,i})^2\\
\frac{\partial \text{obj}}{\partial \beta_u} & = & \sum_{i \in I_u} 2(r(u,i) - R_{u,i})^2 + 2\beta_u\\
\frac{\partial \text{obj}}{\partial \gamma_{u,k}} & = & \sum_{i \in I_u} 2\gamma_{i,k}(r(u,i) - R_{u,i})^2 + 2\gamma_{u,k}.
\label{eq:MFgradient}
\end{eqnarray*}
Note the change of summation in the last two terms: the derivative for user $u$ is based only on items $I_u$ that they consumed (in the training set). Derivatives for $\beta_i$ and $\gamma_{u,k}$ can be computed similarly and are left as an exercise (\Cref{ex:?}).

\paragraph{Other Considerations for Gradient Descent}
When we first introduced gradient descent in \Cref{sec:gradient}, we noted some potential issues in terms of local minima, learning rates etc.
It is worth revisiting some of those issues in light of the more complex model we are fitting here:
\begin{itemize}
    \item The problem in \Cref{eq:MFObjective} is certainly non-convex and has many local minima.\footnote{The proof can roughly be sketched as follows. The objective is smooth, and given any global optimum $\gamma_U$, $\gamma_I$, any permutation applied to both (i.e.,~$\gamma_U\pi$ and $\gamma_I\pi$) will result in an \emph{equivalent} local optimum.} Surprisingly though, this problem is not prone to `spurious' local optima, and if carefully implemented should converge to a global optimum \cite{ge2016matrix}.
    \item Nevertheless the problem is sensitive to initialization. For example one if multiple columns of $\gamma_U$ and $\gamma_I$ are initialized to the same value, they will have identical gradients and will remain in `lock step' during successive iterations. This can normally be avoided simply by randomized initialization.
    \item Rather than computing the full gradient as in \Cref{eq:MFgradient}, alternate approaches such as stochastic gradient descent, or alternating least squares, may converge faster or require less memory.\footnote{Alternating Least Squares notes that the optimization problem in \Cref{eq:MFObjective} has a closed form if either $\gamma_U$ or $\gamma_I$ is fixed; optimization proceeds by alternately fixing one term and optimizing the other.} See e.g.~\cite{bottou2010large,yu2012scalable}.
\end{itemize}
% The gradient descent strategy presented above
% Faster gradient descent (coordinate descent): \cite{yu2012scalable}


\section{Implicit Feedback and Ranking Models}

So far, our discussion of (model-based) recommender systems has focused on predicting real-valued outcomes such as ratings, using objectives based on the Mean Squared Error. That is, we have described model-based recommendation in terms of \emph{regression} approaches.

Just as we developed separate approaches for neighborhood-based recommendation when considering click, purchase, or rating data, here we consider how our regression-based approaches should be adapted to handle binary outcomes (such as clicks and purchases).

Na\"ively, we might imagine that we could adapt our regression-based approaches to handle binary outcomes much in the same way we that developed logistic regression. That is, we could pass the model output (\Cref{eq:MFscore}) through a sigmoid function, such that positive interactions are associated with high probabilities, and negative interactions are associated with low probabilities.

However, when dealing with click or purchase data, we should consider that items which haven't been clicked or purchased are not necessarily \emph{negative} interactions---in fact items that \emph{haven't} been clicked or purchased are exactly the ones we intend to recommend.

Several techniques have been proposed to handle recommendation in this context. Often this setting is referred to as \emph{one-class} recommendation, as only the `positive' class (clicks, purchases, listens, etc.)~is observed.

Also called implicit feedback recommendation

\subsection{Instance Re-weighting Schemes}

One category of methods for dealing with implicit feedback data attempts to reweight instances as having various `confidences' of being positive or negative.

\cite{hu2008collaborative} considers cases where positive instances are associated with `confidence' measures $r_{u,i}$, which could measure e.g.~the number of times a user listened to a song; in \cite{hu2008collaborative} the confidence values encode how many times a user has watched a particular program. Negative instances still have $r_{u,i} = 0$, such that the model essentially captures the fact that negative instances are necessarily associated with low confidence, whereas confidence may vary substantially among positive instances.

Ultimately, the goal is still to predict a binary outcome $p_{u,i}$, and the model is trained to predict
\begin{equation}
    p_{u,i} = \left\lbrace \begin{tabular}{cl}
    1 & if $r_{u,i} > 0$\\
    0 & otherwise
    \end{tabular}\right.
\end{equation}.
The form of the predictor is similar to that of \Cref{eq:MFObjective}, i.e.,~latent user and item factors are use to predict $p_{u,i}$ using a (regularized) Mean Squared Error. The main difference is that the MSE is weighted according to the confidence of each observation:
\begin{equation}
    \argmin_\gamma \sum_{(u,i) \in \mathcal T} c_{u,i}(p_{u,i} - \gamma_u \cdot \gamma_i)^2 + \lambda \Omega(\gamma),
    \label{eq:weightedMF}
\end{equation}
where $c_{u,i}$ is a weighting function associated with each observation, which ultimately is a monotone transform of $r_{u,i}$, e.g.
\begin{equation}
    c_{u,i} = 1 + \alpha r_{u,i}; \quad \text{or} \quad c_{u,i} = 1 + \alpha \log(1 + r_{u,i} / \epsilon),
\end{equation}
where $\alpha$ and $\epsilon$ are tunable hyperparameters. Note that the transform $c_{u,i}$ ensures that negative instances have small but non-zero weight, whereas positive instances receive increasingly higher weight according to their associated confidence.

%Paper 1: Collaborative Filtering for Implicit Feedback Datasets

\cite{pan2008one} approach the problem in a similar way, also fitting a function of the form in \Cref{eq:weightedMF}, though their weighting scheme is applied to negative instances. Several schemes are proposed, two of which are as follows:
\begin{equation}
  c_{u,i} = \alpha \times | I_u |; \quad \text{or} \quad c_{u,i} = \alpha ( m - | U_i | ).
\end{equation}
The first (which they call `user oriented' weighting) suggests that a negative instance should be weighted higher if the corresponding \emph{user} has interacted with many items; the second assumes that a negative instance should be weighted higher if the corresponding \emph{item} has few associated interactions. 
%\cite{pan2008one} also explores sampling schemes,



% Although the scheme above is ultimately a simple heuristic for reweighting the model we developed in \Cref{eq:MFObjective}, experiments in \cite{hu2008collaborative} show that this scheme outperforms models that try to predict $r_{u,i}$ or $p_{u,i}$ directly.

Although the schemes above are ultimately simple heuristics for reweighting the model we developed in \Cref{eq:MFObjective}, experiments in \cite{hu2008collaborative}
and
\cite{pan2008one}
show that these scheme outperforms models that try to predict $p_{u,i}$ (or $r_{u,i}$) directly.


%\subsection{Contrastive Learning for Personalized Ranking}
%\todo{generic section title, could be better}
\subsection{Bayesian Personalized Ranking}

While the above reweighting schemes demonstrate the importance of treating `negative' and `positive' feedback carefully in implicit-feedback settings, 
%they are ultimately simple heuristics whose success depends upon whether they can be adapted or tuned to a specific application or dataset.
they ultimately optimize \emph{regression} objectives, and therefore still seek to assign `negative' scores to unseen instances.

A potential objection to such an approach is that the unseen instances are exactly the ones we \emph{want} to recommend, and thus we should not encourage a model to assign them a negative score. A weaker assumption might state that while unseen instances should have \emph{lower} scores than positive instances, they need not have negative scores. That is, items which we know a user likes are more positive than unseen items, but unseen items could still be positive.

%Paper 2: BPR
%The above papers treated positive and negative instances differently, but still relied on regression objectives. As such, non-observed instances are still 
\cite{rendle2012bpr} built models based on the above principle by borrowing ideas from ranking. Recall from \Cref{sec:ranking} that the above principle is similar to our goal when adapting classifiers for ranking: while positive (or relevant) items should appear near the top of a ranked list, we are not concerned with their actual scores.

Likewise, the principle behind their method, Bayesian Personalized Ranking, is that we should generate ranked lists of items such that positive items appear first. This is achieved by training a predictor $x_{u,i,j}$ that assigns a score based on which of the two items ($i$ or $j$) is preferred (i.e.,~ranked higher) by $u$:
\begin{eqnarray}
    x_{u,i,j} > 0 & \rightarrow & \text{$u$ prefers $i$}\\
    x_{u,i,j} \leq 0 & \rightarrow & \text{$u$ prefers $j$}.
\end{eqnarray}
% \begin{equation}
%     \underbrace{\texttt{+++++++++}}_{\text{positives}}\underbrace{\texttt{?????????????}}_{\text{unseen items}}
% \end{equation}
Now, if we know that $i$ is a positive and $j$ is a negative (or unseen) item for user $u$, then a good model should tend to output positive values of $x_{u,i,j}$.
%we can measure whether the model correctly assigns a positive score by $\delta(x_{u,i,j} > 0)$, which we will later use to define our objective function.
%\todo{poor organization}

$x_{u,i,j}$ could be any predictor, though the most straightforward option is to define it in terms of \emph{difference} between pairwise predictors, e.g.:
\begin{equation}
    x_{u,i,j} = \underbrace{x_{u,i}}_{\mathllap{\text{$u$'s preference toward $i$}}\hspace{-5mm}} - \underbrace{x_{u,j}}_{\hspace{-5mm}\mathrlap{\text{$u$'s preference toward $i$}}}.
\end{equation}
For instance, the pairwise compatibility could be defined via a latent factor model, similar to that of \Cref{eq:???}:
\begin{equation}
    x_{u,i,j} = x_{u,i} - x_{u,j} = \gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j.
\end{equation}
%\todo{Replace all \clap with \mathclap or silimar (same for rlap and llap), which preserves font size}
Again, note that our goal is not that $\gamma_u \cdot \gamma_i$ should be positive for the positive item, nor thot $\gamma_u \cdot \gamma_j$ should be negative for the unseen item, only that the \emph{difference} is positive, i.e.,~that the positive item has a higher compatibility score.

We can now define our objective in terms of whether the model correctly outputs positive values $x_{u,i,j}$ given a positive item $i$ and an unseen item $j$.
%\todo{Kind of want to use x_{uij} rather than x_{u,i,j} (etc.) everywhere}
Ideally, we would like to count how often the model is able to correctly rank positive items higher than unseen items. For a specific user $u$ we have:
\begin{equation}
    \mathrm{AUC}(u) = \frac{1}{|I_u||\mathcal I \setminus I_u|} \underbrace{\sum_{i \in I_u \vphantom{j \in \mathcal I \setminus I_u}}}_{\mathllap{\text{positive items for user $u$}}\hspace{-5mm}} \underbrace{\sum_{j \in \mathcal I \setminus I_u}}_{\hspace{-5mm}\mathrlap{\text{unseen items for user $u$}}} \delta(x_{u,i,j} > 0).
    \label{eq:bprOpt1}
\end{equation}
The name `AUC' stands for `Area Under Curve,' which originates from a different way of deriving the same quantity.\footnote{Specifically, it is the Area Under the Receiver-Operating Characteristic Curve, which is a curve that relates the false positive and true positive rates of a classifier.}
%\todo{Cover ROC curve along with Precision Recall curves, and drop this footnote and previous sentence.}
For an entire dataset we average the above across all users:
\begin{equation}
    \mathrm{AUC} = \frac{1}{| \mathcal U |} \mathrm{AUC}(u)
\end{equation}
%\todo{AUC should probably have an argument, like AUC(\gamma|something)}
Note that this quantity takes a value between 0 and 1, where an AUC of 1 means that the model always ranks positive items higher than unseen items; an AUC of 0.5 means that the model is no better than random.

Optimizing the above presents two issues. First, it is not feasible to consider all $(u,i,j)$ triples; to address this one can randomly sample a fixed number of unseen items $j$ per positive item $i$.\footnote{Furthermore, the sampled items could change during each iteration of training.}

Second, the objective in \Cref{eq:bprOpt1} is a step function, whose derivative is zero almost everywhere. This is much the same issue we encountered when developing logistic regression in \Cref{sec:logisticRegression}; as such we can take the same approach by replacing the step function $\delta(x_{u,i,j})$ with a differentiable surrogate (see \Cref{fig:sigmoid}). Using the sigmoid function allows us to interpret $\sigma(x_{u,i,j})$ as a probability:
\begin{equation}
    p(\text{$u$ prefers $i$ over $j$}) = \sigma(x_{u,i,j}).
\end{equation}
From this point, optimization proceeds in much the same way as we developed logistic regression: we use $\sigma(x_{u,i,j})$ to define a (log-)probability of a model given a training set, and subtract a regularizer:
\begin{eqnarray}
  \mathrm{obj}^{\text{(BPR)}} & = & \ell (\gamma ; \mathcal T) - \lambda \Omega(\gamma)\\
                  & = & \sum_{(u,i,j) \in \mathcal T} \ln \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j) - \lambda \Omega(\gamma).
\end{eqnarray}
Assuming an $\ell_2$ regularizer $\Omega(\gamma) = \| \gamma \|_2^2$, we can compute the derivative, for example with respect to $\gamma_{u,k}$:
\begin{eqnarray}
    \frac{\partial \text{obj}}{\partial \gamma_{u,k}} & = & \sum_{(u,i,j) \in \mathcal T} \ln \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j) - \lambda \| \gamma \|_2^2\\
    & = & \frac{\partial}{\partial \gamma_{u,k}} \sum_{(u,i,j) \in \mathcal T} -\ln(1 + e^{\gamma_u \cdot \gamma_j - \gamma_u \cdot \gamma_i}) - \lambda \| \gamma \|_2^2\\
    & = & \sum_{(i,j) \in I_u} (\gamma_{i,k} - \gamma_{j,k}) (1 - \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j)) - 2\lambda\gamma_{u,k}
\end{eqnarray}
%\todo{Use \ln and \log consistently}
%\todo{\mathcal l or \ell ?}
%\todo{\gamma | T , or \gamma ; T, or something else?}
%\todo{Do I ever call the regularizer an \ell_2 regularizer? Maybe just drop}
(we abuse notation slightly so that $(i,j) \in I_u$ includes both positive and unseen items sampled from a user).
%\todo{finish}

\section{Evaluating Recommender Systems}

%see e.g. http://ceur-ws.org/Vol-910/rue2012.pdf

So far, when developing models to predict ratings as in \Cref{sec:latentFactor}, we have done so by optimizing objectives based on a \emph{sum of squared errors} (or equivalently, a Mean Squared Error). Recall that in \Cref{sec:whyMSE}, we discussed the motivation behind the Mean Squared Error, as well as some potential pitfalls when using it.

In the case of recommender systems, we must be aware of the same pitfalls, but also some different ones. Critically, since since the system is likely used to provide ranked lists of items to the user, actual prediction of ratings may not be critical so long as desirable items appear near the top of the ranking.  
%
% Paper: Precision-Oriented Evaluation of Recommender Systems:
% An Algorithmic Comparison
%
%misclassified decoys (see refs in http://proceedings.mlr.press/v81/ekstrand18b/ekstrand18b.pdf)
%
Consider, for example, some potential problems with the Mean Squared Error in a recommendation context:
\begin{itemize}
  \item When using the MSE, mispredicting a 5-star rating as 4-stars incurs a smaller penalty than mispredicting a 3-star rating as 1-star. Arguably, the latter should have a smaller penalty, as it concerns an item which should never have been recommended anyway.
  \item Similarly, mispredicting ratings of 3 and 3.5 as 4 and 4.5 (respectively) would incur a larger penalty than mispredicting them as 3.5 and 3. However the former preserves the \emph{ordering} of the two items, whereas the latter does not.
  \item As we saw in \Cref{sec:motivationBehindMSE}, the MSE corresponds to an implicit assumption that errors are normally distributed; critically this assumes that outliers are extremely rare and should be penalized accordingly. In practice, outliers may be common, or alternately 
  %\todo{Finish}
  \item \cite{bellogin2011precision} noted the issue of `popularity bias,' whereby strong performance on popular items can mask performance issues for less-popular ones.\footnote{Though this is not an issue with the MSE specifically, but rather a general problem of evaluation in imbalanced datasets.}
\end{itemize}

Ultimately, such problems raise the question of whether reductions in Mean Squared Error actually correspond to increased utility of a recommender system. Interestingly, it is reported in \cite{koren2009collaborative} that a carefully implemented model with temporally-evolving bias terms (which we describe in \Cref{sec:caseStudyNetflixTemporal}) outperformed Netflix's previous solution (\emph{Cinematch}) in terms of the RMSE. Critically, a system without any interaction term can do little more than recommend popular items over time; 

Some of the above issues suggest the use of alternative regression metrics, such as the Mean Absolute Error, which (for example) is less sensitive to outliers, as we argued in \Cref{sec:where?}. 
%Others require more subtle solutions; for example
Others suggest that perhaps a recommender system should be evaluated less like a regression problem and more like a ranking problem. 
Arguably, such problems with the Mean Squared Error are driving research toward settings that rely on implicit feedback (clicks, purchases, etc.)~rather than ratings; alternately, these settings may be preferable simply because such data is more available than explicit feedback such as ratings.
%\todo{finish}

We already saw in \Cref{sec:BPR} one technique to train recommender systems to optimize a ranking loss based on implicit feedback, namely the AUC. Conceptually, the AUC reflects our ability to guess which of two items is `relevant:' an AUC of 1 means that we always correctly select the correct item, whereas an AUC of 0.5 means our guesses are no better than random.

However, the AUC is but one choice of ranking loss, and was primarily chosen for its convenience when formulating the optimization problem in \Cref{eq:BPRloss?}. As in \Cref{sec:rankingLosses?}, when considering cases where recommendations are surfaced to a user via an interface, we may be particularly interested in how the recommender system performs among the top $K$ ranked items.

Below we present a few alternative evaluation functions to measure recommendation performance, most of which are focused on achieving high accuracy among the top-ranked items.

\paragraph{Recall @ $K$ and Precision @ $K$}

When we evaluated classifiers in \Cref{sec:evaluatingClassifiers}, we motivated the precision and recall @ $K$ useful metrics in the context of evaluating user interfaces, where we have a fixed budget ($K$) of results that can be returned. Likewise, when recommending items, we might consider whether relevant items (e.g.~those that a user eventually interacts with) are given a high ranking.

For convenience, when evaluating recommenders in this setting, it is useful to define a variable $\mathit{rank}_{u,i}$ that specifies in what position an item $i$ was ranked for a particular user $u$. That is, given a compatibility function $f(u,i)$, and a set of $N$ items that can potentially be recommended (potentially excluding e.g.~interactions that already appeared in the training set), then $\mathit{rank}_{u,i} \in \lbrace 1 \ldots N \rbrace$ such that
\begin{eqnarray}
    \mathit{rank}_{u,i} < \mathit{rank}_{u,j} & \Leftrightarrow & f(u,i) > f(u,j)\\
    \mathit{rank}_{u,i} = \mathit{rank}_{u,j} & \Leftrightarrow & i = j.
\end{eqnarray}
Now, given a test set of observed interactions $I_u$ we could define the precision@$K$
%\todo_{formatting}
(for a particular user $u$) as
\begin{equation}
  \mathit{prec}@K(u) = \frac{|\lbrace i \in I_u | \mathit{rank}_{u,i} \leq K \rbrace|}{K}.
\end{equation}
As in \Cref{eq:?}, the numerator is the number of relevant items that were retrieved, while the denominator is the number of retrieved items. Now to compute the precision @ $K$ we simply average over all users:
\begin{equation}
    \mathit{prec}@K = \frac{1}{|U|} \sum_{u \in U}\mathit{prec}@K(u).
\end{equation}
%\todo_{formatting, rank_{u,i} or rank(u,i), also prec or precision or Precision or \text{Precision}}
Likewise the recall @ $K$ is defined similarly:
\begin{equation}
    \mathit{recall}@K = \frac{1}{|U|} \sum_{u \in U} \frac{|\lbrace i \in I_u | \mathit{rank}_{u,i} \leq K \rbrace|}{|I_u|}
\end{equation}

%\paragraph{Hit Rate}

%\todo_{Are there any papers that use recall or do they all call it hit rate?}

%One potential challenge with the precision and recall as defined above is that we are averaging across users who may have very different numbers of relevant items $I_u$ in the test set; optimizing an algorithm 
%\todo_{finish}

\paragraph{Mean Reciprocal Rank}

%\todo_{Above is quite sloppy}

Mean Average Precision

NDCG? Probably don't bother

\paragraph{Novelty, Diversity, Serendipity, etc.}
%\todo{Should I mention all of those things? I don't think they're covered elsewhere}

Serendipity: \cite{herlocker2004evaluating} 
%(serendipity, according to http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.62.9683&rep=rep1&type=pdf)

Coverage: An algorithmic framework for performing collaborative filtering (maybe)?
%according to same ref above

Finally, it should be noted that there are several other qualities we might desire from a recommender system beyond its immediate utility to users (i.e.,~its ability to predict the next action). For example, we might be interested in exposing users to diverse viewpoints, ensuring that recommendations aren't biased against certain groups, that users don't get pushed toward `extreme' content, etc. We'll revisit these issues in \Cref{chap:ethics}, when we consider the broader consequences and ethics of personalized machine learning.
%\todo{check above paragraph}


% \subsection{User Satisfaction}

% \subsection{Online Evaluation of Recommender Systems}

\subsection{Online Evaluation}

See some refs in http://proceedings.mlr.press/v81/ekstrand18b/ekstrand18b.pdf

\section{Aggregation Functions for Recommender Systems}
\label{sec:aggregation}
%\todo{funny section titles here, just replace with "deep learning" or something}

\subsection{Why the Inner Product?}

Discuss that recent paper by the BPR author

Could discuss deep learning papers: e.g. the two youtube papers


\section{Retrieval}
%\todo_{Probably collapse these to a paragraph somewhere}

Maximum inner product search

Efficient nearest neighbor

%\subsection{Recommendation based on Euclidean Distance, Translation, Etc.}
%Covered in seequential recsys section



% \section{Recommending for Diversity}
%\todo{Could put in ethics chapter or whatever}

\section{Recommender Systems in Python with \emph{Surprise} and \emph{Implicit}}

Although the types of recommender systems we've seen so far can (with some effort) be implemented `from scratch' either by computing the gradient expressions as in \Cref{eq:latentFactorGradient} or by using high-level optimization libraries like \emph{Tensorflow},
%\todo{capitalization}
the recommendation techniques we've covered so far are reasonably well-supported by popular Python
%\todo{italics?}
libraries.

Here

\subsection{Latent-Factor Models (Surprise)}

\emph{Surprise} \cite{Hug2020}
%\todo{Modified citation, doesn't allow url}
is a library that implements various collaborative filtering
%\todo{Do I ever use this word?}
methods and algorithms based on \emph{explicit feedback} (e.g.~ratings). Below we show how to use \emph{Surprise}'s
%\todo{Formatting}
implementation of a latent factor model as in \Cref{eq:latentWhich?} (`SVD' as in \cite{Koren?}).

First we import the model (`SVD'), and utilities to read and split the dataset:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from surprise import SVD, Reader, Dataset
from surprise.model_selection import train_test_split
\end{lstlisting}
\end{minipage}

While the library has various routines to read in the dataset, the most straightforward is to read the data from a CSV/TSV file. Here we've processed the \emph{Goodreads} `fantasy' data
%\todo{Do I talk about this dataset above?}
to extract just the `user\_id,' `book\_id,' and `rating' fields, though this example could be applied to any similar dataset. After reading in the data, we split it into train and test fractions, with 25\% of the data withheld for testing:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
reader = Reader(line_format='user item rating', sep='\t')
data = Dataset.load_from_file("goodreads_fantasy.tsv", reader=reader)
dataTrain, dataTest = train_test_split(data, test_size=.25)
\end{lstlisting}
\end{minipage}

Next we fit the model, and collect its predictions on the test set:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
model = SVD()
model.fit(dataTrain)
predictions = model.test(dataTest)
\end{lstlisting}
\end{minipage}

From `predictions' we can then extract the 

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
sse = 0
for p in predictions:
    sse += (p.r_ui - p.est)**2

print(sse / len(predictions))
\end{lstlisting}
\end{minipage}

%predictions[0].est




\subsection{Bayesian Personalized Ranking (Implicit)}

\emph{Implicit}\footnote{\url{https://github.com/benfred/implicit}} is a library for recommender systems that operate on implicit feedback datasets, as in \Cref{sec:implicit}. Here we show how to use the library for Bayesian Personalized Ranking, as in \Cref{sec:bpr}.

First, we read in the data. This time, the required data format is a \emph{sparse} matrix describing all user/item interactions. Despite this matrix having hundreds of thousands of rows and columns, only observed interactions are stored:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from implicit import bpr

Xiu = scipy.sparse.lil_matrix((nItems, nUsers)) # Sparse matrix; initialized after extracting the number of users and items
for d in data:
    itemsToUsers[itemIDs[d['book_id']],userIDs[d['user_id']]] = 1 # Only storing positive feedback instances
    
Xui = scipy.sparse.csr_matrix(Xiu.T)
\end{lstlisting}
\end{minipage}
%\todo{Have I shown how to read the data? Could put here}
%\todo{Sometimes call it 'data' and sometimes 'dataset', should be consistent}

Next, we initialize and fit the BPR model:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
model = bpr.BayesianPersonalizedRanking(factors = 5)
model.fit(Xiu)
\end{lstlisting}
\end{minipage}
%\todo{Make the various examples have the same number of factors}

Having fit the model, we 

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
itemFactors = model.item_factors
userFactors = model.user_factors

recommended = model.recommend(0, Xui) # Recommendations for the first user
related = model.similar_items(0) # Highly similar to the first item (cosine similarity)
\end{lstlisting}
\end{minipage}
%#

\section{History and Related Work}

\section{Problems}

\begin{problemlist}

\item Modify the Jaccard similarity implementation from \Cref{sec:?} to implement the cosine similarity. Compare the results of the two implementations on the Amazon data.
%\todo{hastily written}

\item The Jaccard similarity-based recommender we implemented in \Cref{sec:?} proved an effective recommender, though our implementation was inefficient. The main source of inefficiency was due to iteration over \emph{all} items. A more efficient implementation might first build smaller a \emph{candidate set} of items, by noting that only those items with at least one user in common with this the query could potentially have non-zero Jaccard similarity. This candidate set can be built by taking all users who have purchased the query $i$, and taking the union over \emph{other} items they have purchased (other than $i$), i.e.,~$\bigcup_{u \in U_i} I_u \setminus \lbrace i \rbrace$. After modifying our implementation from \Cref{sec:jaccard} to use this candidate set, confirm that it produces identical recommendations, and compare its running time to the na\"ive implementation.


\item Implement rating prediction models based on the formulas in \Cref{eq:simPredict,eq:simPredict2,eq:simPredict3}. Compare the three in terms of their Mean Squared Error (using either the entire dataset or a random sample).
%\todo{Which dataset?}

-- Show that the Pearson correlation is related to the $R^2$ coefficient.

-- Some comparison of the various recommender systems error metrics, possibly following the rating prediction experiment above.

\item Before implementing the recommender system described in \Cref{eq:MFOjbective}, it is instructive to implement simpler variants in order to understand the gradient descent procedure. Try implementing a \emph{bias-only} model, i.e.,~one that makes predictions according to $r(u,i) = \alpha + \beta_u + \beta_i$.
%\todo{finish}

\item Compute the derivatives $\frac{\partial \text{obj}}{\partial \beta_i}$ and $\frac{\partial \text{obj}}{\partial \gamma_{i,k}}$ for the objective from \Cref{eq:MFObjective}.
\item (Hard) In \Cref{sec:latentfactor} we predicted star ratings using a model that optimized the Mean Squared Error. However the ratings we are predicting in many datasets are \emph{integer} valued, e.g.~$r_{u,i} \in \lbrace 1,2,3,4,5 \rbrace$. In light of this, it might be tempting to round the predictions of our model to the nearest integer. In this exercise, we'll show that this type of rounding is not effective, i.e.,~that the rounded predictions will tend to have higher MSEs. For this exercise, suppose that when your model outputs a prediction number $a$, the correct answer is $\lfloor a \rfloor$ with
probability $\lceil a \rceil - a$, and $\lceil a \rceil$ with probability $a - \lfloor a \rfloor$ (e.g.~if it outputs 4.2 then the correct answer is 4 80\% of the time and 5 20\% of the time.
%\todo{Rewrite above, copied from class}
\end{problemlist}

\part{Modern Applications of Personalization}
%Not a great name for part...

\chapter{Content and Structure in Recommender Systems}
%\todo{Could fold into above chapter, depending on scale}

So far, the systems we've built for personalized recommendation have been based purely on interaction data. We argued in \Cref{sec:?} as to why interactions are often sufficient to capture all of the critical signals that we need.

\section{The Factorization Machine}

Factorization Machines \cite{rendle2010factorization} are a general-purpose approach that seeks to incorporate general features into models that capture pairwise interactions.

In essence, the factorization machine extend the approach behind the latent factor model (\Cref{sec:latentfactor}). The latent factor model embeds users and items into low dimensional space via $\gamma_u$ and $\gamma_i$, and then models the interaction between them via an inner product; the factorization machine extends this approach to incorporate arbitrary pairwise interactions between users, items, and other features.

The input to the model is a feature matrix $X$ and a target $y$. In the simplest case, $X$ might simply encode the identity of the user and item via a one-hot encoding, though can be extended to incorporate any additional properties associated with the interaction:
\begin{equation}
{
\left[
    \begin{tabular}{c}
    1000000 $\ldots$ 000100000 $\ldots$ 0001000 $\ldots$ 15.95\\
    0001000 $\ldots$ 000000010 $\ldots$ 0001000 $\ldots$ 12.25\\
    0100000 $\ldots$ 000100000 $\ldots$ 0000010 $\ldots$ 15.00\\
    0000100 $\ldots$ 010000000 $\ldots$ 0010000 $\ldots$ 17.50\\
    1000000 $\ldots$ 000000010 $\ldots$ 1000000 $\ldots$ 19.95\\
    $\smash{\underbrace{0000100}_{\text{user}}}\vphantom{0}$ $\ldots$ 
    $\smash{\underbrace{000010000}_{\text{item}}}\vphantom{0}$ $\ldots$
    $\smash{\underbrace{0000010}_{\text{weekday}}}\vphantom{0}$ $\ldots$
    $\mathclap{\smash{\underbrace{\hphantom{0}}_{\mathrlap{\text{price}}}}}1$
    \end{tabular}
\right]
}
%\todo{Make above not ugly}
\label{eq:FMinteractionMatrix}
\end{equation}

The basic idea of the factorization machine is then to model \emph{arbitrary} interactions between features. Each feature dimension is associated latent representation $\gamma_i$; the model equation is then defined in terms of all \emph{pairs} of (non-zero) features:
\begin{equation}
    f(\mathbf{x}) = \underbrace{w_0 + \sum_{i=1}^F w_i x_i}_{\text{offset and bias terms}} + \underbrace{\sum_{i = 1}^n \sum_{j = i+1}^n \langle \gamma_i, \gamma_j \rangle x_i x_j}_{\text{feature interactions}}.
    %\todo{Summation goes up to n (here and below), but I don't think I ever defined n?}
    \label{eq:FMequation}
\end{equation}

It is instructive to consider the case where the interaction matrix in \Cref{eq:FMinteractionMatrix} includes \emph{only} a user and item encoding. In such a case, \Cref{eq:FMequation} expands to be identical to the latent factor model from \Cref{eq:latentFactor?}, i.e.,~the only interaction term is $\gamma_u \cdot \gamma_i$ for a user $u$ and item $i$. 

As such, the Factorization Machine can be viewed as a generalization of a latent factor model, that allows for additional types of interactions to be considered. For example, if we include an additional one-hot feature in \Cref{eq:FMinteractionMatrix} that encodes the \emph{previous} item the user consumed, then the factorization machine
%\todo{capitalization?}
will include an expression encoding the compatibility of the next item with the previous one, i.e.,~the model can learn how \emph{contextually relevant} the previous item is compared to the next one. It is useful to compare this approach to the models we design specifically to handle sequential inputs in \Cref{sec:sequential}. \cite{rendle2010factorization} discusses such topics, describing the extent to which the general-purpose factorization machine formulation subsumes various approaches designed to handle specific types of features.

\cite{rendle2010factorization} describes how the model equation of \Cref{eq:FMequation} can be computed efficiently (and how parameter learning can be done efficiently), by showing that the interaction term can be rewritten as
\begin{equation}
    \sum_{i = 1}^n \sum_{j = i + 1}^n \langle \gamma_i, \gamma_j \rangle x_i x_j = \frac{1}{2} \sum_{f = 1}^k \left( \left( \sum_{i = 1}^n \gamma_{i,f} x_i \right)^2 - \sum_{i = 1}^n \gamma_{i,f}^2 x_i^2 \right),
\end{equation}
which allows for computation that is $O(kn)$ (i.e.,~the dimension of the latent factors multiplied by the feature dimensionality).


\subsection{Factorization Machines in Python with \emph{FastFM}}

As we shall see later, 

\cite{bayer2016fastfm}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
userIDs,itemIDs = {},{}
for d in data:
    u,i = d['user_id'],d['book_id']
    if not u in userIDs: userIDs[u] = len(userIDs)
    if not i in itemIDs: itemIDs[i] = len(itemIDs)
    
nUsers,nItems = len(userIDs),len(itemIDs)
\end{lstlisting}
\end{minipage}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = scipy.sparse.lil_matrix((len(data), nUsers + nItems))
for i in range(len(data)):
    user = userIDs[data[i]['user_id']]
    item = itemIDs[data[i]['book_id']]
    X[i,user] = 1
    X[i,nUsers + item] = 1

y = scipy.array([d['rating'] for d in data])
\end{lstlisting}
\end{minipage}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X_train,y_train = X[:2000000],y[:2000000]
X_test,y_test = X[2000000:],y[2000000:]

fm = fastFM.als.FMRegression(n_iter=1000, init_stdev=0.1, rank=2, l2_reg_w=0.1, l2_reg_V=0.5)

fm.fit(X_train, y_train)
y_pred = fm.predict(X_test)
\end{lstlisting}
\end{minipage}
%\todo_{Explain above parameters}
%https://ibayer.github.io/fastFM/tutorial.html

\section{Cold-Start Recommendation}

So far, all of the recommendation approaches we have developed have depended on having detailed \emph{interaction histories} associated with users and items. Naturally, we cannot find similar users (as in \Cref{sec:neighborhood}) to a user who has no purchase history; likewise, we cannot estimate latent parameters $\gamma_u$ (or even a bias $\beta_u$) for a user who has never rated or purchased any items (as in \Cref{sec:latentFactor}).

As such, we need to develop recommendation approaches that can be useful in so-called \emph{cold-start} scenarios. Depending on the setting, either users or items may be `cold' (i.e.,~have zero associated interactions). 

We'll investigate two categories of approach to deal with cold-start problems. First, one may attempt to deal with cold-start situations via the use of \emph{side-information} about users or items. Side information could range from product images, to text, or social interactions
%\todo{finish}

\subsection{Addressing Cold-Start Problems with Side Information}
%Pairwise Preference Regression for Cold-start
%Recommendation (feature based)

In the absence of historical interaction data associated with users or items, one option is to resort to secondary signals. 
\cite{park2009pairwise} considers cold-start settings in the context of movie recommendation. For movies, associated features are available such as the release year, genre, (etc.),~which can be encoded (for example) as a one-hot vector. For users, demographic features are used such as a user's age, gender, occupation, or location. These features are captured via user and item feature vectors $x_u$ and $z_i$ for each user $u$ and item $i$.

Recall that at the beginning of \Cref{chap:recsys}, we discussed the differences between recommendation and regression and argued that recommendation was fundamentally different from simple linear regression on user and item features. Critically, we argued that recommender systems must model the \emph{interaction} between users and items, in order to be able to meaningfully personalize predictions for each user.

In order to capture interactions, \cite{park2009pairwise} use what is termed a \emph{bilinear} model, to estimate the compatibility between user and item features. The model parameters can then be described via a matrix $W$, and user-item compatibility can be written as
\begin{equation}
    s_{u,i} = x_u W z_i^T = \sum_{a = 1}^{|x_u|} \sum_{b = 1}^{|z_i|} x_{u,a} z_{i,b} W_{a,b},
\end{equation}
Here, unlike the linear regression model from (e.g.)~\Cref{eq:whyRecDifferent}, $W$ now encodes how user features should \emph{interact} with item features. That is, a parameter $W_{a,b}$ encodes the extent to which the $a^{\text{th}}$
%\todo{consistent way of writing this}
user feature is compatible with the $b^{\text{th}}$ item feature. So, the model can learn (for example) the extent to which users in the 35-50 demographic will respond positively to the teen romance genre.

Both $x_u$ and $z_i$ include a constant feature. These features (or rather the corresponding entries in $W$) roughly fill the role of bias terms (i.e.,~$\alpha$, $\beta_u$ and $\beta_i$ in \Cref{eq:latentSomething?}), that is, they allow the model to learn the extent to which users in a certain demographic, or movies of a certain genre, tend to yield higher or lower ratings than others. 

The model is trained so that the compatibility $s_{u,i}$ should align with observed interactions (e.g.~ratings). \cite{park2009pairwise} achieves this using a specific type of pairwise loss (i.e.,~a loss that considers two items at a time, similar to the BPR loss of \Cref{eq:bpr??}), though this is an implementation detail that is not critical to the main idea of the method.

Ultimately, the method is evaluated on two movie datasets (\emph{MovieLens} and \emph{EachMovie}). Cold users and items in these datasets are simulated, simply by withholding interactions from a subset of users and items at training time, with interactions from those users being used to evaluate the system at test time. Experiments show that when considering cold users and/or items, the method outperforms alternatives that don't make use of features.

\paragraph{Other Types of Cold-Start Recommendation}
%\todo{Capitalize paragraph-level headings?}

Obviously there is significant room to improve upon the above system, for example we might desire a model that simultaneously leverages features for cold users, and interactions for non-cold users, and learns to smoothly transition between the two groups.

We'll revisit the topic of cold-start recommendation regularly in later chapters, as well as `cool-start' settings (where we have only a few interactions per user or item), as we develop systems that operate on features from sequences, text, and images. Whether explicitly designed for cold-start or not, such methods often seek to use side information to circumvent the paucity of available interaction data.

\subsection{Addressing Cold-Start Problems with Surveys}

An alternative to relying on side information in user cold-start settings is simply to directly solicit the preferences of new users once they first interact with the recommender system.

\cite{rashid2002getting} investigated strategies for generating initial user `surveys,' in order to most quickly learn the preference dynamics of new users


Functional Matrix Factorizations for Cold-Start
Recommendation (interview based) \cite{zhou2011functional}

\section{Multisided Recommendation}

So far, our view of recommendation and personalization has consisted of maximizing some predicted utility for each user, e.g.~estimate their ratings or which items they'll interact with. Furthermore, every user has predictions made independently of each other.

Such a setting seems natural when considering contexts such as movie recommendation, but there are a variety of cases where such models are inappropriate. For example, recommendation on an online dating platform would require quite different assumptions. For instance, the problem has symmetries in the sense that the users being recommended are also receiving recommendations---as such, users must be interested in their matches, but the matches must have a reasonable chance of reciprocating. Likewise we must ensure not only that everyone receives recommendations, but that everyone \emph{is recommended to somebody}.

These types of problems are referred to as \emph{multisided} or \emph{multistakeholder} recommendations~\cite{abdollahpouri2017recommender}. Such constraints appear in many settings, many of which we will visit throughout the book. In \Cref{sec:groupRec} we will look at \emph{group} recommendation, where recommendations must simultaneously satisfy the interests of multiple users in a group. And in \Cref{chap:ads}, we will consider advertising settings, where we must consider not only user preferences but also the budgets of individual advertisers (which prevent us from recommending the most compatible ads to everyone). Finally we'll revisit the topic in depth in \Cref{sec:improving}, where we consider issues of fairness, calibration, balance, etc. For example, when recommending movies we might want reasonable coverage of different genres \cite{steck2018calibrated}, or when recommending authors, we might want our recommendations not to be too narrow in terms of gender or nationality \cite{ekstrand2018exploring}.

For now, we'll consider two specific examples of multistakeholder recommendation, namely online dating and bartering (i.e.,~recommending trading partners).

%Multistakeholder Recommendation
%\cite{abdollahpouri2017recommender}

\subsection{Online Dating}

Online dating: \cite{pizzato2010recon}

\subsection{Bartering Platforms}

\cite{rappaz2017bartering} considered the problem of generating recommendation systems for \emph{bartering} platforms, i.e.,~settings in which users exchange goods.

They study several settings in which products are exchanged, including CDs and DVDs, though most of their analysis centers around three datasets, of books (from \emph{bookmooch.com}), beers (from \emph{ratebeer.com}), and video games (from \emph{reddit.com/r/gameswap}).

On each of these websites, users have both a `wishlist' $W_u$ and a `give-away' list $G_u$, i.e.,~sets of items they wish to give or receive. Given this constraint on the problem, one might think that recommending compatible trades is as simple as identifying compatible pairs. However surprisingly the data reveals that `eligible' swapping partners are incredibly rare, and the vast majority of logged trades occur between items that were not expressly included in a user's wishlist; thus there is a need to build a system that can model likely trading partners via latent preferences. They also note that users repeatedly trade with the same partners, indicating that there is a social component to trading.

Given the above two factors, the basic model combines a standard latent factor representation with a social term. Given a user $u$, an item $i$, and a potential trading partner $v$, their compatibility is modeled as
\begin{equation}
    f(u,v,i) = \gamma_u \cdot \gamma_i + S_{u,v}.
\end{equation}
Here $\gamma_u$ and $\gamma_i$ are low-rank factors as in \Cref{eq:latentFactor?}, whereas $S_{u,v}$ is a (potentially full-rank) matrix $S \in \mathbb R^{|U|\times |U|}$;
%\todo_{is the number of users |U|?}
although $S$ potentially encodes a large number of parameters, in practice it is very sparse as the number of observed trading partners is limited.

Note that the above model captures only the interest of one user toward another's item; to model reciprocal interest, \cite{rappaz2017bartering} simply captures the average of interest in both directions (\Cref{fig:bartering}):
\begin{equation}
    f(u,i,v,j) = \frac{1}{2}(f(u,v,i) + f(v,u,j)).
\end{equation}
Other aggregation functions besides the arithmetic mean (such as the harmonic mean) can be used, though the arithmetic mean proved the most effective, indicating that a weak preference from one user can be made up for by a strong preference from another. The model also includes a temporal term encoding timepoints when certain users are particularly active and certain items are particularly popular, though we leave discussion of temporal models to \Cref{chap:temporal}.

Ultimately, the method is evaluated in terms of its ability to assign higher scores to observed interactions compared to non-observed ones (i.e.,~using a BPR-like training and evaluation scheme, as in \Cref{sec:BPR?}). The main conclusion of the experiments is that several components are important in bartering settings: reciprocal interest, a social history of trades, as well as temporal `trends' across users and items.

\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
%  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=8mm]
%\node[fsnode,on chain,draw,circle] (f1) [label=left: 1, label={[label distance=0.7mm]90:women:}] {};
\node[fsnode,on chain,draw,circle] (f1) [label=left: $u_1$] {};
\node[fsnode,on chain,draw,circle] (f2) [label=left: $u_2$] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=8mm]
%\node[ssnode,on chain,draw,circle] (s1) [label=right: 1, label={[label distance=2mm]90:men:}] {};
\node[ssnode,on chain,draw,circle] (s1) [label=right: $i_1$] {};
\node[ssnode,on chain,draw,circle] (s2) [label=right: $i_2$] {};
\end{scope}
%\todo{Hardcoded vertical spacing of men/women NOT CORRECT: label}

% \node[above=of f1] (w1) {women};
% \node[above=of s1] (m1) {men};

\draw[thick] (s1) edge[-] (f1);
% \node [pos=0.66,above=s1] {Transpose};
\draw[dotted,thick] (s1) edge[-] (f2);
\draw[dotted,thick] (s2) edge[-] (f1);
\draw[thick] (s2) edge[-] (f2);




% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}

    \caption[Basic idea behind reciprocal interest; in bartering settings a strong preference from one user compensates for a potentially weaker one from the other.
    %\todo{figure}
    ]{Basic idea behind reciprocal interest, from \cite{rappaz2017bartering}; in bartering settings a strong preference from one user compensates for a potentially weaker one from the other.
    %\todo{figure}
    \label{fig:reciprocalInterest}
    }

\end{figure}
%\todo{Make this figure}

See other examples from multisided fairness paper



\section{Music}

Thesis: Music recommendation and discovery in the long tail
%http://www.mtg.upf.edu/static/media/PhD_ocelma.pdf

%\todo{Does this belong here, or should put it in sequential recommender systems section? Or incorporate audio features into the visual recommendation chapter?}

Features derived from music

Interaction patterns such as skips and completed listens

\section{Group- and Socially-Aware Recommendation}

\subsection{Socially-Aware Recommendation}

Several approaches have sought to incorporate signals from social networks to improve recommendation. The basic idea behind doing so is that social connections will help us to circumvent \emph{sparsity} issues in interaction data. That is, even if a user has only a small number of observed interactions, we can (to some degree) leverage the interactions of their friends, whose opinions they are likely to trust.

Conceptually, the typical approach behind socially-aware recommendation is to use social connections as a form of \emph{regularizer}, which states that a user's preferences should be \emph{similar to} those of their connections in a social network. For example, given a user with few interactions, we might assume that their preferences align with the (average of) their friends; this is a possibly better assumption than the regularizer of \Cref{eq:latentreg3?}, which in practice will essentially discard user latent factors ($\gamma_u$) for users with few interactions.


%SoRec: Social Recommendation Using
%Probabilistic Matrix Factorization
An early attempt to incorporate social networks into recommender systems extended the basic framework of a latent factor model \cite{ma2008sorec}. They looked at data from \emph{Epinions}, which in addition to interaction data in the form of ratings (much like \Cref{eq:interactionMatrix?}), includes a network of `trust' and `distrust' relationships. Unlike a typical social network, these are \emph{signed} relationships, whereby a user explicitly indicates that they `trust' (1) or distrust (-1) another user. That is, in addition to our interaction matrix we have a (directed) adjacency matrix:
\begin{equation}
    A = \underbrace{\left[ \begin{tabular}{ccccc}
    1 & $\cdot$ & $\cdot$ & -1 & 1\\
    $\cdot$ & 1 & -1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 1 & 1 & -1 & $\cdot$\\
    1 & $\cdot$ & 1 & $\cdot$ & 1\\
    -1 & -1 & $\cdot$ & 1 & 1\\
    \end{tabular} \right]}_{\text{users}} \left. \clap{\begin{tabular}{c} \\ \\ \\ \\ \\ \end{tabular}} \right\rbrace \text{\normalsize users}.
    %\todo{In other equations like this, make sure the underbrace text is the right size}
    \label{eq:trustMatrix}
\end{equation}
Ultimately though, the distrust relationships are not used in \cite{ma2008sorec}, as it is argued that the semantics of `distrust' are somewhat more complex than (e.g.)~users having different preference dimensions.

Thus, give a rating matrix $R$, and an adjacency matrix $A$, we want to predict ratings in $R$ in such a way that $A$ informs us about each user's likely latent preferences. The basic idea is to make use of a \emph{shared parameter} $\gamma_u$ for each user. For rating data, $\gamma_u$ is no different from a user latent factor in a matrix factorization model, i.e.,~it is combined with an item latent factor and used to predict the rating, in this case via a sigmoid function:\footnote{The specific choice of the sigmoid function is an implementation detail, and ratings are scaled to be in the range $[0,1]$ to accommodate this choice.}
\begin{equation}
    r_{u,i} = \sigma(\gamma_u \cdot \gamma_i).
\end{equation}
Next, the parameter $\gamma_u$ is \emph{re-used} to predict trust relationships in $A$:
\begin{equation}
    a_{u,v} = \sigma(\gamma_u \cdot \gamma'_v).
\end{equation}
The original paper allows $A$ to be a weighted matrix, indicating varying degrees of social trust, but for simplicity we assume here that $A$ contains only trust (1) and not-trust (0) values.

Note that while $\gamma_u$ is a shared parameter, $\gamma'v$ is not; since the matrix $A$ is directed, $\gamma_u$ can be thought of as explaining why $u$ trusts others, whereas $\gamma'_v$ explains why $v$ \emph{is trusted by} others.
%\todo{confusing, does the above help?}

In practice we are usually not interested in predicting entries $a_{u,v}$; rather, the trust relationships are additional data that should help us to calibrate $\gamma_u$ more efficiently. For example, if a user has few observed ratings, $\gamma_u$ can still be estimated via their trust relationships, giving us an initial estimate of their rating behavior in cold- (or cool-)start settings.
%\todo{Did I already make this argument above?}

The overall objective then takes the form
\begin{equation}
    \underbrace{\sum_{(u,i) \in R} (r_{u,i} - \sigma(\gamma_u \cdot \gamma-i)^2}_{\text{rating prediction error}} + \lambda^{(\text{trust})}\underbrace{\sum_{(u,v) \in A} (a_{u,v} - \sigma(\gamma_u \cdot \gamma'_v)^2}_{\text{trust prediction error}} + \lambda\| \gamma \|_2^2,
\end{equation}
where $\lambda^{(\text{trust})}$ trades-off the importance of predicting the trust network. Ultimately, experiments in \cite{ma2008sorec} show that the trust network helps to predict ratings more accurately than matrix factorization alone. Of course, it should be noted that trust relationships on \emph{epinions} 
%\todo{capitalization, italicization?}
are very closely tied to opinion dimensions, presumably moreso than in other social networks.

Note that the above is essentially a more complex form of cold-start (or `cool-start')
%\todo{make sure not to say the above too many times}
recommendation (as we saw in \Cref{sec:coldstartRec}), in the sense that we are leveraging a form of side-information (social connections) to make up for a paucity of interaction data. In the case of a user who has never rated an item (but has social connections), the system can still reasonably estimate $\gamma_u$ from the preference dimensions of $u$'s friends.


\subsection{Social Bayesian Personalized Ranking}

%Leveraging Social Connections to Improve Personalized
%Ranking for Collaborative Filtering

Above we showed how matrix factorization frameworks can be extended to incorporate signals from social networks. The intuition behind the idea simply stated that users are likely to have preference dimensions aligned with other users who they `trust.'

Next, we'll see how this idea can be adapted to predict interactions (rather than ratings), by incorporating social connections into the Bayesian Personalized Ranking framework from \Cref{sec:BPR}.

Conceptually, using social links to predict
%Personalized ranking 
interactions relies on
a possibly different assumption than we made above. Whereas our previous intuition above was based on some notion of trust, here we are simply assuming that a user is more likely to interact with items (e.g.~to watch movies or read books) if their friends have previously interacted with them.

\cite{zhao2014leveraging} attempted to adapt the assumptions made by Bayesian Personalized Ranking (BPR) to datasets involving social connections. Recall that BPR makes the assumption that a user's compatibility with items they've interacted with ($x_{u,i}$) should be higher than their compatibility with items they haven't interacted with ($x_{u,j}$), which is captured via a sigmoid function:
\begin{equation}
    x_{u,i} \geq x_{u,j} \quad \rightarrow \quad \sigma(x_{u,i} - x_{u,j}) \text{\ should be maximized}
\end{equation}
(see \Cref{eq:BRPOpt1?}). To adapt this to settings involving a social network, \cite{zhao2014leveraging} assumes a third type of feedback: for a user $u$, in adition to \emph{positive} feedback $i$, and \emph{negative} feedback $j$ (as with BPR), we also have \emph{social} feedback $k$, which consists of items consumed by $u$'s connections in a social network.

\cite{zhao2014leveraging} tests two assumptions about how this social feedback should be incorporated. The first states that social interactions are weaker than positive interactions, but still stronger than negative interactions, essentially stating that users are somewhat more likely to interact with items their friends have interacted with:
\begin{equation}
    \underbrace{x_{u,i}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,k}}_{\mathclap{\text{social}}}; \quad \underbrace{x_{u,k}}_{\mathclap{\text{social}}} \geq \underbrace{x_{u,j}}_{\mathclap{\text{negative}}}.
    \label{eq:sbpr1}
\end{equation}
An alternate hypothesis states the opposite: if our friends have interacted with an item but we haven't, this might instead be a signal that we know about the item, but deliberately chose not to interact wit it; in this instance we drop the second assumption from \Cref{eq:sbpr1} and replace it with a weaker assumption:
\begin{equation}
        \underbrace{x_{u,i}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,k}}_{\mathclap{\text{social}}}; \quad \underbrace{x_{u,i}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,j}}_{\mathclap{\text{negative}}}.
    \label{eq:sbpr2}
\end{equation}
Note that neither of these assumptions is `better' than the other; rather they are simply hypotheses that must be tested by determining which best fits real datasets.

To train the model, a BPR-like objective (\Cref{eq:bprOpt1?}) is used, but which now involves two terms. E.g.~using the assumption from \Cref{eq:sbpr1}:
\begin{equation}
    \sum_{(u,i,k) \in \mathcal T} \ln \sigma(x_{u,i} - x_{u,k}) + \sum_{(u,k,j) \in \mathcal T} \ln \sigma(x_{u,k} - x_{u,j}) + \| \gamma \|_2^2,
\end{equation}
where $\mathcal T$ is a training set consisting of positive, negative, and social feedback ($i$, $j$, and $k$) for each user $u$.

Ultimately, \cite{zhao2014leveraging} shows that both models outperform alternatives that don't leverage social connections. Several datasets besides \emph{epinions}
%\todo{italicize, capitalize?}
are used, including data from \emph{Ciao}, \emph{Delicious}, and \emph{LibraryThing} (product reviews, social bookmarks, and books).
Overall, the assumption from \Cref{eq:sbpr2} slightly outperforms that of \Cref{eq:sbpr1} on all datasets.


\subsection{Group-Aware Recommendation}

Somewhat related to the topic of social recommendation is the idea of \emph{group} recommendation, where recommendations should be made collectively to a group of users, rather than to an individual. However, unlike social recommendation, where the goal is normally to use social signals to improve the accuracy of recommendations each individual receives, the goal of group-aware recommendation is generally to produce recommendations that mutually satisfy a group of individuals.

% PolyLens: A Recommender System for
% Groups of Users (basically just has an interface for group recommendations)

Early work on this topic includes \emph{PolyLens}
%\todo{italics}
\cite{o2001polylens}, though this work was mostly concerned with designing user interfaces for the purpose of group recommendation, rather than using group data to improve recommendation performance. Although focused on interface-building, the work does show the utility of group-based recommenders, which could help users to find items that are mutually compatible with the interests of all group members.
%\todo{is that what it actually says?}

Later, \cite{amer2009group} attempted to formalize the notion of group recommendation, by defining useful objectives that define how compatible a set of items is with a group. Given a pre-defined compatibility function $f(u,i)$ (e.g.~the output of a latent factor model), a simple attempt to define group compatibilty between a group of users $\mathcal G$ and an item $i$ might consist of computing 
\begin{equation}
    \text{Average compatibilty of $i$:} \quad \mathit{rel}(\mathcal G, i) = \frac{1}{|\mathcal G|} \sum_{u \in \mathcal G} f(u,i).
    \label{eq:groupRel1}
\end{equation}
Alternately, we could define the compatibility as that of the \emph{least} compatible user in the group, known as \emph{least misery}:
\begin{equation}
\text{Least misery:} \quad \mathit{rel}(\mathcal G, i) = \min_{u \in \mathcal G} f(u,i).
    %2) Least-Misery: rel(G, i) = Min(relevance(u, i))
    \label{eq:groupRel2}
\end{equation}
The latter is preferable in settings where users have constraints, e.g.~to avoid recommending a steakhouse to a group of users, some of whom are vegetarian \cite{amer2009group}.

\cite{amer2009group} argues that in addition to maximizing relevance (or minimizing misery), it is also important that the group has some degree of \emph{consensus} about the quality of an item. That is, users in a group should not drastically disagree about $f(u,i)$, separately from their actual scores. Two disagreement functions proposed are average pairwise disagreement:
\begin{equation}
    \mathit{dis}(\mathcal G, i) = \frac{2}{|\mathcal G|(|\mathcal G| - 1)} \sum_{(u,v) \in \mathcal G, u \neq v} (| f(u,i) - f(v,i) |),
    \label{eq:groupDis1}
\end{equation}
and disagreement variance:
\begin{equation}
    \mathit{dis}(\mathcal G, i) = \frac{1}{|\mathcal G|} \sum_{u \in \mathcal G} \Bigl(f(u,i) - \underbrace{\frac{1}{|G|} \sum_{v \in \mathcal G} f(v,i)}_{\mathclap{\text{average compatibility in $\mathcal{G}$}}}\Bigr)^2.
    \label{eq:groupDis2}
\end{equation}

Finally, \cite{amer2009group} argues that a group \emph{consensus function} should be a combination of both of these factors, i.e.,~a relevance function (\Cref{eq:groupRel1} or \Cref{eq:groupRel2}):
%\todo{maybe use lighter-weight equation references within parentheses?}
\begin{equation}
    \mathcal{F}(\mathcal G, i) = w_1 \times \mathit{rel}(\mathcal G, i) + w_2 \times (1 - \mathit{dis}(\mathcal G, i)),
\end{equation}
where $w_1$ and $w_2$ trade-off the relative importance of the two terms. \emph{Group recommendation} then consists of finding suitable items (e.g.~movies that no user in the group has seen) that maximize a (tuned) group consensus function.

Beyond defining these notions of group consensus, \cite{amer2009group} show how to efficiently select items that maximize the above criteria (note the large number of comparisons involved when performing optimization na\"ively).
%
They also demonstrate experimentally (via a \emph{Mechanical Turk}-based user study of movie recommendations) that both \emph{relevance} and \emph{disagreement} are simultaneously important to achieve satisfaction within a group.

%  GBPR: Group preference based bayesian
% personalized ranking for one-class collaborative filtering \cite{pan2013gbpr}



\section{Other Contextual Features in Recommendation}

\subsection{Geography}
%\todo{And something else...?}

\subsection{Price}

In spite of the obvious impact price has on customer decisions, there is surprisingly little work that seeks to incorporate price features into personalized predictive models.

Notions such as \emph{price sensitivity}, and \emph{price elasticity} (defined as the change in purchase quantity given a change in price) are well understood in economics and marketing \cite{case2007principles}. Understanding such factors can help to guide custom marketing and promotion strategies \cite{zhang2004customizing,zhang2009effectiveness}.

However they are less well understood in terms of their effectiveness in a \emph{predictive} setting, i.e.,~in terms of how price should be used to understand and forecast user actions (or to make recommendations).

Part of the reason that price has received relatively little attention (at least in academic literature) is presumably the lack of useful available data; even when price is observed, it is confounded by numerous other factors, such as brand or manifest aspects of a product; moreover even when price data is available, on rarely has \emph{historical} data on price that allows for measurement of the impact of a \emph{change} in price.

\cite{wan2017modeling} studied price in the context of grocery recommendation. Their research was mostly based on real transaction data from a physical grocery store (in Seattle), though was also validated based public data from \emph{dunnhumby}. Both datasets contain price measurements, and critically measurements of price \emph{variation} over time. As such the main questions center around the extent to which purchase decisions are affected by changes in price.

In the context of grocery shopping, potential questions include:
\begin{itemize}
    \item Will a reduction in price cause a user to buy a \emph{category} of product they otherwise wouldn't have (e.g.~would they buy milk at a discount if it wasn't on their shopping list)?
    \item Will a reduction in price cause users to buy a \emph{specific item} that they otherwise wouldn't have (e.g.~would they buy a different brand of milk because of a discount)?
    \item Will a reduction in price cause users to buy a \emph{larger quantity} of an item than they otherwise would have?
\end{itemize}
To study these questions, prediction is broken down into three subsequent choices:
\begin{multline}
    p_u(\text{buy $q$ units of an item $i$ from category $c$}) = \\
    \begin{tabular}{l}
      $\hphantom{\times} p^{(\text{category})}_u(\text{buy a product from category $c$})$ \\
      $\times p^{(\text{item})}_u(\text{buy product $i$\ } | \text{\ buying from category $c$})$\\
      $\times p^{(\text{quantity})}_u(\text{buy $q$ units\ } | \text{\ buying item $i$})$.
    \end{tabular}
\end{multline}

Each of these three prediction tasks (category, item, and quantity prediction) is based on a predictor $f(u,c,t)$, $f(u,i,t)$, and $f(u,q,t|i)$; the underlying method behind each is a latent-factor model, as in \Cref{eq:latentSomethnig}, including additional features associated with the time (e.g.~what day of the week the trip occurs on). Each is passed through a different activation function, for example quantity prediction is modeled via a \emph{Poisson} function:
\begin{equation}
    p(\text{quantity} = q | \text{buying item $i$}) = \frac{f(u,q,t|i)^{q-1}\exp(-f(u,q,t|i))}{(q-1)!}.
\end{equation}

Next, the change in purchase probability due to price is captured (for each of the three models) using a simple feature encoding the price at a particular point in time. Specifically (e.g.~for quantity)
\begin{equation}
    f'(u,q,t|i) = f(u,q,t|i) + \beta_{u,q} \log P_i(t),
\end{equation}
where $P_i(t)$ is the price of the item at time $t$. $\beta_{u,q}$ is then a bias term which encodes the price sensitivity, i.e., extent to which a particular user $u$, when purchasing $q$ units of an item, will react to changes in price; a negative value of $\beta_{u,q}$ would indicate that a user is less likely to purchase a particular quantity given a price increase. All parameters are learned by training on purchase data, using a BPR-like training scheme.

\emph{Price-elasticity} now reflects how much preference changes given a change in price, e.g.~for a particular item $i$:
\begin{equation}
    e_{i,u}(t) = \frac{df'(u,i,t)}{f(u,i,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx (1 - f'(u,i,t))\beta_{u,i}(t).
\end{equation}
%\todo_{Redo this, fix notation, \beta is the output of some model, not a linear coefficient}
A related notion, \emph{cross-elasticity}, measures the extent to which a change in $i$'s price will change a user's compatibility toward (e.g.~probability of purchasing) a \emph{different} product $j$:
\begin{equation}
    e_{i,j,u}(t) = \frac{df'(u,j,t)}{f(u,j,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx -f(u,i,t)\beta_{u,i}(t).
\end{equation}
%\todo_{CHECK!}
% Compare to equality in https://mengtingwan.github.io/paper/www17_mwan.pdf
% (after eq 15)
%a(\sum_(j neq a) j)b_i  =   \sum_{j neq a} ja b_i

Note that the above equations (price-elasticity and cross-elasticity) are \emph{measurements} after the model has been trained.
%\todo_{Crappy sentence}
The main finding of the model is that price-elasticity applies mostly to product choice, but not to category choice or quantity.
%\todo_{a few other findings}

%\todo{Write f's as PROBABILITIES}


\section{Problems}

\begin{problemlist}
 \item Fit a factorization machine with various features
 \item Incorporate price into some dataset (amazon?) in a few ways
\end{problemlist}






\chapter{Temporal and Sequential Models}
\label{chap:temporalSequential}

\section{Introduction to Regression with Time Series}
%\todo{This would be better in regression chapter, but probably only do that if regression + classification become two separate chapters}

Before investigating more sophisticated models of temporal and sequential data, it is instructive to consider how much progress can be made with techniques we have already developed.

To do so, we'll consider developing predictors that estimate the next value in a sequence (or the next several values). In the simplest case, we are given a sequence of observations $y = \lbrace y_1 \ldots y_n \rbrace$, from which one would like to predict the next value ($y_{n+1}$). For example, one might want to estimate website traffic on the basis of historical traffic patterns.

To solve such a problem using a regression approach (or a classification approach for a binary outcome), we can imagine constructing features based on the previous observations ($\lbrace y_1 \ldots y_n \rbrace$) in order to predict the next one. Presumably, once we observe the true value of $y_{n+1}$, we would like to predict $y_{n+2}$, and so forth. Note that this blurs the line somewhat between `features' and `labels,' since the label for one prediction becomes a feature for the next.

This procedure is known as \emph{auto-regression} (referring to the fact that we are regressing on the same data that was used for prediction). As usual, we are typically interested in defining our regressor (or classifier) such that it minimizes some error between the predictions and the true values. That is, we want to define a predictor $f(y_1 \ldots y_n)$ that estimates the next value ($y_{n+1}$) in the sequence so as to minimize (e.g.)~a Mean Squared Error:
%\todo{Capitalization?}
\begin{equation}
    \frac{1}{n} \sum_{i = 1}^{n} (f(y_1 \ldots y_i) - y_{i+1})^2.
\end{equation}

Trivially, we might imagine several na\"ive techniques for estimating the next value in a sequence, e.g.~we could predict the next value as a weighted sum of previous values:
\begin{eqnarray}
    \text{moving average:}\quad f(y_1 \ldots y_n) & = & \frac{1}{K} \sum_{k = 0}^{K - 1} y_{n - k} \label{eq:movingAvarege}\\
    \text{weighted moving average:}\quad f(y_1 \ldots y_n) & = & \frac{\sum_{k = 0}^{K - 1} y_{n - k}}{\sum_{k=1}^K k}. \label{eq:weightedMovingAverage}
\end{eqnarray}
Although simple, one can imagine how these averages could potentially be better predictors than always predicting the next value to be equal to the previous (for example, see \Cref{ex:autoRegression1}). These types of trivial predictors can also be used to plot trends in noisy data (\Cref{fig:slidingWindows}); averages over larger intervals (i.e.,~larger values of $K$) will produce smoother summaries of the data. Such averages can be efficiently computed for successive values via a dynamic programming solution; code to generate the plots in \Cref{fig:slidingWindows} is presented below:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
xSum = sum(x[:wSize]) # Given data x and y to plot, and a window size wSize
ySum = sum(y[:wSize]) # Sum of first wSize values
xSliding = []
ySliding = []

for i in range(wSize,len(x)-1):
    xSum += x[i] - x[i-wSize] # Strip off oldest value and add newest one
    ySum += y[i] - y[i-wSize]
    xSliding.append(xSum / wSize)
    ySliding.append(ySum / wSize)
\end{lstlisting}
\end{minipage}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/slidingWindows.pdf}
\caption{Moving-average plots of $\simeq$1 week of \emph{Goodreads}
%\todo{italicize?}
Fantasy novel ratings. Although not particularly effective as predictors, moving averages can be used to plot data so as to summarize overall trends. \label{fig:slidingWindows}}
\end{SCfigure}
%\todo{Probably add code here}

The two trivial strategies above are heuristics for predicting the next value, capturing the intuition that it should be similar to recently observed values (\Cref{eq:movingAvarege}), and possibly that more recent values should be more predictive than less recent ones (\Cref{eq:weightedMovingAverage}). However, a better strategy might be to \emph{learn} which of the recent values are the most predictive, i.e.,
\begin{equation}
    f(y_i \ldots y_n) = \sum_{k=0}^{K - 1} \theta_k y_{n - k}.
    \label{eq:autoRegression}
\end{equation}
The values $\theta_k$ now determine which of the previous $k$ values are predictive. For example, in periodic data (e.g.~network traffic, seasonal purchases), the most predictive values may include recent observations, observations from 7 days ago, observations from the same day in the previous month, etc.

%\todo{Include example from some actual dataset? Could use e.g. Bay Area Bike Share, though dataset seems not to be public}

For example, training a simple autoregressive model on a dataset of hourly measurements of bike rentals in the Bay Area\footnote{Each observation $y_h$ is a measurement of how many trips were taken during hour $h$.} yields the following model:
%\todo{Fix equation alignment}
\begin{equation}
%0.47105067, -0.28432565, 0.10555845,  0.01445134, -0.02131945, 0.17501989,  0.539696351
    y_{n} = \begin{tabular}{l}
        0.471 $\times y_{n-1}$\\
      - 0.284 $\times y_{n-2}$\\
      + 0.106 $\times y_{n-3}$\\
      + 0.014 $\times y_{n-4}$\\
      - 0.021 $\times y_{n-5}$\\
      + 0.175 $\times y_{n-24}$\\
      + 0.540 $\times y_{n-24\times 7}$
      \end{tabular}
\end{equation}
Here we see that the two most predictive observations are those from the previous hour ($y_{n-1}$) and from exactly one week ago ($y_{n - 24\times 7}$). Observe that we didn't include \emph{every} previous observation up to $24\times 7$ hours ago as a feature, rather we only included those previous observations that we expected to be predictive.

Note that although the solution in \Cref{eq:autoRegression} uses \emph{only} previous values in the sequence, one can of course include other features associated with the current timepoint or previous values, just like in a normal regression model.

%\todo{Check indentation level for all lstlistings}

Although a simple approach to regression and classification of time series data, the basic idea behind auto-regression (i.e.,~to use previous observations to predict future values in a sequence) will reappear in many of the more complex models we develop, especially as we develop sequential models in \Cref{sec:sequential?}.

\section{Temporal Dynamics in Recommender Systems}
\label{sec:temporalRec}

Several attempts have been made to improve recommendations by incorporating temporal dynamics. There are countless reasons why preferences, purchases, or interactions may change over time, or more simply why knowing the current timestamp may help us to more accurately predict the next interaction.
%\todo{Finish, include items below}
% Changes in the interface
% • People give higher ratings to older movies (or, people
% who watch older movies are a biased sample)
% • The community’s preferences gradually change over time
% • My girlfriend starts using my Netflix account one day
% • I binge watch all 144 episodes of buffy one week and
% then revert to my normal behavior
% • I become a “connoisseur” of a certain type of movie
% • Anchoring, public perception, seasonal effects, etc.

Methods for temporal recommendation broadly fall into two classes. The first make use of the actual \emph{timestamps} of events. Each interaction $(u,i)$ is augmented with a timestamp $(u,i,t)$, and the goal is to understand how ratings $r_{u,i,t}$ vary over time. That is, our goal is to extend models such as the one in \Cref{sec:latentFactor} so that parameters vary as a function of time, e.g.
\begin{equation}
    r_{u,i,t} = \alpha(t) + \beta_u(t) + \beta_i(t) + \gamma_u(t) \cdot \gamma_i(t).
    \label{eq:latentFactorTime}
\end{equation}
Modeling temporal dynamics in this way is effective at capturing long-term shifts of community preferences over time; short-term or `bursty' dynamics, such as purchase patterns being affected by external events, or periodic events, such as purchases being higher at a particular time of day, day of week, or season.

A second class of methods discards the specific timestamps, but preserves only the \emph{sequence} (or order) of events. Thus the goal is generally to predict the next action as a function of the previous one, i.e.,
\begin{equation}
    p(\text{user $u$ interacts with item $i$} | \text{they previously interacted with item $j$}).
\end{equation}
This type of model makes the assumption that the important temporal information is captured by the \emph{context} provided by the most recent event. This is useful in highly-contextual settings, such as predicting the next song a user will listen to, or other items they will place in their basket, etc. In such settings knowing the most recent actions (or most recent few actions) is more informative than knowing the specific timestamp.

These two classes of model are quite orthogonal, both in terms of the settings where they are effective, as well as the techniques involved.
%\todo{Finish this argument}

%\todo{Borrowed paragraphs}
%Some early works that deal with temporal dynamics do so in
%terms of concept drift 
%[23, 37, 39]. 
%Such models are able to account
%for short-term temporal effects (‘noise’), and long-term changes
%in user behavior (‘drift’), for example due to the presence of new
%products within a community.

%Sophisticated models of such temporal dynamics proved critical
%in obtaining state-of-the-art performance on the Netflix challenge
%[3], most famously in [20]. As discussed in [20], few previous
%works had dealt with temporal dynamics, other than a few notable
%exceptions [2, 10, 35]. Around the same time, ‘adaptive neighborhood’ models were proposed [24], that address the problem of
%iteratively training recommender systems whose parameters ought
%to change over time.

% Better performance may be obtained by modeling large-scale
% global changes at the level of entire communities [41], or by developing separate models for short term changes (e.g. due to external
% events), and long-term trends [40].
% Other works that study temporal dynamics at the level of products and communities include [29], where the authors studied how
% existing ratings within communities may influence new users, and
% how community dynamics evolve; and [13], who studied how users
% are influenced by previous ratings of the same product


\subsection[Case Study: Temporal Recommendation and the Netflix Prize]
{Case Study: Temporal Recommendation and the Netflix Prize \cite{koren2009collaborative}}
\label{sec:caseStudyNetflixTemporal}
%\todo{Should include citations in section headings?}

Careful modeling of temporal dynamics was one of the key features characterizing the strongest solutions to the Netflix Prize. Several of the ideas that proved effective for modeling temporal dynamics on Netflix specifically are covered in \cite{koren2009collaborative} (``Collaborative Filtering with Temporal Dynamics''), which we summarize here.

As a motivating example, consider the two plots shown in \Cref{fig:netflixTemporal}.\footnote{These two plots are based on similar figures from \cite{koren2009collaborative}; the purported trends are somewhat more pronounced in the original paper, possible due to differences in sampling schemes.} At top, we see ratings over time averaged across weekly bins. We see, several points where ratings appear to increase suddenly, followed by plateaus of relatively stable ratings; for example at around week 210, average ratings appear to jump from around 3.5 stars to 3.6 stars.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/netflix1.pdf}\\
    \includegraphics[scale=0.75]{fig/netflix2.pdf}
    \caption{Temporal dynamics on Netflix. The top plot shows ratings averaged across each week over the lifetime of the dataset; the bottom plot shows how ratings change for newly-introduced movies, showing that ratings gradually increase during the first few weeks the movie is on Netflix. These plots reveal a combination of sudden and gradual trends in movie ratings over time.     \label{fig:netflixTemporal}}
\end{figure}

Such long-term, population-level changes could owe to several explanations. Changes in rating patterns could be due to changing user base, or from certain movies being added to Netflix; such changes could even be due to world events exogenous to Netflix. Or, the change could owe to a factor as simple as a change in Netflix's User Interface (UI), causing users to rate movies differently.

The bottom plot in \Cref{fig:netflixTemporal} shows another temporal trend, demonstrating that individual movies recieve higher ratings the longer they have been available on Netflix. Again, such trends could be due to a variety of factors: users may be favorably biased toward older movies (e.g.~by nostalgia); or again it could be a function of the UI: a user who specifically sought out an older movie may view it more favorably than a user who had been recommended the same movie from the front page.

Whatever the underlying cause of these trends, they account for significant variability in the observed ratings. And as such, we should model these dynamics to predict ratings more accurately. Na\"ively, one might simply discard (e.g.)~older data that does not correspond to the predominant rating trends. But, a more effective model would attempt to account for the differences between newer and older data.

To model the kinds of long term trends captured in \Cref{fig:netflixTemporal}, \cite{koren2009collaborative} first focuses only on temporally evolving bias terms, i.e.,
\begin{equation}
    b_{u,i}(t) = \alpha + \beta_u(t) + \beta_i(t).
\end{equation}
Starting with item biases, one can capture long-term, gradual variation simply by having different bias terms for different periods, i.e.,
\begin{equation}
    \beta_i(t) = \beta_i + \beta_{i,\mathrm{Bin}(t)}.
    \label{eq:bins30}
\end{equation}
In \cite{koren2009collaborative} the authors suggest $\sim 30$ bins corresponding to about 10 weeks each in the case of the Netflix data.

This basic idea of separating bias terms into bins could likewise be applied to capture periodic trends, much like we encoded periodic terms in \Cref{sec:temporalFeatures}. Here the bias term would again take the form of several bins:
\begin{equation}
    \beta_i(t) = \beta_i + \beta_{i,\mathrm{Bin}(t)} + \beta_{i,\mathrm{period}(t)},
\end{equation}
where $\mathrm{period}(t)$ could represent periodic effects at the level of different days of the week, or months of the year (etc.).

The above ideas are effective for modeling long-term and periodic dynamics, but are quite expensive: e.g.~the model in \Cref{eq:bins30} requires an additional 30 parameters \emph{per item}; this is affordable for items on Netflix since the average item has over 5,000 ratings (100 million ratings of 17,770 titles). However this would likely not be possible for users, as the average user has only around 200 ratings.
% 480k users
Thus one needs a way of parameterizing user temporal dynamics that is considerably cheaper (i.e.,~involves fewer parameters).

A solution suggested in \cite{koren2009collaborative} is to use an `expressive deviation' term for each user:
\begin{equation}
    \mathrm{dev}_u(t) = \underbrace{\mathrm{sign}(t - t_u)}_{\mathclap{\text{before $(-1)$ or after $(1)$ the mean date}}} \cdot |t - t_u |^x.
\end{equation}
The term $t_u$ represents the mean date amongst a particular user $u$'s ratings, so that the term $(t - t_u)$ represents whether a particular point in time $t$ is before or after the midpoint of the user's rating lifetime.
%
The expressive deviation term is depicted in \Cref{fig:expressiveDev}; the exponent of 0.4 was found to work well on Netflix data. 

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/expressiveDev.pdf}
\caption{Expressive deviation term from \protect\cite{koren2009collaborative}. 
\label{fig:expressiveDev}}
\end{SCfigure}

While the expressive deviation term is effective for users who have few interactions, for users who have more it can be useful to fit a more complex model. To do so, a spline function can be used to model gradual shifts in user biases. A spline function smoothly interpolates between a series of control points via the following function:
\begin{equation}
    \beta_u(t) = \beta_u + \frac{\sum_{l = 1}^{k_u} e^{\gamma|t - t_l^u|}b_{t_l}^u}{\sum_{l = 1}^{k_u}e^{-\gamma|t - t_l^u|}}.
\end{equation}
Here $k_u$ is the number of control points for user $u$ (which grows with the number of ratings that user has entered); $t^u$ is a series of uniformly spaced time-points for each user; and $b_{t_l}^u$ is the bias associated with each control point. This type of interpolation is depicted in \Cref{fig:splines}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/splines.pdf}
\caption{Spline interpolation of temporally evolving user bias. \label{fig:splines}}
\end{SCfigure}

The above term is a reasonably flexible way to capture gradual drift in user preferences, though still can't handle sudden changes. \cite{?} addresses this with a `per day' user bias $\beta_{u,t}$, which can be useful for particular days in which users have a lot of activity. Note that while such a bias is unlikely to be helpful when predicting future events, by modeling outliers in this way, the model can


It should be noted though that the goal of this model was to be effictive on Netflix, rather than to capture temporal dynamics in general settings.
%\todo{Pasted this text in here, see if it fits}


\section{Personalized Markov Chains}
\label{sec:markov}

The temporal models we saw in \Cref{sec:temporalRec} directly modeled (or extracted features from) the \emph{timestamps} associated with each interaction. We showed how features extracted from timestamps could include factors like seasonality, the day of the week, or nostalgia effects (etc.)~on the \emph{Netflix} dataset, and later we'll revisit such models in the context of the temporal dynamics of fashion (\Cref{sec:fashionTemporal}).

However in many settings, the best predictor of what a user will do next is simply what they did \emph{last}. For example, if you click on a winter coat, then I should recommend you other winter coats, regardless of whether those items are currently in-season.

Even trivial models such as the item-to-item recommenders which we saw in \Cref{sec:neighborhood} implicitly made this assumption. For example, the recommendations made in \Cref{fig:alsoviewed} are made based purely on the context of what is currently being viewed; the user's historical interactions---or their preferences---are not considered.

\paragraph{Markov Chains}
The assumption described above---that the next action is conditionally independent\footnote{Two variables $a$ and $b$ are said to be \emph{conditionally independent} given a third variable $c$ if $p(a,b|c) = p(a|c)p(b|c)$. Essentially this $c$ `explains' any dependence $a$ and $b$. In the case of Markov Chains, the most recent event is sufficient to explain the next action's dependence on the history.
%\todo{Quick writing}
} of the interaction history given the previous action---describes exactly the setting of a \emph{Markov Chain}. 
%\todo{strictly speaking, I suppose a discrete time markov chain}
%\todo{could have a box about conditional independence}
%\todo{Clean up probability notation}
Formally, given a sequence of interactions (among a discrete set of items $i \in \mathcal I$) $I^{(1)} \ldots I^{(t)}$, a Markov Chain
%\todo{Do I use this notation for the item set?}
assumes that the probability of the next interaction given the history can be written purely in terms of the previous interaction:
\begin{equation}
    P(I^{(t+1)} = i | I^{(t)} \ldots I^{(1)}) = P(I^{(t+1)} = i | I^{(t)})
    \label{eq:markov1}
\end{equation}

\emph{Personalized} Markov Chains generalize \Cref{eq:markov1} by allowing the probability of the next item to depend on both the previous item and the identity of the user $u$. That is, for a given user we have:
\begin{equation}
    P(I_u{(t+1)} = i | I_u{(t)} \ldots I_u^{1}) = P(I_u^{(t+1)} = i | I^{(t)}).
\end{equation}

What this means in practice is just that when predicting a user's next action, our prediction should be a function of their previous action as well as their preference dimensions. 
Most of the time we can ignore the formalism of Markov Chains and more simply state that we are trying to fit a function of the form
\begin{equation}
    f(\overbrace{u,i}^{\clap{\text{score associated with the next interaction}}}|\underbrace{I_u^{(t-1)}}_{\clap{\text{given the user's previous interaction}}}).
    \label{eq:fMarkov}
\end{equation}
That is, whereas our previous models took as inputs a user and an item (i.e.,~$f(u,i)$), or a user, item, and timestamp ($f(u,i,t)$), we now wish to model a user, an item, and the user's previous interaction. We might fit this function via a rating estimation framework (as in \Cref{sec:modelbased}), or a personalized ranking framework (as in \Cref{sec:bpr}), etc.

The key challenge in fitting models of the form in \Cref{eq:fMarkov} is that techniques like matrix factorization can no longer be straightforwardly applied. Whereas we previously modeled user/item interactions by factorizing a $U\times I$ matrix we must now factorize a $U \times I \times I$ \emph{tensor}.

We'll explore this idea by investigating specific
%\todo{Wrong word}
implementations
from case studies in the following section.

\section{Case Studies: Markov-Chain Models for Personalized Recommendation}

\subsection{Factorized Personalized Markov Chains}
\label{sec:FPMC}

SPMC: Socially-Aware Personalized Markov Chains for Sparse Sequential
Recommendation

%\todo{Mention that FPMC is handled by Factorization Machines}

%Cover: Factorized Personalized Markov Chains
An early paper to use Markov Chains for personalized recommendation was \emph{Factorizing Personalized Markov Chains for Next-Basket Recommendation} \cite{rendle2010factorizing}. 
The paper was specifically concerned with predicting what 
Factorized Personalized Markov Chains (FPMC for short) predict what
items a user will purchase next, based on the items in their previous basket; customer basket data from \emph{Rossmann} (a German drugstore) was used to train and evaluate the model.

The basic premise of FPMC is that the contents of the previous basket should help to predict the contents of the next one, but also that the basket contents should be personalized to the user. This is achieved by fitting a function of the form
\begin{equation}
    f(u,i|j)
    \label{eq:fuij}
\end{equation}
where $u$ is a user, $i$ is a potential item to be recommended, and $j$ is an item from the user's previous basket.\footnote{Although the original paper uses basket data, baskets are mostly a complication needed to handle their specific dataset. It is more straightforward to present the work by simply considering item sequences (which is how the method is often adopted by other papers). Where we write a term for a previous item, in \cite{rendle2010factorizing} that expression would usually be replaced by a sum over items in the previous basket}. 

The paper discusses the difficulty of modeling sparse interactions of the form in \Cref{eq:fuij}, and explains how this might be addressed with tensor decomposition. The decomposition used is essentially a generalization of the matrix factorization schemes we saw in \Cref{sec:modelbased}, where $f(u,i|j)$ decomposes into a series of pairwise factors:
\begin{equation}
    f(u,i|j) = \gamma^{(ui)}_u \cdot \gamma^{(iu)}_i + \gamma^{(ij)}_i \cdot \gamma^{(ji)}_j + \gamma^{(uj)}_u \cdot \gamma^{(ju)}_j.
\end{equation}
The three terms above denote the user's compatibility with the next item ($\gamma^{(ui)}_u \cdot \gamma^{(iu)}_i$), the next item's compatibility with the previous item ($\gamma^{(ij)}_i \cdot \gamma_j$), and the user's compatibility with the previous item ($\gamma^{(uj)}_u \cdot \gamma^{(ju)}_j$). In practice, the latter expression cancels out when optimizing the model (which is perhaps intuitive as the expression doesn't include the candidate item $i$), so that the factorization can be rewritten
\begin{equation}
    f(u,i|j) = \overbrace{\gamma^{(ui)}_u \cdot \gamma^{(iu)}_i}^{\clap{\text{user's compatibility with next item}}} + \underbrace{\gamma^{(ij)}_i \cdot \gamma^{ji}_j}_{\clap{\text{next item's compatibility with previous item}}}.
    \label{eq:FPMC}
\end{equation}
Intuitively, this factorization simply states that the next item should be compatible with both the user and the previous item consumed. Note that (as indicated by superscripts), item parameters are not shared between the terms $\gamma^{(iu)}$, $\gamma^{(ij)}$, $\gamma^{(ji)}$, i.e.,~we use separate sets of factors when modeling how an item interacts with a user vs.~another item.

Ultimately, the model is optimized using a BPR-like framework (as in \Cref{sec:bpr}), i.e., using a contrastive loss of the form
\begin{equation}
    \sigma( f(u,i|j) - f(u,i'|j) ),
    \label{eq:FPMC_BPR}
\end{equation}
where $i'$ is a sampled negative item that the user did not consume. 

Experiments compare two variants of FPMC.
%, which are essentially ablations. 
Regular Matrix Factorization (MF), includes only the first term in \Cref{eq:FPMC}; the other, Factorized Markov Chains, includes only the second. That is, they model $f(u,i) = \gamma_u \cdot \gamma_i$ and $f(i|j) = \gamma_i \cdot \gamma_j$. These experiments thus measure (albeit on a specific dataset) the extent to which future actions can be explained by the previous action, versus overall historital preferences.

Ultimately FPMC outperforms both variants, though interestingly FMC and MF outperform each other under different conditions. Importantly, FMC is particularly effective in \emph{sparse} settings (i.e.,~few interactions per user/item) whereas MF works better on dense data.
%\todo{Do I ever define ablation?}
%show (vs.~MF and FMC)

\subsection{Locality-Based Sequential Recommendation}
\label{sec:PRME}

In \Cref{sec:aggregation}, we discussed how different aggregation functions (besides the inner product) could be useful in various contexts. This proves to be the case in various sequential recommendation settings, where sequential actions follow some notion of \emph{locality}.

In \cite{feng2015personalized}, sequential recommendation schemes were studied in the setting of Point-of-Interest recommendation. In such a setting, the context of the previous action is particularly informative, since the following action is likely to be (geographically) close.

If the actual semantics of the problem demand some notion of locality, then arguably similarity in the latent space should also be based on locality (rather than, say, an inner product).

The framework of \cite{feng2015personalized}, \emph{Personalized Ranking Metric Embedding} (PRME) models transition probabilities using an expression of the form:
\begin{equation}
    f(i|j) = -d(\gamma_i - \gamma_j)^2 = - \| \gamma_i - \gamma_j \|_2^2.
    \label{eq:PRME_sequential}
\end{equation}
% where $Z(i)$ is a normalization term:
% \begin{equation}
%     Z(i) = \sum_{k = 1}^{|I|}  e^{- \| \gamma_i - \gamma_k \|_2^2}
% \end{equation}
Note two differences between this model (PRME) and FPMC (\Cref{sec:FPMC}):
\begin{itemize}
    \item The main difference is the use a distance (actually a squared distance) function, so that sequential activities exhibit locality in the latent space.
    \item Unlike FPMC, which used separate latent spaces $\gamma^{(ij)}$ and $\gamma^{(ji)}$ for the next and previous item, PRME uses only a single latent space (which saves parameters).
    % \item Finally, the normalization factor allows the next item to be interpreted as a probability, though this is an implementation detail.
\end{itemize}
Like FPMC, PRME also includes an expression encoding the compatibility between the user and the item (again using a distance function), and also trains the model using a BPR-like framework (i.e.,~including a negative item $i'$ as in \Cref{sec:FPMC_BPR}). Other specific details include an explicit feature encoding geographical distance (based on latitude and longitude), and a temporal feature which down-weights the influence of the sequential term if sequential events are temporally far apart.

Though the authors of PRME argue that Euclidean distance is a more natural way of comparing sequential items (and show that PRME outperforms FPMC for POI recommendation), it should be noted that whether one similarity function is `better' than the other largely depends on the semantics of the specific problem and dataset.

Another paper makes use of a similar model for personalized playlist generation \cite{chen2012playlist}. Like PRME, their model (Factorized Markov Embeddings, or FME) notes that sequential songs in playlists tend to be highly localized, such that a metric embedding is possibly well-motivated. The compatibility function given a user $u$, song $i$, and previous song $j$ takes the form
\begin{equation}
    f(i|u,j) = - d( \gamma^{(\mathrm{start})}_i - \gamma^{(\mathrm{end})}_j )^2 + \gamma_u \cdot \gamma'_i.
    \label{eq:FME_sequential}
    %\todo{f(i|u,j) or f(i,u|j)?}
\end{equation}
Note a few differences between FME and PRME:
\begin{itemize}
    \item FME uses a separate embedding for the next song ($\gamma^{(\mathrm{start})}$) and the previous song ($\gamma^{(\mathrm{end})}$). The basic idea being that songs in playlists should not just be highly local, but should gradually `transition' from one song to the next, so that the `start point' of the next song in latent space should be similar to the `end point' of the previous song (\Cref{fig:translation}).
    \item FME uses a combination of both a distance function (for compatibility with the previous item) and an inner product (for compatibility with the user) in \Cref{eq:FME_sequential}. Again, this demonstrates that the correct choice of compatibility function is highly dependent on problem semantics.
\end{itemize}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/translationIdea.pdf}
\caption{Some sequential models use the principle of translation to model sequential transitions between items. \label{fig:translation}}
%\todo_{Add a second figure for transrec}
\end{SCfigure}

\subsection{Translation-Based Recommendation}

A third class of models for sequential recommendation are also based on the principle of translation. \cite{he2017translation} built recommender systems by borrowing ideas from knowledge-base completion.

Several techniques for knowledge base completion are based on the principle of learning low-dimensional embeddings that describe relationships among entities \cite{bordes2013translating,wang2014knowledge,10.5555/2886521.2886624}. The basic idea is to represent both entities and relationships as vectors in a low-dimensional space, such that a relation vector encodes how to `translate' between entities. For example, we might seek to learn vectors describing entities such as \emph{`Alan Turing'} and \emph{`England,'} such that given a vector describing the relation \emph{`born in'} we should have
\begin{equation}
    d(\overrightarrow{\text{Alan Turing}} + \overrightarrow{\text{born in}}, \overrightarrow{\text{England}}) \simeq 0,
\end{equation}
where $d$ is a Euclidean distance.

\emph{Translation-based recommendation} \cite{he2017translation} adapts this type of approach to personalized recommendation. Whereas for knowledge graph completion relations tell us how to traverse the space of entities, in a recommendation setting, \emph{items} fulfil the role of entities, and \emph{users} traverse the space of items. Then, given two items $j$ and $j$ that are consumed in sequence, we should have
\begin{equation}
    d(\gamma_i + \gamma_u, \gamma_j) \simeq 0.
\end{equation}

Training such a model is quite similar to how we trained FME and PRME in \Cref{sec:PRME}, that is we fit a compatibility function between a user, an item, and a previous item (much like \Cref{eq:FME_sequential}):
\begin{equation}
    f(j|u,i) = \beta_j - \| \gamma_i + \gamma_u - \gamma_j \|_2,
\end{equation}
where $\beta_j$ is incorporated so that the method is capable of capturing overall item popularity as well as pereference. \cite{he2017translation} further constrains item representations to live on a unit ball (i.e.,~$\|\gamma_i\|_2^2 = 1$), which was found to be effective in the knowledge graph completion settings above.

Conceptually




\section{Recurrent Networks}
\label{sec:rnn}

%https://colah.github.io/posts/2015-08-Understanding-LSTMs/
%https://arxiv.org/pdf/1409.2329.pdf

A fundamental limitation of the Markov-Chain-based models we saw in \Cref{sec:markov} is that they have a very limited notion of `memory,' due to the assumption that the next event is conditionally independent of all historical events, given the most recent observation. This assumption may be sufficient in certain scenarios, such as recommendation settings that are highly dependent on the context of the previously clicked item (for example). However, as we begin to model text data (\Cref{chap:text}, or sequence data such as heart-rate logs (\Cref{sec:caseStudyHR}), which have longer-term semantics (such as grammatical structures in a sentence, or even an individual's level of `fatigue' in a heart-rate trace).

\emph{Recurrent Neural Networks} (RNNs) seek to achieve this notion of `memory' by maintaining a `hidden state' during each step. The hidden state is a vector of latent variables that somehow capture the `context' that the model needs to know to capture the long-term semantics of the problem. Formally, we can visualize the RNN as taking a sequence of inputs ($x_1 \ldots x_N$), and producing a sequence of outputs ($y_1 \ldots y_N$), and maintaining hidden states that update at each step ($h_1 \ldots h_N$). We might visualize this model as follows:
\begin{center}
\includegraphics[scale=0.25]{fig/rnn1.pdf}
\end{center}
The RNN cell, depicted here as a box, is now responsible for determining how the hidden state should change at each step, and what outputs should be generated.

More complex RNN models repeat this idea across multiple \emph{layers}, so that RNN cells can be stacked, e.g.:
%\todo{improve figures, latex renders too small}
\begin{center}
\includegraphics[scale=0.25]{fig/rnn2.pdf}
\end{center}
In this depiction (and in many treatments of the topic), the model is described only in terms of hidden states: the model receives an input, updates its hidden state, and passes this state to the next timestep and the next layer. The first layer may receive inputs (i.e.,~observed values $x$) while the last layer generates outputs (i.e.,$y$).

\subsection{The Long Short Term Memory Model}

\begin{figure}
    \centering
\begin{tikzpicture}[
    % GLOBAL CFG
    font=\sf \scriptsize,
    >=LaTeX,
    % Styles
    cell/.style={% For the main box
        rectangle, 
        rounded corners=5mm, 
        draw,
        very thick,
        },
    operator/.style={%For operators like +  and  x
        circle,
        draw,
        inner sep=-0.5pt,
        minimum height =.2cm,
        },
    function/.style={%For functions
        ellipse,
        draw,
        inner sep=1pt
        },
    ct/.style={% For external inputs and outputs
        circle,
        draw,
        line width = .75pt,
        minimum width=1cm,
        inner sep=1pt,
        },
    gt/.style={% For internal inputs
        rectangle,
        draw,
        minimum width=4mm,
        minimum height=3mm,
        inner sep=1pt
        },
    mylabel/.style={% something new that I have learned
        font=\scriptsize\sffamily
        },
    ArrowC1/.style={% Arrows with rounded corners
        rounded corners=.25cm,
        thick,
        },
    ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        thick,
        },
    ]

%Start drawing the thing...    
    % Draw the cell: 
    \node [cell, minimum height =4cm, minimum width=6cm] at (0,0){} ;

    % Draw inputs named ibox#
    \node [gt] (ibox1) at (-2,-0.75) {$\sigma$};
    \node [gt] (ibox2) at (-1.5,-0.75) {$\sigma$};
    \node [gt, minimum width=1cm] (ibox3) at (-0.5,-0.75) {Tanh};
    \node [gt] (ibox4) at (0.5,-0.75) {$\sigma$};

   % Draw opérators   named mux# , add# and func#
    \node [operator] (mux1) at (-2,1.5) {$\times$};
    \node [operator] (add1) at (-0.5,1.5) {+};
    \node [operator] (mux2) at (-0.5,0) {$\times$};
    \node [operator] (mux3) at (1.5,0) {$\times$};
    \node [function] (func1) at (1.5,0.75) {Tanh};

    % Draw External inputs? named as basis c,h,x
    \node[ct, label={[mylabel]Cell}] (c) at (-4,1.5) {\empt{c}{t-1}};
    \node[ct, label={[mylabel]Hidden}] (h) at (-4,-1.5) {\empt{h}{t-1}};
    \node[ct, label={[mylabel]left:Input}] (x) at (-2.5,-3) {\empt{x}{t}};

    % Draw External outputs? named as basis c2,h2,x2
    \node[ct, label={[mylabel]Label1}] (c2) at (4,1.5) {\empt{c}{t}};
    \node[ct, label={[mylabel]Label2}] (h2) at (4,-1.5) {\empt{h}{t}};
    \node[ct, label={[mylabel]left:Label3}] (x2) at (2.5,3) {\empt{h}{t}};

% Start connecting all.
    %Intersections and displacements are used. 
    % Drawing arrows    
    \draw [ArrowC1] (c) -- (mux1) -- (add1) -- (c2);

    % Inputs
    \draw [ArrowC2] (h) -| (ibox4);
    \draw [ArrowC1] (h -| ibox1)++(-0.5,0) -| (ibox1); 
    \draw [ArrowC1] (h -| ibox2)++(-0.5,0) -| (ibox2);
    \draw [ArrowC1] (h -| ibox3)++(-0.5,0) -| (ibox3);
    \draw [ArrowC1] (x) -- (x |- h)-| (ibox3);

    % Internal
    \draw [->, ArrowC2] (ibox1) -- (mux1);
    \draw [->, ArrowC2] (ibox2) |- (mux2);
    \draw [->, ArrowC2] (ibox3) -- (mux2);
    \draw [->, ArrowC2] (ibox4) |- (mux3);
    \draw [->, ArrowC2] (mux2) -- (add1);
    \draw [->, ArrowC1] (add1 -| func1)++(-0.5,0) -| (func1);
    \draw [->, ArrowC2] (func1) -- (mux3);
    %Outputs
    \draw [-, ArrowC2] (mux3) |- (h2);
    \draw (c2 -| x2) ++(0,-0.1) coordinate (i1);
    \draw [-, ArrowC2] (h2 -| x2)++(-0.5,0) -| (i1);
    \draw [-, ArrowC2] (i1)++(0,0.2) -- (x2);

\end{tikzpicture}
    \caption{Caption}
    \label{fig:my_label}
\end{figure}
% Explain the cell state
%https://colah.github.io/posts/2015-08-Understanding-LSTMs/

The Long Short-Term Memory Model \cite{?} is a specific implementation of an RNN
Although not particularly relevant for the current discussion
%\todo{finish}

When designing this cell, one might consider:
\begin{itemize}
    \item Based on the current hidden state, what output should be generated?
    \item How should the hidden state change as a function of the input that was just seen?
    \item What part of the hidden state should be preserved, and what part should be `forgotten'?
\end{itemize}

Note: Hidden Markov Models also do this...



\section{`User-Free' Sequential Models}

When introducing sequential models based on Markov Chains in \Cref{sec:markov?}, we argued that it was valuable to model \emph{both} compatibility between the user and the item, as well as the compatibility between adjacent items in a sequence. In short, both sources provide useful and complementary signals that each explain variation in users' activities.

However it is unclear whether this argument applies in the limit. Clearly, a single previous interaction does not provide the full context of a user's preferences

NARM \cite{li2017neural}
SASRec \cite{kang2018self}
BERT4Rec \cite{sun2019bert4rec}


%\todo{Anything about session-based models? I think one of the user-free models is session based, so maybe discuss the idea there}

\section{Case Study: Personalized Heart-Rate Modeling}
\label{sec:caseStudyHR}


\section{History of Personalized Temporal Models}
%\todo{History sections should have similar title styles}

Although the \emph{Netflix Prize} was one of the early drivers for the use of temporal models in recommendation scenarios, the basic idea of 
%\todo{italicization}
\emph{concept drift}, in which the distribution of labels changes over time, dates back significantly further. For example, the \emph{Dynamic Weighted Majority} algorithm \cite{kolter2007dynamic} considers using an ensemble of classifiers (`experts'); the weighting of these classifiers' decisions changes over time as a function of the empirical performance of the classifiers. \cite{kolter2007dynamic} also gives a historical perspective on the problem of concept drift, dating back to early papers on the topic \cite{schlimmer1986incremental}.

While \cite{koren2009collaborative} showed the effectiveness of temporal factors on the Netflix data,
%\todo{repetitive writing}
a few earlier efforts to use temporal dynamics in recommendation scenarios are noted.
%Adaptive web search based on user profile constructed without any effort from users
For example,
\cite{sugiyama2004adaptive} considered the problem of \emph{personalized search}, though their solution to this problem is actually a form of recommender system. Theirs is an interesting case-study in the use of simple similarity-based methods (like those of \Cref{eq:pearsonSim} and \Cref{eq:simPredict3}) for a problem (personalized search) that doesn't immediately appear to be a recommender systems problem. In terms of temporal dynamics, the main idea is to distinguish between `persistent preferences' and `ephemeral prefernces' when building user profiles (from interaction histories), though this is mostly achieved by considering interactions within differently sized recency windows.




Time weight collaborative filtering
\cite{ding2005time}




%https://colah.github.io/posts/2015-08-Understanding-LSTMs/



\section{Problems}

\begin{problemlist}
 \item Our approaches to similarity-based recommendation (write something that uses a similarity-based recommender but considers both user-item and item-to-item similarity)
 \item Something with auto-regression (if it's in this chapter). E.g.~compare sliding windows vs auto-regression in terms of the MSE.
 \item In \Cref{ex:developLatentFactorModelBPR}, we developed a model that included user and item latent factors $\gamma_u$ and $\gamma_i$.
 %\todo{This exercise should be a BPR-like model, not a rating prediction model. Requires that there is a BPR exercise in the previous section}
 Adapt the model so that instead of modeling the user, it models the previous item as $\gamma_j$
 \item In \Cref{chap:sequence} we introduced Factorization Machines as a technique to incorporate various types of features into recommendation approaches (and discussed libraries to do so in \Cref{sec:implicitLibrary}). When introducing Factorized Personalized Markov Chains in \Cref{sec:FPMC} we argued that this type of model can be represented via a factorization machine.
\end{problemlist}


\chapter{Personalized Models of Text}
\label{chap:text}

Make the points that (1) many personalization tasks involve text somehow; (2) models from natural language are often used when modeling sequence data -- item interactions etc. can benefit from similar sequence models.

\section{Basics of Text Modeling: The Bag-of-Words Model}

Many of the datasets we've already been working with contain natural language data. Several tasks can potentially benefit from from this data when making predictions, and as we'll see later, text can drive a variety of unique applications.
%\todo{maybe cover this in intro to chapter, otherwise this paragraph is too isolated anyway}

\subsection{Sentiment Analysis}
\label{sec:sentiment}

To understand why modeling textual data is difficult, let's consider the seemingly simple task of predicting a rating based on the text of a review:
\begin{equation}
    \text{rating} = X^{(\text{review})} \cdot \theta
    \label{eq:sentiment0}
\end{equation}
%\todo{Capital X isn't really consistent notation}
This is the same as the type of problem we set up in \Cref{chap:basics}, except that our features $X$ are derived from the reviews. Intuitively it makes sense that reviews should help us predict ratings, as they are specifically intended to explain a user's rating.

The task described in \Cref{eq:sentiment0} captures the basic setting of \emph{sentiment analysis}, i.e.,~learning what types of features are associated with `positive' (i.e.,~high ratings) and `negative' language. The main challenge is how to appropriately extract meaningful features from text.

\begin{floatingbox}
\caption{What's the point of sentiment analysis?}
%TODO: Need citations for this box!
Sentiment analysis, viewed superficially, may seem like an odd (or a `toy') task: why would we ever want to predict ratings from reviews, given that in practice we never observe a review \emph{without} a rating? Nevertheless, sentiment analysis is one of the core topics in natural language processing whose importance transcends the immediate task of predicting ratings. Sentiment analysis research is generally focused on:
\begin{itemize}
    \item Understanding the sociolinguistic
    %TODO: Right word?
    dimensions of sentiment, rather than the immediate task of predicting the rating.
    \item Building sentiment models that are \emph{general purpose}, i.e., models that can be trained on one corpus (e.g.~of reviews), but can be used to predict sentiment in settings where ratings aren't available.
    \item As a benchmarking task to test scalability, and the ability of NLP systems to understand detailed nuances in language.
\end{itemize}
\end{floatingbox}

The first problem that we must deal with is that the features in \Cref{eq:sentiment0} are \emph{fixed length} (i.e.,~$X$ is a matrix), whereas text is sequence data. Later, we'll see how to model sequences more directly in \Cref{sec:RNN},
%\todo{will we?}
but for now let's see how much progress we can make just by extracting a pre-defined set of features from each review.

\emph{Bag-of-words} models attempt to solve this by composing $X$ out of features that encode the presence or absence of certain words in a model. Thus it ignores key details such as the order in which words appear (see \Cref{fig:bowExample}).

\begin{floatingbox}
\caption{Bag-of-Words models}
\begin{center}
\parbox{0.48\textwidth}{ \small Loved every minute. So sad there isn't another! I thought JK really made Harry an even stronger archetypal hero - almost in a Paul Maud'Dib from Dune kind of way. He's fighting the ultimate evil, he's brave and takes risks, and believes in himself and doesn't give up despite many hardships.}\hspace{0.025\textwidth}\parbox{0.48\textwidth}{ \small risks, Paul and hardships. believes the almost sad ultimate kind up every - an there in brave hero I fighting Dune another! way. himself made really he's despite He's Loved from archetypal minute. and a Maud'Dib isn't even evil, of in many stronger So takes JK thought Harry give and doesn't}
\end{center}
\caption{The two reviews above have identical \emph{bag-of-words} representations (the second randomly shuffles the words of the first). The review at right misses details that depend on syntax. Is there still enough information in the review at right to tell whether the overall sentiment is positive? \label{fig:wobExample}}
\end{floatingbox}

A key component in the bag-of-words model is the \emph{dictionary} that is used to build our feature vector. Our first attempt to build this dictionary might merely consist of compiling \emph{every} word in a given corpus, e.g.:
%\todo{Find better way to include code}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
wordCount = defaultdict(int)
for d in data:
  for w in d['review/text'].split():
    wordCount[w] += 1

print(len(wordCount))
\end{lstlisting}
\end{minipage}
%\todo{Do I want print statements in these?}

Doing so on just the first 5,000 of our beer reviews
%\todo{Use this or something else}
reveals a total of \emph{36,225} unique words. In other words, each review (on average) contains around seven previously unseen words. This number sounds large, but we might think that by the time we have seen 5,000 reviews that we have `saturated' the vocabulary of English words, and will not see too many more. However, if we repeat the same experiment for (e.g.)~10,000 reviews, we obtain 55,699 unique words---not quite double, but still a substantial increase, revealing that we do not quickly saturate the vocabulary.

Looking at a few of the actual words in our dictionary, we see words like:
\begin{equation*}
    \text{...the}; \quad \text{09:26-T04.}; \quad \text{Hopsicle}; \quad \text{beery}; \quad \text{\#42}; \quad \text{\$10.65}; \quad \text{(maybe}; \quad \text{(etc.)}
\end{equation*}
That is we see words including proper nouns, unusual spelling, prices, punctuation, etc. From this, we quickly see that we will not quickly run out of unique `words,' given any reasonable number of reviews.

To use a bag-of-words representation, we will need to reduce this dictionary to a manageable size. Some potential steps to do so include:

\paragraph{Removing capitalization and punctuation}
Removing punctuation will reduce our dictionary size substantially, as words like `(maybe' (i.e.,~a word following a parenthesis) will be resolved to a common word. Likewise, we might ignore different capitilization patterns simply by converting all documents to lower-case.

\paragraph{Stemming}
Stemming, i.e.,~resolving similar words to their common word stems. Words like `drink,' `drinking,' and `drinks' would all map to `drink'.\footnote{Or words like `argue,' `arguing,' and `argus' would map to `argu'---the stem need not be an actual word.} Such techniques are mostly motivated by search and retrieval settings (i.e.,~to make sure results are retrieved even if a query uses a different word inflection than the result), though they could also be used to reduce our dictionary size. See e.g.~\cite{lovins1968development,porter1980algorithm} for examples of stemming algorithms.

\paragraph{Stopwords}
Stopwords are common English words that likely carry (relatively) little predictive power compared to their frequency in a document. Standard stopword lists\footnote{See e.g.~\texttt{nltk.corpus.stopwords} in Python.} include words such as `am,' 'is,' `the,' `them,' etc. Removing them can reduce our dictionary size a little, or otherwise prevent our feature representations from being overwhelmed by common words (though we will see other ways to address this in \Cref{sec:tfidf}).

Decisions like whether to remove punctuation, whether to stem, or whether to remove stopwords, are largely dataset and application dependent. Characters like exclamation points could be predictive of sentiment;\footnote{Often, important punctuation characters are preserved by treating them like separate words, e.g.~the string `great!'~would be replaced by `great !'.} or different word inflections (such as `drinks' or `drinker' in a corpus of beer reviews)
%\todo{not the best example}
may have different meanings; or stopwords like `i' and `her' could change the meaning of a sentence.

As such, the procedures above essentially amount to feature engineering choices: we should ultimately accept or reject the above procedures based on whether they improve performance for our given application (again, these are the kind of choices we'd make with our validation set as in \Cref{sec:validation}).

For the moment, let's consider removing punctuation and capitalization, e.g.:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
for d in data:
  r = ''.join([c for c in d['review/text'].lower() if not c in string.punctuation])
  for w in r.split():
    wordCount[w] += 1
\end{lstlisting}
\end{minipage}
%\todo{Indentation (probably four spaces always)}
After removing them, we are left with 19,426 unique words. This is a reduction of nearly half compared to what we had before removing them, but is still a fairly large dictionary size.

A more straightforward way to reduce our dictionary to a manageable size is simply to include only the most commonly occurring words:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
counts = [(wordCount[w], w) for w in wordCount]
counts.sort()
counts.reverse()

words = [x[1] for x in counts[:1000]]
\end{lstlisting}
\end{minipage}
Although perhaps not a completely satisfactory solution, this representation at least allows us to build a feature representation. 
%Put later
%In this case the 1,001$^\text{st}$ most popular word in this example appears in around one percent of all reviews. 
A simple bag-of-words-based sentiment analysis model would now consist of predicting:
\begin{equation}
    \mathrm{rating} = \theta_0 + \sum_{w \in \mathcal D} \mathrm{count}(w) \cdot \theta_w.
    \label{eq:sentiment1}
\end{equation}
Here $\mathcal D$ is our \emph{dictionary} (i.e.,~our set of words); $\theta$ is indexed by a word $w$, but in practice we would replace each word by an index (here from 1 to 1000) for the sake of building a feature matrix.

Fitting such a model on our beer review dataset\footnote{Here with an $\ell_2$ regularizer with coefficient $\lambda=1$, 4,000 data points for training and 1,000 for testing.} yields a training MSE of 0.27 and a testing MSE of 0.51; this attests to both the high accuracy of such models, as well as their ability to overfit.

Examining the coefficients $\theta$, we find the five words associated with the most negative coeficients are:
\begin{eqnarray*}
 \theta_\mathrm{skunk} & = & -0.364;\\
 \theta_\mathrm{oh} & = & -0.312;\\
 \theta_\mathrm{skunky} & = & -0.292;\\
 \theta_\mathrm{bland} & = & -0.284;\\
 \theta_\mathrm{recommend} & = & -0.267,
%(-0.3635808995815085, 'skunk'),
%(-0.31193813774769147, 'oh'),
%(-0.29234181779758006, 'skunky'),
%(-0.2839149980664344, 'bland'),
%(-0.26701385744199396, 'recommend')
\end{eqnarray*}
and the five most positive words are:
\begin{eqnarray*}
\theta_\mathrm{raisins} & = & 0.204;\\
\theta_\mathrm{impressed} & = & 0.224;\\
\theta_\mathrm{keeps} & = & 0.234;\\
\theta_\mathrm{always} & = & 0.256;\\
\theta_\mathrm{exceptional} & = & 0.320
%(0.2043839147952575, 'raisins'),
%(0.22364690773701465, 'impressed'),
%(0.23419283704758836, 'keeps'),
%(0.2564967471468373, 'always'),
%(0.3197211175655953, 'exceptional')
\end{eqnarray*}
For example, every instance of the word `skunk' (a negative association with beer) decreases our prediction of the rating by around one third of a star; likewise every instance of the word `exceptional' increases our prediction by around the same amount. A few more observations:
\begin{itemize}
    \item Words like `skunk' and `skunky' presumably convey the same information, indicating some redundancy in our representation; possibly this could be addressed by stemming.
    \item The word `oh' is extremely negative, despite conveying little meaning by itself. Presumably, it occurs in phrases like `oh no' (whereas `no' itself appears in a wide variety of phrases so is not as negative); possibly we could account for such confounds by using n-grams (\Cref{sec:ngrams}).
    \item Likewise words like `always' or `keeps' are highly positive despite conveying little sentiment in isolation; the word `raisins' possibly appears in the context of a specific popular item.
    \item The word `recommend' is highly \emph{negative}, in spite of seeming to convey positive sentiment. Presumably, this word frequently appears in negative phrases (`would not recommend,' etc.); we could account for such confounds by better handling of negation.
\end{itemize}
Finally, it is notable that the words `skunk' and `exceptional' are the 962$^\text{nd}$ and 991$^\text{st}$ most popular words in our corpus---that is, they are words we nearly discarded when we selected a 1,000 word dictionary. On the one hand, this might be an argument that our dictionary size was too small, since we nearly missed the most important words. On the other hand, it is almost true by definition that the most predictive words will be rare ones: it is, after all, quite exceptional for an item to be described as `exceptional'!

\subsection{N-grams}
\label{sec:ngrams}

Some of the oddities with our sentiment model from the previous section possibly arise due to the simplifying assumptions made by the bag-of-words model. Critically, a bag-of-words model has no notion of \emph{syntax} in a document, and as such cannot handle even simple concepts such as negation (e.g.~`not bad') or compound expressions that carry different meaning together than alone (e.g.~`oh no').

\emph{N-gram} models attempt to address some of these issues by considering words that frequently co-occur in sequence. That is, \emph{bigrams} consist of pairs of words that are adjacent in a document; \emph{trigrams} consist of groups of three words that appear consecutively in a document (etc.). \text{not great writing}

For example, the N-grams associated with the following sentence would be:
\begin{eqnarray*}
\begin{tabular}{ll}
Sentence: & `Dark red color, light beige foam'\\
Unigrams: & [`Dark', `red', `color,', `light', `beige', `foam']\\
Bigrams: & [`Dark red', `red color,', `color, light', `light beige', `beige foam']\\
Trigrams: & [`Dark red color,', `red color, light', `color, light beige', $\ldots$]\\
(etc.)
\end{tabular}
\end{eqnarray*}
From the above example we can already see some potential benefits of using an N-gram representation, e.g.~several of the words in the above sentence are adjectives that modify the words `color' and `foam;' without an N-gram representation we might fail to correctly understand those nouns in context.\footnote{Though one could argue the opposite: if an adjective like `beige' rarely occurs in any other context, then a regular bag-of-words representation may be sufficient to capture the relevant information.} Similarly, negated terms (e.g.~`not bad' etc.)~are readily handled by an N-gram representation.

N-grams are straightforward to extract in Python, e.g.:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
sentence = "Dark red color, light beige foam"
unigrams = sentence.split()
bigrams = list(zip(unigrams[:-1], unigrams[1:]))
trigrams = list(zip(unigrams[:-2], unigrams[1:-1], unigrams[2:]))
\end{lstlisting}
\end{minipage}

Having extracted N-grams, our approach to modeling the data is much the same as that of our bag-of-words model, i.e.,~we extract counts associated with each N-gram, and use those counts as features to predict some outcome.

Note that generally we would not use (e.g.)~bigrams exclusively, but rather would use a representation that included both unigrams and bigrams simultanously. Much like our previous approach from \Cref{sec:sentiment}, which simply found the most popular words (i.e.,~unigrams), we might simply count the popularity of unigrams and bigrams simultaneously.

On the same review corpus as in \Cref{sec:sentiment}, the majority of popular terms are still unigrams---N-grams comprise 452 of the 1,000 most popular terms.

A few of the most popular N-grams include terms like `with a,' `in the,' `of the,' etc. At first glance, these don't look particularly useful, and are mostly combinations of stopwords. Likewise, few of the top N-grams appear to include negation---for example among the top 1,000 terms, only eleven include the word `not', including e.g.~`but not', `not a', `is not', `not much', `not too', etc. Again these seem unlikely to be informative given that they do not modify meaningful terms.

The above example highlights that N-grams are not quite a simple panacea for the issues mentioned above about handling adjectives and negation (etc.). Indeed, one must carefully trade off the fact that N-grams introduce substantial redundancy into our feature vector with the possibility that they introduce some useful compound terms. Note that it is not straightforward to get the `best of both worlds' in this scenario: assuming a dictionary of fixed size, we are potentially discarding informative unigrams (like `exceptional' from \Cref{sec:sentiment}) in favor of less-informative N-grams. Again, these issues are model- and dataset-specific, and likely could be addressed by better accounting for stopwords (or possibly by using a larger dictionary). Mostly this example simply highlights that we must make extra considerations when incorporating N-grams into a model, and that they will not confer benefits unless carefully handled.

Some of these potential advantages and disadvantages of N-gram representations are summarized in \Cref{box:Ngrams}. Many of the disadvantages center around the redundancy of the representation, which can partly be mitigated by carefully selecting \emph{important} features. We will revisit the topic of word important in \Cref{sec:tfidf}

\begin{floatingbox}
\caption{Arguments For and Against N-grams. \label{box:Ngrams}}
To summarize our discussion in \Cref{sec:ngrams}, N-grams are not always beneficial in language modeling tasks, partly because one has to trade of the predictive value of N-grams against the redundancy in encoding them. Some positive and negative points are summarized below:
\begin{itemize}
    \item N-grams can straightforwardly allow us to handle negation, and various forms of compound expressions, 
    \item Our dictionary size quickly multiplies when using N-grams. Assuming that we can handle a fixed dictionary size, in practice this sometimes means some informative unigrams will be replaced by uninformative N-grams.
    \item Stopwords, which made up a small fraction of our unigram dictionary, quickly multiply when building N-gram representations; thus there is additional redundancy in the N-gram representation.
    \item An N-gram representation may add substantial redundancy (or co-linearity) between features, e.g.~as informative unigrams are now duplicated among several N-gram features.
    %TODO: This
    %\item Colinearity / redundancy (maybe two points
    %\item Not useful words?
\end{itemize}
\end{floatingbox}

Finally, let's evaluate an N-gram representation on the same task from \Cref{sec:sentiment}. Again we'll build a `bag-of-ngrams' model by taking the 1,000 most popular N-grams (for any value of N). Once extracted our model is again the same as that from \Cref{eq:sentiment1}, the only difference being that our dictionary consists of a combination of N-grams of different lengths.

After fitting the model, performance in fact slightly degrades compared to the model from \Cref{sec:sentiment} (testing MSE of 0.54 compared to 0.51 in \Cref{sec:sentiment}). Examining some of the most predictive N-grams (i.e.,~largest and smallest values of $\theta_w$) we find terms such as
\begin{equation*}
    \theta_{\text{pitch black}} = -0.397; \quad \theta_{\text{ptich}} = 0.354.
\end{equation*}
Upon further inspection, the word `pitch' \emph{always} appears in the expression `pitch black', to the point that the two terms will mostly `cancel out'; again this highlights an issue of redundancy (and co-linearity, as in \Cref{sec:colinear}) of our representation.
%\todo{rushed prose}

Possibly we can address this simply by further regularizing our model. Increasing the regularization coefficient to $\lambda = 10$ improves the performance somewhat (test MSE of 0.506), and results in more reasonable coefficients, such as:
\begin{equation}
    %\theta_{\text{water}} = -0.232; \quad
    \theta_{\text{wonderful}} = 0.177; \quad
    \theta_{\text{not bad}} = 0.174; \quad
    \theta_{\text{low carbonation}} = 0.137,
\end{equation}
which appear to correctly handle negation (`not bad') and compound words (`low carbonation').

%\subsection{Personalized Sentiment Analysis}

\subsection{Word Relevance and Document Similarity}
\label{sec:tfidf}
%\todo{Is this the right section label?}
%\todo{Worth having this section at all?}

%A question that arose in the previous section was how to prevent our feature representations from being weighed down by uninformative or `irrelevant' words. To investigate this idea further, we need to specify what it means for a word to be `relevant' in a document.
%\todo{mediocre motivation, since we don't really use tf-idf to build dictionaries}

%When building a dictionary in \Cref{sec:sentiment} we simply counted the number of times each word appeared across all documents, i.e.,~we took the most popular words. However, this left us with many stopwords (or otherwise irrelevant words), which (probably not surpringly) are the most common words in most documents.

Suppose we wanted to build a system that recommends articles that are similar in content to ones a user had recently interacted with. As in \Cref{sec:neighborhood}, we might do so by defining an appropriate similarity metric between articles, and recommend those articles that are most similar:
\begin{equation}
    f(i) = \argmax_j \mathrm{Sim}(i,j).
\end{equation}
Given that our goal is to define similarity based on article \emph{content} (rather than interaction histories as in \Cref{sec:neighborhood}), we might consider defining similarity based on the feature representations we developed in \Cref{sec:bagOfWords}. For example, we could compare the cosine similarity of two bag-of-words representations:
\begin{equation}
    \mathrm{Sim}(i,j) = \text{Cosine Similarity}(x_i, x_j).
\end{equation}
%\todo{sim / Sim is not consistently capitalized (in various sections)}
%\todo{Jaccard is italicized (in other section), but should it be?}
%\todo{Never refered to BoW vectors as x_i. Maybe call x^Bow_i or something}
However as we discussed in \Cref{sec:bagOfWords}, the vectors $x_i$ and $x_j$ will be dominated by the most common words in the corpus (i.e.,~the largest magnitude words will likely be stopwords).

In practice a user would probably not regard two documents as `similar' just because they used words like `the,' `of,' or `and' in similar proportions. As such, we presumably want a feature representation $x_i$ that focuses on \emph{relevant} terms.

In \Cref{fig:tfidfexamp} we see an example of a book review (of \emph{Watership Down}) with the most frequently occurring words in boldface. Naturally, we would not say that the topic of this review was `i' or `a,' even though those words are the most frequently occurring.

\begin{figure}
\parbox{0.48\textwidth}{ \small \textbf{I} read \textbf{this} after hearing from \textbf{a} few people that \textbf{it} \textbf{was} among their all-time favorites. \textbf{I} \textbf{was} almost put off when  \textbf{I} \ul{saw} \textbf{it} \textbf{was} \textbf{a} \textbf{story} \ul{\textbf{about}} rabbits, originally written as \textbf{a} tale by \textbf{a} \ul{father} \textbf{to} his \ul{children}---but I'm \ul{glad} \textbf{I} wasn't. \textbf{I} found \textbf{the} folk \ul{tales} \ul{\textbf{about}} El-ahrairah \textbf{to} be very impressive. \textbf{The} author clearly had \textbf{a} vivid imagination \textbf{to} create so much of \textbf{the} rabbits culture \textbf{and} \ul{history}. But \textbf{I} think \textbf{this} book \textbf{was} \ul{worth} reading as it's really \textbf{a} \textbf{story} \ul{\textbf{about}} survival, leadership, \textbf{and} \ul{human} \ul{nature}. Oh \textbf{and} Fiver rocks. \textbf{And} BigWig is \textbf{the} man.}\hspace{0.04\textwidth}\parbox{0.48\textwidth}{I was delighted by this book... the only fault is that it was too short! What a fantastic idea; a refuge for the children who have had adventures \& now cannot fit back into the identity assigned to them. How many of us are not comfortable in the families we were born to? I loved the way the different doorways were sorted; one would think that adventures shared would be a bonding moment. Rivalries will be ever present; guess that is human nature. I don't want to describe too much \& ruin the magic[...]
%this author is the heir apparent to Neil Gaiman. Im so glad she has more books to discover !
}
\caption{Term frequency and \emph{tf-idf} comparison. At left, the top 10 words by term frequency are bolded (i.e.,~the most common words in the review), and top 10 tf-idf words (based on a sample of 50,000 reviews) are underlined. A highly-similar review (based on cosine similarity of \emph{tf-idf} vectors is shown at right. \label{fig:tfidfexamp}}
\end{figure}

Rather, we might argue that the words like `nature' or `children' are more characteristic of the document, presumably because they do \emph{not} occur in most documents. As such, we might consider words characteristic of a particular document to be those that occur frequently in that document but not in others.

To capture such a definition we should separately consider word frequency in a particular document, and frequency across a corpus at large. To do so we define two terms. First, the \emph{term frequency} of a word $w$ in a document $d$ is simply the number of times that word appears in the document:
\begin{equation}
    \text{Term Frequency}(w,d) = \mathit{tf}(w,d) = |\langle t \in d | t = w \rangle|.
\end{equation}
Note that this is essentially the same as the Bag-of-Words representation we developed in \Cref{sec:bagOfWords} (although the latter includes a fixed dictionary size).

Next, we define the \emph{document frequency} of a word across a set of documents in terms of a word $w$ and a corpus $\mathcal D$:
\begin{equation}
    \text{Document Frequency}(w,\mathcal D) = \mathit{df}(w,\mathcal D) = |\langle d \in D | w \in D|.
\end{equation}

Now, for a word to be `relevant' in a particular document, we want the term frequency $\mathit{tf}(w,d)$ to be high, and the frequency of the word across the entire corpus $\mathit{dF}(w,\mathcal D)$ to be relatively low. The \emph{tf-idf} measure (term frequency-inverse document frequency) is a heuristic which achieves this goal via the following function:
\begin{equation}
    \mathit{tf-idf}(w,d,\mathcal D) = \mathit{tf}(w,d) \times \log_2 \left( \frac{|\mathcal D|}{\mathit{df}(w,\mathcal D)} \right).
    \label{eq:tfidf}
\end{equation}
%\todo{How to get a small hyphen in tf-idf in math mode?}
%\todo{Should change all of the above (and below) to be (t,d) rather than (w,d)}
While the expression above captures our intuition that the term frequency should be high while the document frequency is relatively low, the specific expression may seem somewhat arbitrary (e.g.~the inclusion of the $\log_2$ term). Indeed, this expression is merely a heuristic, as described in the original implementation \cite{jones1972statistical}. Later work has attempted to justify this choice, e.g.~by interpreting $\log_2\frac{|\mathcal D|}{\mathit{df}(w,\mathcal D)}$ as a log-probability of a word appearing in a document \cite{robertson2004understanding}, though these are post-hoc justifications for what was ultimately a heuristic choice.

Likewise, the term frequency is also a heuristic and is often modified for use in specific contexts. For instance, two alternate definitions of the term frequency include:
\begin{eqnarray}
\mathit{tf}'(w,d) & = \delta(w \in d)\\
\mathit{tf}''(w,d) & = \frac{\mathit{tf}(w,d)}{\max_{w' \in d} \mathit{tf}(w',d)}.
%\todo{change w to t, maybe w' to w}
\end{eqnarray}
Both of the above are essentially normalization schemes, intended to prevent \emph{tf-idf} scores being higher for longer documents.

\subsection{Using TF-IDF for Search and Retrieval}

Cover BM-25?

% \section{Recurrent Models of Text}


\section{Distributed Word, Item, and Document Representations}

\emph{Word2Vec} is a popular approach to developing \emph{semantic representations} of words \cite{mikolov2013distributed}. Such representations are somewhat analogous to the user and item representations ($\gamma_u$ and $\gamma_i$) we have been studying throughout this book. 
%That is, just as our user and item representations help us to predict which items `go with' which user (based on some latent representation of user preferences and item properties), word representations ought to tell us which words are likely to `go with,' or appear in the 
That is, just as our latent item representations give us a sense of which items are `similar to' which others (likewise for users), we would like to find latent word representations $\gamma_w$ that tell us which words are similar, or are likely to appear in the same context as each other.

These types of `distributed' word representations are potentially useful for a variety of reasons:
\begin{itemize}
    \item Unlike bag-of-words models (\Cref{sec:bagOfWords}), distributed representations offer a natural mechanism to handle synonyms. That is, words $w$ and $v$ that are synonyms of each other ought to have nearby representations $\gamma_w$ and $\gamma_v$, since they will tend to appear in the same contexts.
    \item Beyond synonyms, distributed representations might allow us better understand \emph{relationships} among words.
    %\todo{Say something more about this (king vs. queen etc.)
    \item In certain settings, distributed representations allow us to avoid dimensionality issues associated with bag-of-words models. For instance, when developing generative models of text (which we'll touch on briefly in \Cref{sec:generativeText}), documents are typically represented as sequences of low-dimensional word vectors $\gamma_w$, rather than as vectors of (e.g.)~word IDs via a bag-of-words model.
\end{itemize}

Below we briefly outline \emph{word2vec} as described in \cite{mikolov2013distributed}; in \Cref{sec:item2vec} we describe how this idea applies to learning item representations $\gamma_i$ for recommendation.
Although the latter and the former are essentially equivalent, the latter may feel more familiar, as it is similar to the way we learned item representations $\gamma_i$ using sequence models in \Cref{sec:?}.
%\todo{Is above useful? And does it reflect the final structure?}

Methodologically, \emph{word2vec} seeks to model the probability that a word in a sequence $w_t$ appears nearby words $w_{t+k}$, i.e.,~$p(w_t | w_{t+k})$. So, for a sequence of words $w_1 \ldots w_T$, we would like to learn word representations that maximize the (log) probability
\begin{equation}
    \frac{1}{T} \sum_{t=1}^T \sum_{-c \leq j \leq c, j \neq 0} \log p(w_{t+j} | w_t).
    \label{eq:word2vec0}
\end{equation}
%https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf
Here $c$ is the size of a \emph{context window}, which determines how many neighboring words we consider; this is a hyperparameter
%\todo{do I define this word somewhere?}
that may be chosen to balance accuracy and training time, though potentially can vary depending on the word $w_t$.
A simple way to define the probability $p(w | v)$ is to say that words $w$ are likely to appear near words with similar representations. In  \cite{mikolov2013distributed} this is defined in terms of the inner product between representations:
\begin{equation}
    p(w_o | w_i) = \frac{e^{\gamma'_{w_o} \cdot \gamma_{w_i}}}{\sum_{w \in \mathcal W} e^{\gamma'_w \cdot \gamma_{w_i}}}
    \label{eq:word2vec1}
\end{equation}
%\todo{Do I normally use \mathcal W for a dictionary or word set?}
where $\mathcal W$ is the set of words in the dictionary.
The numerator in the above encodes the compatibility between the `input' and `output' words $w_i$ and $w_o$; the denominator simply normalizes the value so that it corresponds to a probability (much as we saw in \Cref{sec:multiclassClassification}).
%\todo{Do I discuss multiclass anywhere?}

Note also that we learn \emph{two} representations, $\gamma_w$, and $\gamma'_w$ for each word (referred to as the `input' and `output' representation, respectively). 
Although doing so doubles the number of parameters, this type of representation avoids symmetries, for example a word is not likely to appear near itself. This is similar to the idea we saw when developing item-to-item recommender systems, and
sequential recommender systems in \Cref{sec:?},
%\todo{Do I ever describe such an example? Maybe in clothing?}
again the intuition being that an item is not likely to be co-purchased with itself.

Since the denominator in \Cref{eq:word2vec1} requires normalizing across all words in the dictionary $\mathcal W$, it is not efficient to compute. \cite{mikolov2013distributed} suggests a few schemes to overcome this issue, though the most straightforward is simply to sample a small number of `negative' words, rather than normalizing over the whole dictionary. As such each computation of $p(w_o | w_i)$ is replaced by an approximation:
\begin{equation}
    \log p(w_o | w_i) \simeq \sigma(\gamma'_{w_o} \cdot \gamma_{w_i}) + \sum_{w \in \mathcal N} \log \sigma(-\gamma'_{w} \cdot \gamma_{w_i})
    \label{eq:word2vec2}
\end{equation}
where $\mathcal N$ is a sampled set of negative words. \cite{mikolov2013distributed} proposes various schemes for choosing the sample $\mathcal N$, though most critically argues that the sampling probability should be proportional to the overall frequency of each word.

%\subsection{Doc2Vec}
%Probably skip, check e.g. https://medium.com/wisio/a-gentle-introduction-to-doc2vec-db3e8c0cce5e

\subsection{Item2Vec}
\label{sec:item2Vec}

\emph{Item2Vec}
\cite{barkan2016item2vec}
%ITEM2VEC: NEURAL ITEM EMBEDDING FOR COLLABORATIVE FILTERING (is this the actual item2vec paper?)
adapts the basic idea behind \emph{word2vec} as a means of learning item representations $\gamma_i$. The main difference between item2vec and word2vec is simply that sequences of words in sentences/documents are replaced by ordered sequences of items that each user has consumed. In practice this simply means that the probability in \Cref{eq:word2vec2} is replaced by
\begin{equation}
    \log p(i | j) \simeq \sigma(\gamma'_i \cdot \gamma_j) + \sum_{i' \in \mathcal N} \log \sigma(-\gamma'_{i'} \cdot \gamma_j),
\end{equation}
where $\mathcal N$ is a set of negative items, again sampled proportionally to overall item frequency.

\cite{barkan2016item2vec} discusses the effectiveness of this type of item representation in the setting of item-to-item recommendation on a corpus of song listens from \emph{Microsoft Xbox Music}. It was shown that the method naturally identifies latent dimensions that are associated with song genres; and it was argued qualitatively that related items are semantically more meaningful than those produced by alternate item-to-item recommendation techniques.

\subsection{Word2Vec and Item2Vec with Gensim}

Finally, we show how \emph{word2vec} and \emph{item2vec} work in practice, using interaction and review data from beer reviews, much as we did in \Cref{sec:?}.

To learn word representations, the input to the model is a list of documents (in this case reviews), each of which is a list of tokens (words). For this example we first remove capitalization and punctuation before tokenizing, as in \Cref{sec:?}.

Here we use the \emph{Gensim} implementation of \emph{word2vec}.\footnote{\url{https://radimrehurek.com/gensim/}}
%\todo{Don't know if I want urls floating around}
The model takes as input our tokenized reviews (in this case, we use a corpus of 50,000), a minimum word frequency, a dimensionality (i.e.,~$|\gamma_i|$), and a window size (i.e.,~$c$ in \Cref{eq:word2vec0}). The final argument specifies which specific version of the model is used, which corresponds to the model presented above:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from gensim.models import Word2Vec

model = Word2Vec(reviewTokens, # Tokenized documents
                 min_count=5, # Minimum frequency before words are discarded
                 size=10, # Model dimensionality
                 window=3, # Window size
                 sg=1) # Skip-gram model

model.wv.similar_by_word("grassy")
\end{lstlisting}
\end{minipage}

In the final line, we retrieve the most similar words for a particular query; in \emph{Gensim} this is based on the cosine similarity (\Cref{eq:cosinSim}) between the two word vectors:
\begin{equation}
    \max_{w} \frac{\gamma_w \cdot \gamma_{\mathit{grassy}}}{\| \gamma_w \| \| \gamma_{\mathit{grassy}} \|} = \text{`citrus'},
\end{equation}
followed by `citric', `floral', `flowery', `piney', `herbal', etc.
%, i.e.,~the words that maximize $\gamma'{w_o} \cdot \gamma_{w_i}$

Similarly, we can use the same code to run \emph{item2vec}, where our tokenized reviews are replaced by lists of items (i.e.,~product IDs) that each user has consumed (ordered by time).

After training a model on review histories, we find that the most similar beers to \emph{Molson Canadian Light} are other light beers such as \emph{Miller Light}, \emph{Molson Golden}, \emph{Piels}, \emph{Coors Extra Gold}, \emph{Labatt Canadian Ale}, (etc.) In \Cref{fig:item2vec} we train a two-dimensional item2vec model
%\todo{italics}
for the sake of visualizing the data, which reveals that beers belonging to different categories tend to occupy different parts of the item space.\footnote{A more effective visualization might be produced by using higher-dimensional embeddings, followed by a distance-preserving visualization technique like t-SNE \cite{maaten2008visualizing}, though for here we plot the embedding dimensions directly for simplicity.}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/item2vec.pdf}
\caption{Item representations ($\gamma_i$) using a two-dimensional \emph{item2vec} model. Representations for items from three distinct categories are shown. \label{fig:item2vec}}
\end{SCfigure}



\section{Personalized Sentiment and Recommendation}

The models for text we have explored so far, although they can be applied to data like reviews, and can be used to recommend related documents, are ultimately not personalized.

Several attempts have been made to combine models of text with models of users and preferences, and in particular with recommendation approaches. For example, just as we saw techniques in \Cref{chap:content} that improve recommender systems through the use of side-information, \emph{text} can be useful to efficiently understand the dimensions of user's opinions.

Often there is a significant amount of textual data available in addition to interactions that might be leveraged to fit better models. Other than helping understand sentiment, text can help to understand the dimensions of items and preferences, e.g.~the different properties of products and the different aspects that users care about.

Text can also be helpful for model \emph{interpretability}. So far, the recommender systems we've developed are essentially `black boxes,' whose predicions (as in \Cref{eq:latentFactor?}) are defined purely in term of latent factors. Models of text can be used to understand what aspects these latent dimensions correspond to, to summarize reviews, or to explain recommendations.
%\todo{Add references if I eventually have sections on these things}

However, extracting \emph{meaningful} information from text is challenging. E.g.~most of the simple text representations we've seen so far (\Cref{sec:bagOfWords}) are high-dimensional and not particularly sparse; simply incorporating such features into general-purpose feature-aware models is possibly not effective, and one must instead design methods to work specifically with text. Below we cover a few representative approaches.

%TODO: Anything about personalized sentiment analysis specifically?

\subsection{Case Studies: Review-Aware Recommendation}

\emph{Product reviews} are often used as a source of information to improve recommendation performance. Conceptually, reviews ought to tell us much more about preferences and opinions than (e.g.)~a single rating can. This could be especially true for latent dimensions ($\gamma_u$ and $\gamma_i$ for users and items) since product reviews are intended to `explian' opinions the underlying dimensions behind users' decisions.

Roughly speaking, there are two schools of thought as to how reviews should be incorporated to improve recommendation performance. One option is to treat review text as a form of \emph{regularization}, essentially to encourage the low-dimensional representations of users or items (via $\gamma_u$ and $\gamma_i$) to be similar to low-dimensional representations extracted from text. Others seek to extract representations from text that can be used to improve feature-based matrix factorization methods. We give examples of both below.

\paragraph{Hidden Factors as Topics}
%\todo{probably rearrange so that this is a subsection}

An early attempt to incorporate text into recommender systems attempted to do so by making use of \emph{topic models} applied to product reviews \cite{mcauley2013hidden}.
%
Topic models \cite{blei2003latent} learn low-dimensional representations of text (essentially finding structure in the types of bag-of-words representations we covered in \Cref{sec:bagOfWords}). `Topics' then correspond to sets or clusters of words that tend to co-occur together. For example, a topic model trained on movie reviews might dicsover that groups of words associated with `action,' `comedy,' or `romance,' might tend to co-occur together, and therefore that these are distinct topics among movie reviews (we give examples of actual topics in \Cref{fig:?}).
%\todo{Separate topic on dimensionality reduction in text?}

The basic idea in \cite{mcauley2013hidden} is that the low dimensional structure among \emph{reviews} should be related to the low-dimensional structure in \emph{ratings}---after all, reviews are intended to explain why a user rated a product a certain way. Furthermore, even though a single rating can tell us very little about the underlying dimensions that explain a user's ratings, a single review potentially contains enough information to 

The method is somewhat reminiscent of the social recommendation approach we covered in \Cref{sec:social?} \cite{??}, in which a shared parameter $\gamma_u$ was tasked with simultaneously explaining rating dimensions as well as social connections. The argument we made in \Cref{sec:social?} was that in the absence of sufficient rating data, our estimate of $\gamma_u$ will then be estimated from $u$'s friends.

Likewise, \cite{mcauley2013hidden} suggested that a shared parameter $\gamma_u$ could simultaneously explain rating dimensions via a latent factor model, as well as the topics in reviews:
\begin{equation}
    \underbrace{\sum_{(u,i) \in \mathcal T} (\alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i - r_{u,i})^2}_{\text{rating error}} + \lambda \underbrace{\sum_{(u,i) \in \mathcal T} \sum_{w \in d_{u,i}} \log p(w | \gamma_u, \psi)}_{\text{topic likelihood}},
\end{equation}
%\todo{(u,i) \in or u,i \in ?}
%\todo{equation renders poorly}
where $\psi$ is a set of additional (non-shared) parameters specific to the topic model (much like the approach in \Cref{sec:social?} had shared and non-shared social parameters).

Critically, the above model assumes an alignment between latent rating dimensions and review topics. In practice, there may be dimensions in reviews (i.e.,~topics) which are not related to rating dimensions (e.g.~if a user discusses the plot in a book review, it may have little connection to their rating); likewise there may be `intangible' latent dimensions that don't correspond to topics expressed in reviews. By assuming a one-to-one relationship between topics and latent dimensions, the model is useful in cold- (or cool-) start settings by forcing the topic model to discover \emph{those topics that are capable of explaining the variation in ratings}. These topics in particular will be the ones that help us to quickly understand the dimensions that explain user ratings from a few interactions.
%\todo{wordy}

Examples of the types of topics discovered by this model are shown in \Cref{fig:hftTopics}; mostly, the discovered factors correspond to fine-grained product categories, which mostly reflect users' tendency to favor certain types of item over others.

\begin{figure}
    \centering
\begin{tikzpicture}
 %\draw[dashed] (0,0) rectangle (5,4) node[pos=.5] (A) {Dataset $\lbrace X; y \rbrace$};
% \node [below=of A,yshift=0em] {Estimate $\theta$};
\node [style=circle,draw] (gamma1) {$\gamma_i$};
\node [right=of gamma1, circle, fill, inner sep=0.1em] (pt1) {};
\node [style=circle,draw,right=of pt1] (gamma2) {$\gamma_u$};
\node [right=of gamma2, circle, fill, inner sep=0.1em] (pt2) {};
\node [style=circle,draw,right=of pt2] (psi) {$\psi$};
\node [below=of pt1] (out1) {rating};
\node [below=of pt2] (out2) {word frequency};
\draw[-stealth] (pt1) -- (out1);
\draw[-stealth] (pt2) -- (out2);
\draw (gamma1) -- (pt1);
\draw (pt1) -- (gamma2);
\draw (gamma2) -- (pt2);
\draw (pt2) -- (psi);

\node[style=dashed,draw, inner sep=3mm, yshift=0.2em, fit=(gamma1) (gamma2)] (Fit1) {};
\node[style=dashed,draw, inner sep=3mm, yshift=-0.2em, fit=(gamma2) (psi)] (Fit2) {};

\node[above=of Fit1,yshift=-3em] (label1) {ratings};
\node[above=of Fit2,yshift=-2.6em] (label2) {document topics};

\end{tikzpicture}
    \caption{Similar to the social recommendation models from \Cref{sec:?}, personalized models of text often make use of a \emph{shared} parameter that must simultaneously explain structure in interactions and documents.}
    \label{fig:hftIdea}
    %\todo{Reference this somewhere!}
\end{figure}

\paragraph{Other Topic-Modeling Approaches}

The above is a simple approach to combine low-dimensional representations of text (via a topic model) with low-dimensional representations of interactions (via a latent factor model).
%\todo{Do I have the above sentence already somewhere?}
Several others have adopted a similar approach, typically by modifying how user factors $\gamma_u$ and topic dimensions are related to each other.

\cite{ling2014ratings} and \cite{diao2014jointly} both consider the same setting as above, in which reviews are used to improve the performance of rating prediction. Both note the limitations of assuming a simple one-to-one mapping between review topics and user preferences as in \cite{mcauley2013hidden}, and 

\cite{wang2011collaborative} proposed a similar approach to the problem of recommending scientific articles, where document representations are extracted from article text, and user preferences are modeled to predict which articles they will include in their libraries. This differs from the above formulation in that text is associated with items (documents) rather than users, and the setting is essentially an instance of `one-class' recommendation (as in \Cref{sec:oneclassRec}), since one generally doesn't have explicit negative feedback about the articles a user didn't read.



\paragraph{Neural-Network Approaches}

\cite{zheng2017joint} CNN based approach, treat user and item reviews as a document.

\begin{figure}
    \centering
    {
    \small
    \begin{tabular}{cccccccccc}
    % \toprule
    theaters & spas & mexican & vietnamese & snacks & italian & medical & donuts & coffee & seafood\\
    % \midrule
theater & massage & mexican & pho & cupcakes & pizza & dr & donuts & coffee & sushi\\
movie & spa & salsa & vietnamese & cupcake & crust & stadium & donut & starbucks & dish\\
harkins & yoga & tacos & yogurt & hotel & pizzas & dentist & museum & books & restaurant\\
theaters & classes & chicken & brisket & resort & italian & doctor & target & latte & rolls\\
theatre & pedicure & burrito & beer & rooms & bianco & insurance & subs & bowling & server\\
movies & trail & beans & peaks & dog & pizzeria & doctors & sub & lux & shrimp\\
dance & studio & taco & mojo & dogs & wings & dental & dunkin & library & dishes\\
popcorn & gym & burger & shoes & frosting & pasta & appointment & frys & espresso & menu\\
tickets & hike & carne & froyo & bagel & mozzarella & exam & tour & stores & waiter\\
flight & nails & food & zoo & bagels & pepperoni & prescription & bike & gelato & crab
% \bottomrule
    \end{tabular}
    }
    %\includegraphics{}
    \caption[Example of topics that explain variance in rating dimensions on \emph{Yelp}.]
    {
    Example of topics that explain variance in rating dimensions on \emph{Yelp}, from \cite{mcauley2013hidden}.
    }
    \label{fig:hftTopics}
\end{figure}
%\todo{Should be a figure or a table? Depends if I have enough tables for it to be worth having an index for them}
%\todo{Fix table formatting}
%\todo{Move figure to where it is referenced}





% \paragraph{How Useful are Reviews in Practice?}

\begin{floatingbox}
\caption{How Useful are Reviews for Recommendation?}
Although user reviews have been used in several academic studies of text-based recommendation, it is worth thinking about how useful such data would be in practice. After all, in real-world settings the vast majority of interactions would not be associated with reviews. \cite{sachdeva2020useful} studied the settings
\label{box:usefulReviews}
%\todo{referenced anywhere?}
\end{floatingbox}


\section{Question Answering Systems}

\section{Personalized Text Generation}

In \Cref{sec:lstm} we presented Recurrent Neural Networks as general-purpose models that can be used to estimate the next value in a sequence, or to generate sequences, based 

\begin{figure}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
itemc/.style={item,on chain,join}]
%  \begin{scope}[start chain=going right,nodes=itemc,every
%  join/.style={-latex,very thick},local bounding box=chain]
%  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
%  {$A$};
%  \end{scope}
%  \node[left=1em of chain,scale=2] (eq) {$=$};
 \node[item] (AL1) {$A$};
 \node[item, right=of AL1, xshift=-1em] (AL2) {$A$};
 
%  \path (AL.west) ++ (-1em,2em) coordinate (aux);
%  \draw[very thick,-latex,rounded corners] (AL.east) -| ++ (1em,2em) -- (aux) 
%  |- (AL.west);
%  \foreach \X in {0,1,2,t} 
%  {\draw[very thick,-latex] (A\X.north) -- ++ (0,2em)
%  node[above,item,fill=gray!10] (h\X) {$h_\X$};
%  \draw[very thick,latex-] (A\X.south) -- ++ (0,-2em)
%  node[below,item,fill=gray!10] (x\X) {$x_\X$};}
%  \draw[white,line width=0.8ex] (AL.north) -- ++ (0,1.9em);
 \draw[thick,-latex] (AL1.north) -- ++ (0,2em) node[above,item,fill=gray!10] {$h_t$};
 \draw[thick,latex-] (AL1.south) -- ++ (0,-2em) node[below,item,fill=gray!10] {$x_t$};
%  \path (x2) -- (xt) node[midway,scale=2,font=\bfseries] {\dots};
\end{tikzpicture}
\caption{Recurrent neural network for text generation.}
\end{figure}

Several papers have focused on learning language models to generate coherent reviews.
\cite{radford2017learning}

\cite{dong2017learning} (attribute based generation)

\cite{li2017neural} (abstractive tip generation)

\cite{ni2017estimating} (Jianmo paper)

\begin{figure}
\parbox{0.52\textwidth}{12 oz.~bottle, excited to see a new Victory product around, \textbf{A:}
Pours a dark brown, much darker than I thought it would be, rich
creamy head, with light lace. \textbf{S:} Dark cedar/pine nose with some
dark bread/pumpernickel. \textbf{T:} This ale certainly has a lot of malt,
bordering on Barleywine. Molasses, sweet maple with a clear
bitter melon/white grapefruit hop flavour. Not a lot of complexity
in the hops here for me. Booze is noticable. \textbf{M:} Full-bodied,
creamy, resinous, nicely done. \textbf{D:} A good beer, it isn't exactly
what I was expecting. In the end above average, though I found
it monotonous at times, hence the 3. A sipper for sure.}\hspace{0.04\textwidth}\parbox{0.44\textwidth}{\textbf{A:} Pours a very dark brown with a nice finger of tan
head that produces a small bubble and leaves decent
lacing on the glass. \textbf{S:} Smells like a nut brown ale.
It has a slight sweetness and a bit of a woody note
and a little cocoa. The nose is rather malty with some
chocolate and coffee. The taste is strong but not overwhelmingly sweet. The sweetness is overpowering,
but not overwhelming and is a pretty strong bitter finish. \textbf{M:} Medium bodied with a slightly thin feel. \textbf{D:}
A good tasting beer. Not bad.}
\caption{ \label{fig:genConcat}}
\end{figure}
%\todo{add highlights, and maybe select a different review}

\subsection{Text-Based Explanations and Justifications}

(1) Text-based justifications (abstractive)

(2) justifications (extractive, e.g. Diego's paper)

\section{Case Study: Google's \emph{Smart Reply}}

Google's \emph{Smart Reply} is a system developed for \emph{GMail} to automatically recommend short responses for e-mail (\Cref{fig:smartreply}). 
%\todo{shrink figure, convert to black and white}
Given an e-mail thread as input, the system is tasked with surfacing (three) likely responses; although the goal is ultimately to maximize engagement with the feature, the system is trained by taking a large corpus of thread/response pairs, and learning to predict (or maximize the likelihood of) users' historical responses to a given thread.
%\todo{somewhat clunky}

\begin{SCfigure}
\fbox{\includegraphics[scale=0.125]{fig/smartreply.png}}
\caption{Examples of short responses suggested by Google's \emph{Smart Reply}. \label{fig:smartreply}}
\end{SCfigure}

As a case-study in personalized text generation, this system is interesting for several reasons:
\begin{itemize}
    \item As a form of personalized machine learning, Google's solution does not have explicit user parameters (i.e., it is similar to the non-parametric systems described in
    %\todo{where?}).
    Rather, `personalization' is done implicitly by learning from the context already present in the e-mail thread.
    \item Google describes two successive---and quite different---solutions to this problem in a sequence of papers. The first is based on a sequence-to-sequence language model (i.e.,~a text generation framework); the second is a seemingly more trivial nearest-neighbor based solution.
    \item Other interesting facets of the studies include how they deal with scalability, diversification, appropriateness of suggesting a reply for a given context, etc.
\end{itemize}

The models we will discuss are described in \cite{kannan2016smart} and \cite{henderson2017efficient}. We will mainly discuss the latter, more recent paper, as the solution mostly supercedes the former one, though we will make comparisons between both.



\section{Case Study: Personalized Recipes}



\chapter{Online Advertising}

\section{What Makes Ad Recommendation Different?}

Superficially, surfacing advertisements to users seems no different from any other form of personalized recommendation. That is, we can imagine learning user `preferences' and advertisement `properties' from (e.g.)~clicked advertisements in much the same way that any other recommender system is trained.

While ad recommendation does indeed have many similarities to other forms of personalized recommendation, 
%However as we discuss in this chapter, 
there are several properties that demand different solutions compared to what we have seen so far. In particular:
\begin{itemize}
 \item Advertisers have budget constraints. In most recommendation settings we can tolerate considerable imbalance among the items that are recommended (e.g.~a highly popular movie might be recommended to a substantial fraction of all users); this is impossible when recommending ads, given that each advertiser can afford to surface only a limited number of ads (and at the same time, we want to ensure that all advertisers have some ads recommended).
 \item Likewise, each user may only be shown a limited number of ads; while this seems a common enough feature in most recommendation scenarios, it is especially apparent when surfacing ads as users are unlikely to explicitly request additional ads.
 \item Ad recommendations need to be made immediately. Again, this feature is common enough in many recommender systems, but is especially challenging given the considerations above: we cannot find a globally optimal solution that maximizes utility while satisfying advertisers' budget constraints. Instead, we must develop schemes that make local decisions in a way that approximates the globally optimal solution.
 \item Ad recommendation is highly contextual. Whereas most recommender systems rely heavily on user interaction histories, these are presumably less reliable in ad recommendation scenarios (where users' interactions with ads are extremely sparse), if an interaction history is available at all. As such, one has to rely more heavily on user \emph{context} (e.g.~a user's query to a search engine).
 \item Even if a user is responsive to a certain type of ad, there is diminishing value in repeatedly showing similar ads. Instead, we must sometimes recommend ads with low expected utility in the hope of discovering new user interests (this is the basic principle behind an explore/exploit tradeoff).
\end{itemize}

These are the problems we will investigate in order to build systems for online advertising. In Section
%\todo{Finish introduction above}

Note that in this section we will mostly ignore the question of how `compatibility' between a user and an ad is estimated---this could itself be the output of a recommender system, or could simply be a bid that an advertiser places on a user or query.

\section{Matching Problems}

First we will consider the problem of incorporating constraints into (ad) recommendation problems. We want to consider situations where each user can only be shown a fixed number of ads, and each ad can only be shown to a fixed number of users. This is already an oversimplification, as advertisers would usually have a \emph{budget}, and may (for example) bid different amounts per user or query (though we consider these issues in \Cref{sec:?}).

To begin with, we'll consider the case where each ad is shown to exactly one user, and each user is shown exactly one ad. That is, we would like to select a function $\mathit{ad}(u)$ that maps users to ads, such that
\begin{equation}
    \mathit{ad}(u) = \mathit{ad}(v) \rightarrow u = v.
\end{equation}
Alternately, we could write this as an adjacency matrix $A$ such that $A_{u,a} = 1$ if $\mathit{ad}(u) = a$. Then our constraint could be written as
\begin{equation}
    \underbrace{\forall a \sum_u A_{u,a} = 1}_{\text{each advertiser shows one ad}}; \quad \underbrace{\forall u \sum_a A_{u,a} = 1}_{\text{each user sees one ad}}.
\end{equation}
%\todo{realign braces with \clap etc.}
Then, we would like to choose the mapping that maximizes the utility between users and the ads they are shown:
\begin{equation}
    \max_A \sum_{u,a} A_{u,a} f(u,a)
    \label{eq:matchingObj}
\end{equation}
where $f(u,a)$ is a measure of the compatibiliy between a user and an ad.

This type of problem is known as a \emph{matching problem}. Conceptually, it can be viewed as matching two sets of nodes to form a bipartite graph (\Cref{fig:matching}, left), where each possible edge has an associated weight (i.e.,~the compatibility between a user and an ad). Our constraint above now says that every node should be incident on one edge.

\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label=left: 1, label={[label distance=0.7mm]90:users:}] {};
\foreach \i in {2,3,...,6}
  \node[fsnode,on chain] (f\i) [label=left: \i] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label=right: 1, label={[label distance=2mm]90:ads:}] {};
\foreach \i in {2,3,...,6}
  \node[ssnode,on chain] (s\i) [label=right: \i] {};
\end{scope}
%\todo{Hardcoded vertical spacing of users: label}

\draw (s1) -- (f2);
\draw (s2) -- (f5);
\draw (s3) -- (f1);
\draw (s4) -- (f4);
\draw (s5) -- (f6);
\draw (s6) -- (f3);
% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}
\hspace{4mm}
\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label=left: 1, label={[label distance=0.7mm]90:users:}] {};
% \foreach \i in {2,3,...,6}
%   \node[fsnode,on chain] (f\i) [label=left: \i] {};
\node[fsnode,on chain] (f2) [label=left: 2] {};
\node[fsnode,on chain] (f3) [label=left: 3] {};
\node[fsnode,on chain] (f4) [label=left: 4, fill=white] {};
\node[fsnode,on chain] (f5) [label=left: 5, fill=white] {};
\node[fsnode,on chain] (f6) [label=left: 6, fill=white] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label=right: 1, label={[label distance=2mm]90:ads:}] {};
\foreach \i in {2,3,...,6}
  \node[ssnode,on chain] (s\i) [label=right: \i] {};
\end{scope}
%\todo{Hardcoded vertical spacing of users: label}

\draw (s5) -- (f3);
\draw (s1) -- (f2);
\draw (s3) -- (f1);
\end{tikzpicture}
    \caption{Ad recommendation can be viewed as a \emph{bipartite matching problem} (left), where users are shown a fixed number of ads, and each ad is shown to a fixed number of users (in this figure each ad is shown to exactly one user, and vice versa). In an online setting (right), users arrive one at a time; we seek a solution that will be as close as possible to the solution we would have obtained in an offline setting at left.}
    \label{fig:matching}
\end{figure}

Outside of ad recommendation, this type of matching problem appears in many settings, for example it is equivalent to the US's \emph{National Residency Matching Program}, in which medical school students are matched to residency programs: each student can only be matched to a single residency, and each program has a limited number of slots; the matching should be chosen so as to optimize students' preferences for programs (and vice versa). Similar problems appear in various resource allocation settings \cite{gusfield1989stable}; the original paper proposing the solution outlined below considered a setting related to college admissions \cite{gale1962college}.
%\todo{not beautiful writing}

Although \Cref{eq:matchingObj} is a combinatorial optimization problem, it admits efficient approximation (and a polynomial exact solution). The approximation we discuss below is known as \emph{stable marriage}, which draws an analogy between bipartite matching and finding (heterosexual) marriage partners: we seek a solution which, while not necessarily optimal, is `stable,' in the sense that no unmatched pair would have an incentive to `cheat.'

For each man/woman pair $m$ and $w$, we assume a compatibility $f(m,w)$.\footnote{The original solution assumed \emph{different} compatibility functions for men toward women, and women toward men, though we discard this for the application discussed here.} `Stability' then means that there should be no pairs $(m,w)$ and $(m,w')$ such that $m$ and $w'$ are matched but $f(m,w) > f(m,w')$.
%\todo{confusing writing}

and initially assume that there are an equal number of men and women (and that everyone gets married):
%\todo{put this somewhere}

Pseudocode for the stable marriage problem is given below:
\begin{itemize}
    \item All men and all women are initially ‘free’ (i.e.,~not engaged)
    \item while there is a free man $m$, and a woman he has not proposed to
    \begin{itemize}
        \item $w = \max_w f(m,w)$
        \item if ($w$ is free):
        \begin{itemize}
            \item $m$ and $w$ become engaged (and are no longer free)
        \end{itemize}
        \item else ($w$ is engaged to $m'$):
        \begin{itemize}
            \item if $w$ prefers $m$ to $m'$ (i.e.,~$f(m,w) > f(m',w)$):
            \begin{itemize}
                \item $m$ and $w$ become engaged
                \item $m'$ becomes free
            \end{itemize}
        \end{itemize}
    \end{itemize}
\end{itemize}
%\todo{actual pseudocode environment, and above is crappy}
%\todo{algorithm is basically the wikipedia one, but where does it really come from?}

Theorems:
\begin{itemize}
    \item The algorithm eventually terminates; each step includes a new proposal, and there are only $n^2$ possible proposals. Furthermore, each iteration improves the matching score of \Cref{eq:matchingObj} (or does nothing).
    \item The solution is stable. The proof is relatively straihtforward: one assumes an unstable solution in which 
    %$m$ is married to $w'$ but would prefer to be married to $w$; 
    $m$ and $w$ would prefer to be married to each other rather than their current spouses $m'$ and $w'$;
    in this case $m$ must have proposed to $w$ first (since he always proposes to his most compatible match first); so either $w$ was already engaged to somebody preferable, or $w$ accepted and later broke off the engagement; neither of these two scenarios could have ocurred if $m'$ is less compatible.
    \item Note that a consequence of the above is the possibly surprising conclusion that every matching problem as a stable solution.
    \item The solution requires $O(n^2)$ steps, since each step includes a proposal, and there are at most $n^2$ possible proposals. In practice the running time is often substantially faster.
    %\todo{What about sorting the ranked lists for each man?}
    %\todo{Cite for O(nlogn) running time?}
    \item The solution is not guanarteed to be optimal, and the optimal solution is not guaranteed to be stable. Examples of both are given in \Cref{fig:matchingOpt}.
\end{itemize}


\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
%  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=8mm]
%\node[fsnode,on chain,draw,circle] (f1) [label=left: 1, label={[label distance=0.7mm]90:women:}] {};
\node[fsnode,on chain,draw,circle] (f1) [label=left: 1] {};
\node[fsnode,on chain,draw,circle] (f2) [label=left: 2] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=8mm]
%\node[ssnode,on chain,draw,circle] (s1) [label=right: 1, label={[label distance=2mm]90:men:}] {};
\node[ssnode,on chain,draw,circle] (s1) [label=right: 1] {};
\node[ssnode,on chain,draw,circle] (s2) [label=right: 2] {};
\end{scope}
%\todo{Hardcoded vertical spacing of men/women NOT CORRECT: label}

\node[above=of f1] (w1) {women};
\node[above=of s1] (m1) {men};

\draw[dotted,thick] (s1) edge[-] (f1);
% \node [pos=0.66,above=s1] {Transpose};
\draw[thick] (s1) edge[-] (f2);
\draw[thick] (s2) edge[-] (f1);
\draw[dotted,thick] (s2) edge[-] (f2);




% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}

    \caption{Examples of matching solutions that are stable but not optimal (left), and optimal but not stable (right).
    %\todo{figure}
    \label{fig:matchingOpt}
    }

\end{figure}



%tikz:
%https://tex.stackexchange.com/questions/15088/bipartite-graphs

\section{AdWords}

So far, we have discussed the issue of developing recommendation approaches that consider some notion constraints or `budgets' for the perspective of users and advertisers. While incorporating constraints is potentially useful in a variety of recommendation scenarios, our solution still doesn't fully address the setting of \emph{ad} recommendation; in particular, when recommending ads, we are unlikely to see the entire set of users (or queries) in advance, before having to select advertisements.

As such, we desire an algorithm that makes decisions (i.e.,~assigns ads to users) one at a time, while still conforming to matching (or budget) constraints, as in \Cref{fig:advertising} (right).

Formally, the above describes the distinction between an \emph{offline} and an \emph{online} algorithm, i.e.,~one which sees the entire problem in advance, versus one 

\emph{AdWords} \cite{mehta2007adwords} is a specific instance of this type of recommendation problem developed for \emph{Google's} online advertising platform.

Mostly, the setting follows the one we described above, though includes a few additional components. Specifically,
\begin{itemize}
    \item Each advertiser $a$ has a \emph{bid} $f(q,a)$ that they are willing to make for each query $q$.
    \item The bid generally refers to how much the advertiser will pay \emph{if} the ad is clicked on; this is determined by an estimated \emph{click-through-rate}, $\mathit{ctr}(q,a)$. As such the expected profit would be $f(q,a) \times \mathit{ctr}(q,a)$, which one can think of as being analogous to the compatibility from \Cref{sec:sectionAbove}.
    \item Each advertiser has a \emph{budget} $b(a)$ (e.g.~for a one-week period).
    \item As in \Cref{sec:sectionAbove}, there is a limit on the number of ads that can be returned for each query.
\end{itemize}


Of course the actual implementation of AdWords contains many features not described here, for instance Adwords
%\todo{italics}
uses a second-price auction (the winning advertiser pays the amount that the \emph{second} highest bidder bid), and advertisers don't bid on exact queries, but rather are matched using a `broad matching' criteria that can include subsets, supersets, or synonyms of keywords being bid on. We refer to \cite{rajaraman2011mining} for further description of these details, and \cite{mehta2007adwords} for a more detailed technical description.

\section{Bandit Algorithms}

So far, when developing matching algorithms in \Cref{sec:matching?} and AdWords in \Cref{sec:adwords},
%\todo{anything else?}
we've assumed that a compatibility function between between users and ads was available in advance, and merely considered the question of how ads should be \emph{allocated} to users based on budgets and the given compatibility.

However, ad recommendation (among other settings) could present unique challenges in terms of how we estimate compatibility in the first place.

Compared to other recommendation settings, it is unlikely that we could learn user-to-ad compatibility by collecting historical data of ads users have previously interacted with. Some potential issues include:
\begin{itemize}
    \item Item 1
\end{itemize}

%\section{Case Study?}

\subsection{Bandit Algorithms for Recommendation}

A Contextual-Bandit Approach to Personalized News Article Recommendation \cite{li2010contextual}

\section{News Recommendation}

Doesn't exactly go here but might go somewhere?

Turning down the noise

News search engines and the challenge to traditional journalistic
roles



\chapter{Personalized Models of Visual Data}

\section{Visually-Aware Recommendation and Personalized Ranking}
\label{sec:visualRec}

Just as we saw in \Cref{sec:recText} that text can be used to improve both the fidelity and interpretability of recommendations, visual data can likewise be included to improve the accuracy of models in settings where personal preferences are significantly guided by visual signals.

Visual data are critical in domains like fashion (for example). For one, fashion recommendation problems are highly personalized, and problems like compatibility among items depend on complex factors that are hard to precisely define. Presumably, visual attributes play a key role in users' decisions. Second, fashion recommendation scenarios frequently suffer from \emph{cold-start} (or `cool start') problems, given the long tail of new and rarely consumed items.

Both of the above issues suggest systems for recommendation and ranking that make use of visual features. Much as we saw with text in \Cref{sec:?}, visual data is difficult to incorporate directly into recommender systems, given that feature representations are high-dimensional, and dense. Below we discuss a few of the main approaches to incorporate visual data into recommendation and personalized ranking models.

\subsection{Visual Bayesian Personalized Ranking}

Initial attempts to incorporate visual data into ranking models extend the Bayesian Personalized Ranking framework from \Cref{sec:BPR} to incorporate observed image features $f_i$ associated with each item, such as a product image from an e-commerce website.

That is, we want to define a compatibility function $x_{u,i,j} = x_{u,i} - x_{u,j}$ (as in \Cref{eq:BPRCompat}) that estimates which of two items $i$ and $j$ are more compatible with the user.

Starting with a simple latent-factor based compatibility model, and given our goal of building a system that works in (item) cold-start scenarios, we might first consider simply replacing our (latent) item representations $\gamma_i$ with our observed image features, i.e.,
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + \gamma_u \cdot f_i.
    \label{eq:vbprAttempt1}
\end{equation}
In this way, $\gamma_u$ would now determine which features are most compatible with each user (in fact, this is a linear model as in \Cref{chap:basics}).
Although conceptually reasonable, the issue in doing so becomes quickly apparent once we consider that image features are typically very high dimensional. For instance, visual features extracted from ImageNet (as used in \cite{he2015vbpr}) are 4,096 dimensional. Incorporating them directly into $x_{u,i}$ as in \Cref{eq:vbprAttempt1} would thus require fitting thousands of parameters \emph{per user}, which is not feasible in datasets that typically consist of only (e.g.)~tens of interactions per user.

\emph{Visual Bayesian Personalized Ranking} \cite{he2015vbpr} attempts to address this by projecting images into a low-dimensional embedding space via a matrix $\mathbf{E}$. Here $\mathbf{E}$ is $|f_i| \times K$ matrix (e.g.~$4,096 \times K$), 
%\todo{Don't think 4,096 renders right in math mode}
which projects the image into a $K$ dimensional space. Following this, the projected image dimensions can be matched to user preference dimensions:
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + \gamma_u \cdot (\mathbf{E}f_i).
    \label{eq:vbprAttempt2}
\end{equation}
Note that the projected features $\mathbf{E}f_i$ fulfil much the same role as $\gamma_i$ in a typical latent factor model, except that they are learned based on observed features. 

Note that $\mathbf{E}f_i$ is a \emph{learned} embedding, and is fit so as to maximize the probability of observed interactions, as with all other terms in \Cref{eq:vbprAttempt2}. Note also that while $\mathbf{E}$ is high dimensional (e.g.~around 40,000 parameters if $|f_i|=4,096$ and $K=10$), it is a \emph{global} term that is shared among all items; thus for a large enough dataset it accounts for only a small fraction of the model's parameters.

Because the embedding is low-rank, we are assuming that users' preferences toward these visual dimensions can be explained via a small number of factors. While this is similar to the assumption made by a `standard' latent factor model (e.g.~as in \Cref{eq:latentFactor?}), we are further assuming that these factors can be explained by visual dimensions. However in practice there could be several latent factors \emph{not} explainable by visual features (e.g.~factors due to price, material, brand, etc.) To address this the original paper includes \emph{both} latent item factors $\gamma_i$ as well as visual item factors $\mathbf{E}f_i$:
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + 
    \underbrace{\gamma_u(\mathbf{E}f_i)}_{\text{visual preference dimensions}} +
    \underbrace{\gamma'_u \cdot \gamma_i}_{\text{latent preference dimensions}} + \underbrace{\beta^{(f)} \cdot f_i}_{\text{visual bias}}.
    \label{eq:vbpr}
\end{equation}
Correspondingly, there are two sets of user terms: $\gamma_u$, which explains preferences toward visual factors, and $\gamma'_u$, which explains preferences toward non-visual factors. Intuitively, the two terms will play different roles depending on how `cold' an item is: for a cold (or `cool') item, visual features will be much more reliable than latent factors; whereas for 
`hot' items (i.e.,~those with many associated interactions)
%\todo{Have I already defined these terms?}
$\gamma_i$ will be able to capture additional non-visual dimensions. \Cref{eq:vbpr} also includes a `visual bias' term $\beta^{(f)}\cdot f_i$ ($\beta^{(f)}$ is a $|f_i|$-dimensional vector) that is able to estimate item biases in cold scenarios.

Several other considerations must be made to implement such an algorithm efficiently. For instance, accessing the (high-dimensional) image features at random (e.g.~within a stochastic gradient ascent algorithm), leads to poor caching performance; likewise computing the projection $\mathbf{E}f_i$ is expensive. In practice these issues are dealt with by pre-computing all projections $\mathbf{E}f_i$ (which can be performed as a single matrix-matrix product), and updating $\mathbf{E}$ only periodically during gradient ascent.

Visual Bayesian Personalized Ranking is effective in settings where items have few associated interactions. In the original paper it is demonstrated on a clothing dataset from \emph{Amazon}, as well as a clothing trading dataset (\emph{tradesy}).
%\todo{Check spelling of tradesy, consistent italics}
The latter is particularly challenging because traded items are not associated with long transaction histories, meaning that model predictions must largely rely on visual signals.

\subsection{Case Study: Modeling the Visual Evolution of Fashion Trends}

\cite{he2016ups} extended the above ideas from Visual Bayesian Personalized Ranking to incorporate temporal dynamics. Modeling temporal dynamics in this setting is interesting partly because the patterns of temporal variation in (e.g.)~clothing purchases are different from those that were successful in other settings, such as on \emph{Netflix} (\Cref{sec:caseStudyNetflixTemporal}).

The main idea in \cite{he2016ups} is simply to break the training dataset into a sequence of \emph{epochs}, each of which have their own parameters. These epochs are somewhat akin to the `bins' used to model long-term temporal dynamics on Netflix, though a key difference is that the bin sizes are variable and placed at learned intervals (using a dynamic programming procedure); given that the model has a large number of parameters, this helps to ensure that fewer bins are used during time periods with little temporal variation, whereas more (and smaller) bins are used during periods which are more dynamic.

Potentially, models of temporal dynamics in fashion are also interesting because they might reveal historical trends in fashion over time.

% Mention point about projecting images into low dimensional space only once per iteration to save on computational costs

\section{Visual Compatibility of Items}

%Look for refs in WC Pinterest paper

In the previous section
%\todo{Is it the previous section?}
we saw how visually-aware recommender systems can be used to match items (or images of items) to users' preferences. In domains like fashion, there is also interest in establishing compatibility between items (or recommending one item on the basis of another), for example to generate outfits, or even wardrobes of mutually compatible items. In \Cref{sec:neighborhood} we discussed several types of recommendation approaches that considered similarity between items; such measures guide `item-to-item' recommendation approaches, as in \Cref{fig:alsoviewed}.

Here, we would like to develop similar approaches that establish visual similarity (or compatibility) between items. Rather than basing similarity on interaction histories as we did in \Cref{sec:neighborhood}, here we can base similarity directly on the visual appearance of items.

\section{Case Studies: Fashion Compatibility}
%\todo{Decide whether there should be sections with multiple case-studies like this}

Many studies on visual compatibility are specifically concerned with \emph{fashion images}. Estimating compatibility in such a domain has obvious applications
%\todo{finish}

Some of the specific characteristics that makes this problem difficult (and different from other forms of (e.g.)~item-to-item recommendation, are as follows:
\begin{itemize}
    \item It is challenging to construct datasets that act as `groundtruth' for visual compatibility, i.e.,~pairs of items that are known to `go well' together.
    \item Further to the above, any groundtruth of compatible items are bound to be highly noisy, and highly subjective; successful methods need to account for these challenges, and possibly learn compatibility in a personalized way.
    \item The features that make items visually compatible in settings like fashion could be incredible subtle, and could be quite different from the information available in co-purchase data, or even in most visual feature descriptors.
    \item Finally, the notion of `compatibility' is semantically quite different from `similarity.' E.g.~clothing items that go together should be similar in some ways but complementary in others.
\end{itemize}

Approaches to these problems mainly differ in their specific solutions to the problems above. We describe a few key papers below.

\subsection{Estimating Compatibility from Co-purchases}
\label{sec:hengelPaper}

Early approaches to estimating visual compatibility built datasets from co-purchases, for example using publicly-available datasets of reviews from \emph{Amazon}.
%\todo{how to italicize company names etc.}
%\todo{Is amazon data already mentioned above? Only mention it once.}

\cite{mcauley2015image} crawled data from Amazon's surfaced recommendations (`people who bought X also bought Y' etc.~as in \Cref{fig:alsoviewed}), and, in the case of clothing, treated these as `groundtruth' examples of items that are visually compatible.

Having defined such a compatibility function, the goal is to learn an appropriate distance function, such that frequently co-purchased items tend to be closer together than others. The distance function is then used in a simple binary classification framework (similar to logistic regression) to predict:
\begin{equation}
    p(i \text{\ co-purchased with\ } j) = \sigma(c - d(i, j)).
\end{equation}
Previously, we considered how to learn distance functions for problems such as next Point-of-Interest recommendation (\Cref{sec:distancePOI}). When doing so,
%\todo{Check whether above is true of all cases}
items (and users) were projected into a latent space via parameters $\gamma$. To recommend compatible clothing, we might instead use features extracted directly from (the product images of) $i$ and $j$: First, general-purpose visual features are readily available, and are likely to be informative in fashion compatibility scenarios; second, reliance on features is desirable in cold-start settings, which might be common in settings (like fashion) where item vocabularies are large and changing; third, a model based only on visual features can be more straightforwardly \emph{transferred} to settings where user data is not available.

Given image features associated with the items $f_i$ and $f_j$, \cite{mcauley2015image} discusses several strategies for establishing visual similarity. Trivially, one could directly consider the (squared) distance between $f_i$ and $f_j$ (i.e.,~$\| f_i - f_j \|_2^2$), however general-purpose image features may not focus on attributes that are relevant to fashion.

A second solution is to learn a weighted distance function that discovers which features are relevant and discards those that are not, i.e.,~$\sum_k w_k (f_{i,k} - f_{j,k})^2$. However it is argued that in fashion scenarios `compatibility' cannot be captured by modeling similarity between features---for example, a user generally would not select a shirt because it looks `similar' to a pair of pants. To address this, a similarity function is proposed which projects the images into a low-dimensional `style space':
\begin{equation}
    d(i,j) = \| s_i - s_j \|_2^2; \quad \text{where} \quad s_i = M \times f_i.
    \label{eq:hengelPaper1}
\end{equation}
In the case of \cite{mcauley2015image} $f_i$ is a 4096-dimensional image descriptor, extracted from a model trained on ImageNet \cite{jia2014caffe}; $M$ is then a $4096 \times K$ vector, where $K$ is some small embedding dimension (on the order of $K=10$). Ultimately the embedded vector $s_i = M\times f_i$ is analogous to the latent vectors $\gamma_i$ from previous models, in the sense that it captures the underlying dimensions that explain variation in co-purchases.

Finally, although the approach is mainly designed for item-to-item recommendation (and as such is not \emph{personalized}), a personalized version can be developed by adding a user latent vector $\gamma_u$ that encodes which dimensions of this `style space' are important to them:
\begin{equation}
    d_u(i,j) = \sum_k (\gamma_{u,k} s_{ik} - \gamma_{u,k} s_{jk})^2
    %\todo{fix (u,k) vs (uk) everywhere
\end{equation}
(in this case, the model is trained on triples $(u,i,j)$ of co-purchases of items by each user $u$).

%\todo: Note similarity compared to cold-start paper from previous section (both bilinear models)
Pairwise Preference Regression for Cold-start
Recommendation

Ultimately, the paper shows that this type of model can be used in several ways. First, it can predict co-purchases accurately, especially when predictions are personalized. Second, the use of image data is effective at \emph{visualizing} the parameters of the model, i.e.,~determining what are the primary dimensions that explain variance in users' `styles.' Finally, since the model (as in \Cref{eq:hengelPaper1}) takes only images as input, it can be transfered to assess compatibility (and arguably, `fashionability') of outfits outside of the original training data.

%Various other papers have made use of the same co-purchase data, or followed up on the basic idea above. For example, 
\cite{veit2015learning} made use of the same co-purchase data to solve the same task, but did so directly from the `pixel level,' i.e.,~by training a Convolutional Neural Network, rather than using a pre-trained image representation. This type of architecture is depicted in \Cref{fig:Siamese}: two input images (items), labeled as `compatible' or `incompatible,' are passed through two CNNs, both of which share the same parameters. The CNNs learn low-dimensional representations $\phi(x)$ and $\phi(y)$ for the two items; these are essentially equivalent to the `style space' embeddings of \Cref{eq:hengelPaper1}, except that they are learned from the pixel level, and therefore could potentially capture subtle characteristics not available in pre-trained representations. Like \Cref{eq:hengelPaper1}, the model is trained to learn a metric, so that compatible items have nearby embeddings and incompatible items do not. 

\begin{figure}
%tikz code from https://github.com/PetarV-/TikZ/blob/master/Generative%20adversarial%20network/README.md
\centering
\begin{tikzpicture}
[
ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        thick,
        },
]

	\node (z) {item $y$};
	\node[circle, draw, thick, right=5em of z] (x) {$\phi(y)$};
	\draw[-stealth, thick] (z) -- node[above] {CNN} (x);
	%\node[left=of z] (i) {};
	%\draw[-stealth, thick] (i) -- node[above] {$p_\theta(\vec{z})$} (z);
	\node[above=of x, circle, draw, thick] (xt) {$\phi(x)$};
	\node[left=5em of xt] (it) {item $x$};
	\draw[-stealth, thick] (it) -- node[above] {CNN} (xt);
	\node[thick, right=4em of x, yshift=3.1em, xshift=-0.9em] (D) {$\| \phi(x) - \phi(y) \|$};
	\node[right=2em of D] (out) {compatible?};
	\draw[-stealth, thick] (D) -- (out);
			
% 	\node[right=2.5em of x, circle, fill, inner sep=0.15em] (pt1) {};
% 	\node[right=2.5em of xt, circle, fill, inner sep=0.15em] (pt2) {};

			
	\draw[dashed, thick] (x) edge[bend right] (xt);
			
	\node[circle, draw, thick, fill=white, inner sep=0.15em, left=of D] (pt3) {};
			
% 	\draw[-stealth, thick] (x) -- (pt1);
% 	\draw[-stealth, thick] (xt) -- (pt2);
	\draw[-stealth, thick] (pt3) -- (D);

\end{tikzpicture}
\caption{Basic Siamese setup for item-to-item compatibility. \label{fig:Siamese}}
%\todo{edit this for this specific case, improve caption}
%\todo{also add \gamma_u to the input to the generator}
\end{figure}

\cite{he2016learning} showed that prediction performance can be improved in fashion settings by using separate embeddings for the `query' and `target' items $i$ and $j$, i.e.,
\begin{equation}
    d(i,j) = \| \gamma_i - \gamma'_j \|_2^2.
\end{equation}
Critically, by using two different latent spaces, `compatibility' need not follow the assumptions of a metric space, e.g.~an item need not be compatible with itself. In this way the model can learn which aspects should be systematically different when matching items---such as blue pants going with brown shoes.
%\todo{A little too much about my own papers, could shrink}

\subsection{Learning Compatibility from Images in the Wild}
%\todo{finish}

The above papers showed that visual data can be effective when predicting co-purchases in domains like fashion. However, it is arguable whether such models actually learn a useful notion of `fashionability.' For one, a co-purchase is a very noisy indicator of whether two items are compatible: in practice, two purchases on one account may not even be for the same person. Secondly, the images in such datasets (e.g.~product images from Amazon) are not `wild' images (they are usually scaled, centered objects on a white background), so could not be used to assess the fashionability of an outfit in a photo (for example).

\cite{kang2019complete} tried to address this issue by developing models of fashionability that operated directly on images in the wild. At training time, the approach consists of an image (or `scene') that is known to contain a particular item; each item also has a `clean' product image much like those in \Cref{sec:hengelPaper}. The basic idea is then that item must be compatible with other object in the scene.

To build a training set, the known items are cropped from each training image, each resulting in a scene image that does \emph{not} contain the known item, but ought to contain compatible objects. Ultimately, this means that we separately have an item, plus a (cropped) scene that is known to be compatible with the object, but does not contain it. From here, fashion compatibility can be estimated by learning a relationship between the scene and product images:
\begin{equation}
    d(s,p) = \| f(s), f(p) \|_2^2.
\end{equation}
Much like \Cref{eq:hengelPaper1}, this distance function is based on learned embeddings of the scene and product images ($s$ and $p$). Several differences exist between these embeddings and those used in \Cref{sec:hengelPaper}, mostly to account for the fact that the scene image likely contains a large number of \emph{irrelevant} objects. Critically, the method uses an \emph{attention mechanism} (see e.g.~\cite{xu2015show}), which is used to identify regions of the scene image that are relevant for compatibility detection. For example, in an outdoor image, the attention mechanism may learn to focus on the person in the image and their outfit, while ignoring `background' objects in the surrounding scene.

Note that the above is ultimately not a \emph{personalized} model, i.e.,~there are no parameters associated with the individual users. However the system is arguably more useful than that of \Cref{sec:hengelPaper} from a personalization perspective, in the sense that it would allow a user to upload an image of themselves and receive recommendations that complement their personal style. As such, it is an example of the type of \emph{contextual} or non-parametric personalization that we discussed in \Cref{sec:nonparametric}.

In addition to experiments on personalized fashion, \cite{kang2019complete} shows that the same technique can be used for complementary item recommendation in other settings, such as recommending compatible furniture based on items in a living room.

%\todo{Finish, also mention the use for furniture}

%\todo{Mention that other ruining paper that learned non-metric embeddings?}

\subsection{Generating Fashionable Wardrobes}

The papers above consider \emph{pairwise} compatibility among items as a proxy for selecting sets of items that will form compatible outfits.

\cite{hsiao2018creating} considers the more challenging problem of finding \emph{collections} of items that can be used to generate compatible outfits (called `capsule wardrobes').

\subsection{Domains other than Fashion}

Learning visual similarity for product design with convolutional neural networks
\cite{kang2019complete} (furniture)

Kang paper (art) - also mention that there is a social aspect to art recommendation.
%\todo{This}

\section{Personalized Generative Models of Images}

In \Cref{chap:text}, we examined personalized text models from two directions: first, we used text within \emph{predictive} tasks, e.g.~we saw how text can be used for regression problems (\Cref{sec:bagOfWords}), and to improve the performance of recommender systems (\Cref{sec:textBasedRecommenders}). Second, we saw how to personalize \emph{generative} models of text (\Cref{sec:personalizedGenerativeText}), i.e.,~to generate text that matches a user's writing style or preferences.

Likewise, our discussion of visual data has so far considered using images to improve predictive performance; it is worth spending a little time exploring how to personalize \emph{generative} models of images.

Briefly, the basic framework we will consider extending is that of the \emph{Generative Adversarial Network}, or GAN.

%\todo{taken from my own paper, maybe change text a bit more}
Generative Adversarial Networks (GANs) are an unsupervised learning framework in which two components `compete' to generate realistic looking outputs (in particular, images) \cite{goodfellow2014generative}. One component (a generator) is trained to generate images, while another (a discriminator) is trained to
distinguish real versus generated images. Thus the generated
images are trained to look `realistic' in the sense that they are
indistinguishable from those in the dataset.

%\todo{same with this paragraph}
Such systems can also be conditioned on additional inputs, in order to sample outputs with certain characteristics \cite{mirza2014conditional}. 

\begin{figure}
%tikz code from https://github.com/PetarV-/TikZ/blob/master/Generative%20adversarial%20network/README.md
\centering
\begin{tikzpicture}
[
ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        thick,
        },
]

	\node[circle, draw, thick] (z) {$\vec{z}$};
	\node[circle, draw, thick, right=5em of z] (x) {$\vec{x}_{\mathit{gen}}$};
	\draw[-stealth, thick] (z) -- node[above] {$G(\vec{z})$} node[below] {generator} (x);
	\node[left=of z] (i) {};
	\draw[-stealth, thick] (i) -- node[above] {$p_\theta(\vec{z})$} (z);
	\node[above=of x, circle, draw, thick] (xt) {$\vec{x}_{\mathit{real}}$};
	\node[left=5em of xt] (it) {};
	\draw[-stealth, thick] (it) -- node[above] {$p_{\mathit{data}}(\vec{x})$} (xt);
	\node[circle, draw, thick, right=5em of x, yshift=2.5em] (D) {$\vec{x}$};
	\node[right=7em of D] (out) {real?};
	\draw[-stealth, thick] (D) -- node[above] {$D(\vec{x})$} node[below] {discriminator} (out);
	
	\node[below=of i, yshift=1em] (dotted1) {};
	\node[below=of out, right=of dotted1, xshift=25em] (dotted2) {};
			
	\node[right=2.5em of x, circle, fill, inner sep=0.15em] (pt1) {};
	\node[right=2.5em of xt, circle, fill, inner sep=0.15em] (pt2) {};
	
	\draw[dashed] (dotted1) -- (dotted2);

	\draw[dashed, thick] (pt1) edge[bend left] (pt2);
			
	\node[circle, draw, thick, fill=white, inner sep=0.15em] at ([xshift=-0.85em, yshift=3.9em]pt1.north) (pt3) {};
			
	\draw[-stealth, thick] (x) -- (pt1);
	\draw[-stealth, thick] (xt) -- (pt2);
	\draw[-stealth, thick] (pt3) -- (D);
	
	%\node[below=of pt1, circle, fill, inner sep=0.15em] (pt3) {};
	\node[below=6em of D, circle, draw, thick] (pt3) {};
	\node[right=0.5em of pt3, circle, fill, inner sep=0.15em] (pt4) {};
	\node[right=0.5em of pt4, circle, draw, thick] (pt5) {};
	
	\draw[-, ArrowC2] (x) |- node[above, xshift=3em] {hello} node[below, xshift=3em] {$\Phi(x_{gen}$} (pt3);
	\node[right=0.1em of pt5] (gamma) {$\gamma_u$};
	\node[below=of pt4] (out2) {compatibility};
	\draw[-stealth, thick] (pt4) -- (out2);
	
	%\draw[-stealth, thick] (pt4) -- (out2);
%	\draw[-stealth, thick] (z) -- node[above] {$G(\vec{z})$} node[below] {generator} (x);

\end{tikzpicture}
\caption{Basic setup of a Generative Adversarial Network (GAN), and a personalized GAN. The components above the dotted line depict the `standard' GAN setup, in which a generator `competes' with a descriminator to generate images that are indistinguishable from real data. Components below the dotted line are used to develop a `personalized' GAN that generates images compatible with a particular user. \label{fig:GANdiagram}}
%\todo{edit this for this specific case, improve caption}
%\todo{also add \gamma_u to the input to the generator}
\end{figure}

%We follow a similar approach based on activation maximization [11] to generate images that best match a selected user's personal style. GANs have been used for broad applications, though to our knowledge we are the first to apply this architecture to fashion image generation

The basic setup of a Generative Adversarial Network 
%(GAN) 
is depicted in \Cref{fig:GANdiagram}. 

\cite{kang2017visually}


\section{Personalized Image Search and Retrieval}
%\todo{Put this section before or after recommendation?}

%Before visiting visual data in the context of recommendation, it is worth briefly considering how visual data is handled in `traditional' settings like image search.
Having studied the use of visual data in the context of recommendation, it is worth briefly considering how visual data is handled in `traditional' settings like image search and retrieval.


Paper: Image ranking based on user browsing behavior
% http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.707.502&rep=rep1&type=pdf

\cite{trevisiol2012image}


%Paper: Click-through-based cross-view learning for image search

A basic concept in image retrieval settings, as we saw in \cite{sec:SameThingButWithText}, is that of learning a joint embedding space between a query $q$ and an image $i$:
\begin{equation}
%Sec 3.2 of that paper
    d(q,i) = \| g(q) - g(i) \|_2^2
    \label{eq:imageRetrieval1}
\end{equation}
For example, in \cite{pan2014click}, $f(q)$ and $f(i)$ were based on simple linear embeddings of (textual) query features $f_q$ and (visual) image features $f_i$:
\begin{equation}
    g(q) = f_q W^{(\text{query})}; \quad g(i) = f_i W^{(\text{image})}.
\end{equation}
Note that this setting is very similar to that of \Cref{sec:hengelPaper}, in which a query image is linearly projected into `style space' (\Cref{eq:hengelPaper1}) so that neighboring images can be retrieved. This type of image retrieval system operates on the same principle, except that the query features are extracted from text (as we studied in \Cref{sec:textQueryFeatures}).

Finally, $W^{(\text{query})}$ and $W^{(\text{image})}$ are trained so that distances in \Cref{eq:imageRetrieval1} are minimized based on click-through data, again following a learning approach similar to that from \Cref{sec:hengelPaper}. That is, distances should be small between query/image pairs associated with a large number of clicks.\footnote{Note that this approach assumes that query and click data is available at training time, presumably from a method \emph{not} based on visual embeddings.}





Paper: Learning to personalize trending image search suggestion

Paper: Prism: Concept-preserving social image search results
summarization (maybe not...)


\section{History and Related Work}

Personalized recommendation problems involving visual data have been studied for several years. Initial attempts ignore visual data altogether, and long predate solutions that model images explicitly. For example, an early system for clothing recommendation \cite{hu2014style} learns a user's `style' in order to recommend clothing, but does so using `likes' rather than any analysis of visual features. Likewise, \emph{YouTube's} early recommendation approaches \cite{davidson2010youtube} are based on heuristic `relatedness scores' based on co-visitation (essentially a form of neighborhood-based approach, as in \Cref{sec:neighborhood}), though some features based on video metadata are included in the model; newer solutions (based on deep learning) adopt more complicated candidate generation and ranking strategies, though again make little if any use of explicit visual signals \cite{covington2016deep}.



\chapter{The Ethics of Personalized Machine Learning}

\section{How Can Personalized Machine Learning be Improved?}



\subsection{Filter Bubbles and Extremification}

\cite{pariser2011filter} (coins the term filter bubble)
%see https://www2.kbs.uni-hannover.de/fileadmin/institut/pdf/webscience/2016-17/papers/nej1.pdf
% for reading list on filter bubble background

The idea that personalized recommendation traps users in `filter bubbles,' amplifies existing biases toward popular items, or guide users toward extreme content, are often reported in popular media and discussed anecdotally, though such concepts are often not precisely defined. It is also difficult to measure these types of dynamics empirically, as one rarely has the ability to analyze the counterfactual scenario in which the recommender wasn't in place.
%\todo{Ugly opening paragraph}

%Paper: Recommender Systems and their Impact on Sales Diversity

\paragraph{Exploring Diversity through Simulation}

An early paper that attempted to define and analyze the impact that recommender systems have on interaction diversity did so via simulation \cite{fleder2009blockbuster}. They noted the existence of two competing hypotheses as to why recommender systems might encourage or discourage diversity: on the one hand, recommender systems can guide content \emph{discovery}, which can increase the diversity of item interactions \cite{brynjolfsson2006niches}; on the other hand, recommender systems might reinforce the popularity of already-popular products, thus reducing diversity \cite{mooney2000content}. Their attempt to resolve this question built a simple simulation which generates recommendations, 
in which the probability of an item being recommended, or the probability of a user accepting it, can be controlled.
%which simulated users have a controlled probability of accepting. 
%By controlling the acceptance probability (or other properties of the simulation), they can simulate recommendations with varying levels of influence.
By varying the controllable parameters, they show that under almost all conditions (i.e.,~except in edge cases), the recommender system leads to a concentration effect (i.e.,~results in a reduction in interaction diversity), as measured by the Gini coefficient (a measure of the statistical disbursion of item interaction frequencies).

Of course, the above is not necessarily true of \emph{every} recommender system; indeed as we'll see in \Cref{sec:?}, one can design a recommender system so as to explicitly target the diversity of recommended items.
Rather, the above result is simply a demonstration that under fairly minimal conditions, recommender systems \emph{can} lead to a concentration effect.

\paragraph{Directly Measuring Algorithmic Recommendation and Diversity}
%\todo{Crappy section title}

Following \cite{fleder2009blockbuster}, which studied the possibility of filter bubbles via simulation, \cite{nguyen2014exploring}
was the first to empirically measure the effect of recommender systems on content diversity in a real setting.

The research questions in \cite{fleder2009blockbuster} are similar to the ones studied via the simulations above, namely: do recommender systems gradually expose users to narrower content over time; and, how does this effect vary as a function of how receptive users are to recommendations.
%\todo{finish, tag genome, etc....}

\paragraph{Exploring Diversity through \emph{YouTube} Recommendations}
%\todo{Dunno about these paragraph or overall organization}

%
%Paper: The impact of YouTube recommendation system on video views .
%
Following this work, \cite{zhou2010impact} conducted an empirical study of recommendations on \emph{YouTube},
%\todo{Rewrite the above sentence, since I've reorganized}
%
%For example, an early paper studying \emph{YouTube} recommendations argued that \emph{YouTube's}
%\todo{italics?}
and argued that \emph{YouTube's} recommendations (specifically the `related videos' feature) have a positive impact on content diversity.
They showed that recommendations drive a large fraction of views on youtube,
%\todo{captilaitaion and italicize}
and that views driven by recommendations have higher diversity than views from a popularity-driven system (however, it is hard to argue that recommendation-driven views are therefore `diverse,' given that popularity-driven views would presumably lead to high concentration, and as such are arguably a weak baseline).
%\todo{not beautifully worded}

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/recPointsInner.pdf}
    \includegraphics[scale=0.75]{fig/recPointsNN.pdf}
    \caption{Recommendations selected for a user by maximizing an inner product (left) or taking nearest neighbors (right). \label{fig:InnerNN}}
\end{figure}
%\todo{nowhere to put this}



\paragraph{Diversity and Extremification}

However, even if consumption is `diverse,' as defined by the overall distribution of interactions across an item vocabulary, this does not guarantee that users are necessarily exposed to diverse viewpoints, or are not pushed toward extreme content.
%Let's consider the trivial motivating example in \Cref{fig:InnerNN}
%\todo{Could introduce it here}
For example, returning
%\todo{is it cited before?}
to the motivating example from \Cref{fig:InnerNN}, recommendations based on maximizing an inner product (left) would expose users to more `diverse' content than the nearest neighbor model (right), but may gradually drive users toward `fringe' items that are aligned with, but are more extreme versions of, the content they initially consumed.

\cite{ribeiro2020auditing} attempted to empirically analyze the pathways via which users arrive at extreme content on \emph{YouTube}. 
%\todo{Finish}
The authors used curated lists of channels (for their study, of alt-right political channels) in order to establish a ground-truth of what is meant by `extreme' content; they also collected less extreme content (`alt-lite,' general media, etc.)

%Auditing Radicalization Pathways on YouTube (https://arxiv.org/pdf/1908.08313.pdf); separate section on extremification?

\section{Diversification Techniques} % Filter bubble

% https://www.sciencedirect.com/science/article/abs/pii/S0950705117300680



one can imagine how recommendations based on nearest neighbors or maximum inner-products, as in \Cref{fig:InnerNN}, might focus exclusively on a user's predominant interest alone.


Bias Disparity in Recommendation Systems

Diversifying Search Results (Agarwal)


%\todo{Move diversity to recommender systems chapter?}

\subsection{Maximal Marginal Relevance}
\label{sec:MMR}
%\todo{I think I found this from a DPP paper, probably the youtube one. Maybe check the argument they make relating this to other ideas}

A simple notion of diversity that is commonly used in document retrieval scenarios is that among a (ranked) list of retrieved documents, each retrieved item should simultaneously be relevant, but at the same time not too similar to the already-returned items.

This notion is captured by the \emph{Maximal Marginal Relevance} \cite{carbonell1998use} procedure (MMR). The approach was originally designed for retrieving sets of text passages that best summarize a document (with respect to some query): each document should be similar to the query, but also dissimilar from the documents already retrieved.

The same concept can straightforwardly be applied in recommendation scenarios, given that we have notions of both \emph{relevance} and \emph{similarity} available (e.g.~relevance might be the output of a latent factor model, while similarity could be defined in terms of cosine similarity, or as an inner product between item representations $\gamma_i$ and $\gamma_j$ for two products).

To apply the concept to recommendation, we would define the Maximal Marginal Relevance as follows:
\begin{equation}
    \mathit{MMR} = \argmax_{i \in R \setminus S} \Bigl[ \lambda\underbrace{\mathit{Sim}^{\text{user}}(i,u)}_{\mathclap{\text{relevance to the user}}} - (1 - \lambda)\overbrace{\max_{j \in S}\mathit{Sim}^{\text{item}}(i,j)}^{\mathclap{\text{similarity to already-recommended items}}}\Bigr],
\end{equation}
where $R$ is an initial candidate set of recommendations (most trivially e.g.~a list of items the user hasn't already interacted with), and $S$ is a set of items retrieved so far. $\mathit{Sim}^{\text{user}}$ and $\mathit{Sim}^{\text{item}}$ are item-to-user and item-to-item similarity functions (respectively); the former is presumably the compatibility function returned by a recommender system,
%\todo{repetitive writing compared to above paragraph}
the latter is any item-to-item similarity measure (e.g.~cosine similarity or inner product between item representations).

Note that the above is computed iteratively, that is we add one result at a time by maximizing the MMR until the list $S$ has the desired size. Finally, $\lambda$ trades off the extent to which we care about compatibility versus diversity.

\subsection{Re-ranking Approaches to Diverse Recommendation}

%Improving aggregate recommendation diversity using ranking-based techniques \cite{adomavicius2011improving}

Similar to Maximal Marginal Relevance, several approaches have been specifically designed for re-ranking in recommendation scenarios. \emph{Re-ranking} approaches, including MMR, assume that we are given an initial ranking function which we trust to find items of high relevance, but which lack diversity; thus one seeks a re-ranking procedure that balances the two concerns.

One such approach for re-ranking for recommendation was proposed in \cite{adomavicius2011improving}. The approach assumes the presence of three components: First, they assume the availability of a compatibility score, e.g.~a rating prediction $r(u,i)$. Second, a \emph{relevance-oriented} ranking technique, $\mathit{rank}_u(i)$; this could be any ranking function, though most trivially one might simply order predictions by a predicted rating score $r(u,i)$. And third, a second `diversity oriented' ranking function; conceptually, this should focus on recommending items to users that they would \emph{not} normally consider.

An example of a diversity-oriented loss from \cite{adomavicius2011improving} is to sort items by popularity, with the \emph{least} popular items being ranked first:
\begin{equation}
    \mathit{rank}^{(\text{pop})} = | U_i |.
\end{equation}
Recommending \emph{unpopular} items does not at first appear to be a particularly effective recommendation strategy; however this ranking is used in conjunction with the prediction score $r(u,i)$. Specifically, to encourage diversity we want to find \emph{unpopular} items that \emph{this} user is likely to enjoy. Spiritually, this is somewhat reminiscent of our \emph{TF-IDF} approach to finding important words in documents (\Cref{sec:tfidf}).
%; here `words' correspond to items, and documents (roughly) correspond to the set of items this user 

The specific (re-)ranking objective from \cite{adomavicius2011improving} then looks like
\begin{equation}
    \mathit{rank}'_u(i, t) = \left\lbrace
    \begin{tabular}{ll}
    $\mathit{rank}^{(\text{pop})}(i)$ & if $r(u,i) \geq t$\\
    $\alpha_u + \mathit{rank}_u(i)$ & otherwise
    \end{tabular}
    \right..
\end{equation}
Here $t$ is a threshold term, essentially determining whether one of the low-popularity recommendations has a high enough score to recommend; $\alpha_u$ is an offset term ensuring that the popularity-based recommendations appear first in the ranking before those of $\mathit{rank}_u(i)$.

\cite{adomavicius2011improving} shows that as the threshold $t$ is changed, the system gradually trades-off between recommendation precision and diversity. 
Several different ranking functions are considered, for example to replace popularity-based ranking by alternatives based on the average rating, rating variance, etc.

They also note that the type of diversity achieved by this ranking mechanism is quite different from that in \Cref{sec:MMR}, as it does nothing to encourage diversity (or dissimilarity) among an individual user's item list. Instead, they discuss the related notion of \emph{aggregate diversity}, which defines diversity across the item vocabulary itself, i.e.,~recommendations across \emph{all} users should have reasonable coverage of the complete item vocabularity. This is related to the notion of $P$-fairness
%\todo{Is this C or P? I think P is right}
that we'll discuss in \Cref{sec:multisided}.

\subsection{Topic Diversification}

Improving recommendation lists through topic diversification \cite{ziegler2005improving}

Avoiding Monotony: Improving the Diversity of Recommendation Lists \cite{zhang2008avoiding}
%can't find pdf

Category-Driven Approach for Local Related Business
Recommendations
%https://dl.acm.org/doi/pdf/10.1145/2806416.2806495

\subsection{Novelty, Discovery, and Diversity}
%\todo{Don't like section title, as diversity is covered in other subsections above and below}

\cite{vargas2011rank} and \cite{vargas2011new} studied the relationship between novelty and diversity when making recommendations, and presented several metrics to evaluate existing systems in terms of such characteristics.

%\todo{Maybe. Somewhat too complex. Probably couple of pages to write}
% Rank and Relevance
% in Novelty and Diversity Metrics for Recommender Systems \cite{vargas2011rank}

% New Approaches to Diversity and Novelty in
% Recommender Systems \cite{vargas2011new}

\subsection{Determinantal Point Processes}
%Youtube papers

In \Cref{sec:MMR} and \Cref{sec:ImprovingRecommendationListsPaper}
%\todo{Others}
we discussed various approaches that attempt to balance accuracy and diversity via what are essentially `heuristic' strategies that greedily select items that maximize utility while being sufficiently novel compared to the rest.
%\todo{Is the above a fair description?}
%\todo{Needs gluing sentence}

Determinantal Point Processes (or DPPs) \cite{kulesza2012determinantal} are a \emph{set-based} optimization technique that can be used to identify subsets of items that simultaneously maximize item quality and diversity among items. Specifically, given a set of items $I$, a DPP assigns a probability $P(S)$ to every subset $S \in \mathbb{P}(I)$, where $\mathcal{P}(I)$ is the \emph{powerset} (i.e.,~the set of all subsets) of $I$. The goal is then to model (i.e.,~parameterize) this probability, either globally or for individual users, such that finding the subset of items which maximizes $P(S)$ has an optimal tradeoff between utility and diversity.

\cite{wilhelm2018practical} studied the application of DPPs to diversify recommendations on \emph{YouTube}.
%\todo{italics, caps}

The method assumes a few inputs. First, as with previous diversification techniques, we assume a utility or `quality' estimate $f(u,i)$ is given (e.g.~from a pre-trained recommender system), which encodes the probabilty that user $u$ will interact with item $i$ given $i$'s features; we also assume a predefined distance function $d(i,j)$ between two items.

Next, we have historical sets of items that have been surfaced to the user (i.e.,~the outputs of an existing system), along with subsets of items that the user selected (i.e.,~binary labels indicated by $y_{u,i}$). The goal is to select subsets of items that will maximize the total number of interactions, which in practice is trained by maximizing the Cumulative Cain:
\begin{equation}
    \sum_{u}\sum_{i} \frac{y_{u,i}}{\mathit{rank}(u,i)},
\end{equation}
where $\mathit{rank}(u,i)$ is the new rank assigned by the proposed algorithm. That is, the items that the user interacted with ($y_{u,i} = 1$) should have high rank.
%\todo{Cover Cumulative Gain in recommender systems evaluation or something}

Note that the above seems reminiscent of `traditional' approaches to recommendation, i.e.,~we are ranking items such that positive interactions should have high rank (which seems similar to what we saw in \Cref{sec:CumGanEval}). The main difference here is simply the observation that the total number of interactions will be maximized when utility and diversity are balanced (e.g.~a user will quickly become bored if recommendations cover only one of their interests).

Next, given a candidate set of $N$ items, we next define a matrix $L^{(u)}$ such that diaginal entries $L^{(u)}_{i,i}$ encode the utility of an item $i$ and off-diagonal entries $L^{(u)}_{i,j}$ encode the similarity of two items $i$ and $j$. The specific parameterization used in \cite{wilhelm2018practical} is:
\begin{eqnarray}
  L^{(u)}_{i,i} & = & f(u,i)^2\\
  L^{(u)}_{i,j} & = & \alpha f(u,i) f(u_j) \exp(- \frac{d(i,j)}{2\sigma^2}) \quad \text{for $i \neq j$}.
  \label{eq:dppParams}
\end{eqnarray}

Now, the quality of a subset $S$ is proportional to the determinant of the submatrix of $L$ induced by $S$, $\det(L_S)$. Specifically:
\begin{equation}
    P(S) = \frac{\det(L_S)}{\sum_{S' \in \mathbb{P}(I)} \det(L_{S'})}.
    \label{eq:dpp1}
\end{equation}
%\todo{Could rewrite everything with subset notation rather than element of powerset}
Critically, the denominator of the above equation can be computed efficiently as
\begin{equation}
    \sum_{S' \in \mathbb{P}(I)} \det(L_{S'}) = \det(L + I),
\end{equation}
where $I$ is the identity matrix.

To understand (roughly) why the determinant is diversifying, it helps to consider the trivial example where $S$ consists of only two items $i$ and $j$; then the determinant is given by
\begin{equation}
    \det \left( \left[ \begin{tabular}{cc}
      $L_{i,i}$ & $L_{i,j}$\\
      $L_{j,i}$ & $L_{j,j}$
    \end{tabular} \right] \right) = L_{i,i}L_{j,j} - L_{i,j}L_{j,i};
\end{equation}
this value will be maximized when the utility is high ($L_{i,i}L_{j,j}$) and the similarity is low ($L_{i,j}L_{j,i}$).

In spite of the relatively simple form of \Cref{eq:dpp1}, it is still not practical to solve the (NP-hard) problem of finding the optimal subset. In \cite{wilhelm2018practical} this is addressed by a simple greedy algorithm (similar to that of \Cref{sec:MRR}, among others), in which one starts with the empty set of videos $S = \emptyset$, and iteratively adds the item $i$ which maximizes the determinant $\det(L_{S \cup \lbrace i \rbrace})$

Note that the parameterization in \Cref{eq:dppParams} includes two parameters, $\alpha$ and $\sigma$, which in practice are tuned using grid search. Intuitively these parameters control the relative weight of utility versus diversity ($\alpha$), and the `tightness' of the similarity function ($\sigma$). These parameters are global, but in principle could be learned per-user.
%\todo_{Have I defined grid search anywhere?}

Ultimately, the experiments find that implementing the above DPP on a user's video feed increases user satisfaction (as measured by session duration), compared to various other diversification strategies.

%Practical Diversified Recommendations on YouTube with Determinantal Point Processes


\subsection{Calibration}

A related notion to diversity is that of \emph{calibration} of predictions or recommentations. Whereas a diversity metric might suggest (for example) that we should expose users to a wide distribution of recommendations, which potentially span beyond their explicit preferences, \emph{calibration} refers to the idea that recommendations should be made \emph{in proportion} to expressed preferences. For instance, if a user watches 40\% sci-fi movies and 60\% romantic comedies, they should not exclusively be recommended romantic comedies, as might happen when na\"ively recommending nearest neighbors or maximizing inner products.

\cite{steck2018calibrated} introduces such a notion of \emph{calibrated recommendations}, in the context of movie recommendations on \emph{Netflix}. Their work discusses both metrics to assess calibration, as well as methods to calibrate the outputs of an existing recommender system.

Their notion of calibration operates over a pre-defined set of item \emph{genres}, described using a stochastic genre vector $p(g|i)$ (e.g.~a movie might be categorized as 80\% `action' and 20\% `sci-fi'); this could potentially be adapted to other attributes toward which one desired calibration. The basic idea behind a calibration metric is then that the distribution of genres among a 
user's history $\i \in \mathcal H$
should match the distribution of \emph{recommended} items $i \in \mathcal I$. 
The two terms are defined (respectively) as:
\begin{eqnarray}
p(g|u) & = \frac{\sum_{i \in \mathcal H} w_{u,i} \cdot p(g|i)}{\sum_{i \in \mathcal H} w_{u,i}}\\
q(g|u) & = \frac{\sum_{i \in \mathcal I} w_{r(i)} \cdot p(g|i)}{\sum_{i \in \mathcal H} w_{r(i)}}.
\end{eqnarray}
Both expressions include a `weighting' term $w$. In the case of the historical distribution $w_{u,i}$ might weight items according to recency (for example), or for the case of recommendations $w_{r(i)}$ might weight recommendations according to their position in a list (i.e.,~their ranking); either term could also be ignored.

Now, the goal is to generate a set of recommended items $\mathcal I$ such that the two distributions should match closely. The difference between the two distributions can be measured by (e.g.)~the Kullback Liebler divergence:
\begin{equation}
    \text{KL}(p,q) = \sum_{g} p(g|u)\log \frac{p(g|u)}{q(g|u)}.
\end{equation}
Of course, the recommendations should also be highly compatible according to the recommender system itself. This is achieved in \cite{steck2018calibrated} with a simple expression that trades off recommendation utility and calibration (via a trade-off hyperparameter $\lambda$):
\begin{equation}
  \mathcal I = \argmax_{\mathcal I} \underbrace{(1 - \lambda) \cdot \sum_{i \in I} s(i)}_{\text{compatibility}} - \underbrace{\lambda \cdot \text{KL}(p, q(\mathcal I))}_{\text{calibration}}
  \label{eq:calibrated}
\end{equation}
\cite{steck2018calibrated} notes that the above is a hard combinatorial optimization problem, but can be approximated greedily (with a certain optimality guarantee), by iteratively adding one item at a time to $\mathcal I$ so as to optimize the above criterion until the desired number of items is reached.

An appealing property of this approach is that it can be applied in a purely post-hoc fashion to the outputs of \emph{any} recommender system that associates scores between users and items. The experiments in \cite{steck2018calibrated} show (by varying $\lambda$ in \Cref{eq:calibrated}) that a reasonable degree of calibration can be achieved with minimal loss in recommendation utility.

%Calibrated recommendations: https://dl.acm.org/doi/10.1145/3240323.3240372



%\todo{Figure needs to go elsewhere}

% \section{Clickbait, Fake News, and Review Fraud}

%  Review fraud:
% BIRDNEST: Bayesian Inference for Ratings-Fraud Detection

% \section{Optimizing Short-Term versus Long-Trem Goals}

\section{Case Studies on Diversity}

\subsection{Diversity on Spotify}

In \Cref{sec:?}, we saw an empirical study of diversity on \emph{YouTube}, which argued that recommender systems led to diverse views, though this analysis was limited in that the point of comparison was a popularity-based alternative (which is almost certainly not diversity-inducing). \cite{anderson2020algorithmic} tries to study the effect that recommendation algorithms have on diversity more directly.

The paper considers the listening patterns of around 100 million users on \emph{Spotify}. Unlike (e.g.)~\cite{fleder2009blockbuster}, which defined `diversity' in terms of the Gini coefficient (i.e.,~the statistical dispersion of \emph{which} items get consumed), \cite{anderson2020algorithmic} defines diversity in terms of song representations, i.e.,~essentially the $\gamma_i$ values in a recommender system.\footnote{Of course, this version of `diversity' has its own limitations, as it assumes that the learned latent space accurately captures the diversity among items.
%\todo{Is this limitation addressed in the paper?}
}

The specific embeddings $\gamma_i$ on \emph{Spotify} are estimated using an \emph{item2vec}-like method (as we saw in \Cref{sec:item2Vec}). The \emph{musical diversity} of a user $u$'s listening activity is defined in terms of a score which they term \emph{generalist-specialist} (or $\mathit{GS}$), following previous work \cite{waller2019generalists}. Specifically, we start by defining the centroid of a user's listening history (which we'll term $\gamma_u$) as
\begin{equation}
    \gamma_u = \frac{1}{|H|} \sum_{j = 1}^{|H|} \gamma_{H_j},
\end{equation}
where $H$ is a list of songs in the user's \emph{listening history}, with repetition, such that repeated listens will count more to the average. Then the $\mathit{GS}$-score is defined as the average cosine similarity between the user representation $\gamma_u$ and the items they listen to:
\begin{equation}
    \mathit{GS}(u) = \frac{1}{|H|} \sum_{j = 1}^{|H|} \frac{\gamma_{H_j}\cdot \gamma_u}{\| \gamma_{H_j} \| \| \gamma_u \|}.
\end{equation}
Intuitively, \emph{specialists} (high $\mathit{GS}(u)$) tend to have songs $\gamma_i$ in their listening history oriented primarily in a certain direction;  \emph{generalists} (low $\mathit{GS}(u)$) do not, ostensibly corresponding to a broader range of prefrences.
%has a broad range of preferences, such that songs $\gamma_i$ in their history do not tend to `point' in any specific direction; 

Part of the analysis in \cite{anderson2020algorithmic} is a study of the relationship between diversity (as measured by $\mathit{GS}(u)$) and various other attributes. For example, less active users tend to be specialists (high $\mathit{GS}(u)$), generalist users are less likely to abandon the system (`churn'), and more likely to subscribe to subscribe to the `premium' version of the product.

However the main feature in the analysis is 
%the effect that \emph{recommendations} have on diversity. 
to study the relationship between recommendations and diversity, and in particular how generalists and specialists respond differently to algorithmic recommendations.
%\todo{Finish}
This is measured experimentally by exposing real users of \emph{Spotify} to different recommendation conditions. Three types of recommender system are used: one which merely ranks songs (within a specific predefined subgenre) by popularity; one which is a simple relevance ranker based on user-to-item similarity (essentially a form of heuristic recommendation); and one which is a learned recommender specifically trained to maximize the probability that a user will listen to a song to completion.

First, compared to popularity, recommendation approaches lead to a substantial increase in the number of songs streamed for both groups (they also lead to an increase in the number of songs \emph{skipped}, but this is more than made up for in additional streams). That is, users could be said to be more \emph{engaged} when interacting with recommendations compared to a popularity baseline.
Second, the benefit of recommendations appears across both groups (generalists and specialists), though is significantly more pronounced for specialists: this aligns with the paper's hypothesis in the sense that specialists are more sensitive to songs matching their personal relevance criteria.
%
Finally, the learned ranker confers a slight additional benefit over the relevance ranker, though the benefit is surprisingly modest, indicating that simple relevance ranking is sufficient in this context.

\subsection{Filter Bubbles and Online News Consumption}

Much of the discussion of `filter bubbles' has been in the context of online news, where concerns generally center around whether recommender systems (or more simply, algorithmic ranking techniques) will limit the ideological diversity of content users consume.

\cite{bakshy2015exposure} studied the extent to which users on \emph{Facebook} tend to consume news that confirms to their political ideology. The analysis begins by training a supervised learning system to label news articles as `liberal,' `conservative,' or `neutral,' based on shares by users who volunteer their political affiliation as part of their profile.
%Exposure to ideologically diverse news and opinion on Facebook (or somewhere else)

The main questions of interest center around the extent to which users are exposed to (or choose to interact with) content that is aligned with their own ideology versus content which is `cross-cutting.' `Exposure' refers to algorithmic feed ranking choosing to surface the content, whereas `interaction' refers to a user's choice to click on exposed content.

There are many confounding factors in such an analysis, which the study attempts to control for. For example, users' social networks are primarily composed of friends who share a common ideology, so naturally the content users could potentially be exposed to via their social network is predominantly not cross-cutting. Likewise, users' tendency to interact with (i.e.,~click on) content is confounded by the fact that the feed ranker already factors in click probability when determining which (and how prominently) content is surfaced to the user in the first place.

After attempting to control for these effects, the study's main findings are that algorithmic ranking indeed exposes users to less ideologically diverse news than would be expected by the ideological makeup of their social group, however users interact with ideologically diverse content even less then they are exposed to it; from this, the authors conclude that individual choice plays the largest role in users' exposure to content that is ideologically homogeneous.

However, the argument above does not refute the possibility of a `filter bubble' of online news consumption, it merely argues that its primary cause (in the case of \emph{Facebook}) is not necessarily algorithmic.

\cite{flaxman2016filter}
%Filter bubbles, echo chambers, and online news consumption
%\todo_{Write this section, see if this fits in between}

\paragraph{Filter Bubbles on Google News}

%\cite{haim2018burst}
%Burst of the Filter Bubble?
%Effects of personalization on the diversity of Google News
\cite{haim2018burst} conducted an exploratory study of recommendation on \emph{Google News}, in order to determine the effects of personalization on content diversity. Like \cite{bakshy2015exposure}, they broadly argue that the effects of filter bubbles are somewhat overstated, or otherwise that the patterns of bias in recommendations are not the same as what is anecdotally understood to be a `filter bubble.'

They conduct two studies, to look at `explicit' and `implicit' personalization. Both are based on empirical observation of the actual news recommendations provided by \emph{Google News}, sampled from several synthetic user accounts. Recommendations are then compared against `traditional' (i.e.,~non-personalized, curated) news sources in terms of topic and content diversity.

In the `explicit' setting, they make use of a \emph{Google News} feature that allows users to specify the types of news they are interested in, among a set of broad categories (e.g.~sports, entertainment, politics). Annotators then labeled recommended articles according to these categories in order to quantify the alignment between the explicit preferences and the recommended articles.

The first finding is simply that \emph{Google News} does indeed respect users' explicit preferences, in the sense that the proportion of recommended articles matching the desired topic far exceeds their proportion in a non-personalized setting.

\cite{haim2018burst} also evaluates recommendations in terms of source diversity (i.e.,~in terms of the original news sources that \emph{Google News} aggregates). Here, they find surprisingly that a few somewhat niche news sources dominate recommendations, whereas more mainstream sources are underrepresented; this result is relatively consistent across each of the personalized accounts.

In the `implicit' setting, \cite{haim2018burst} made use of several social media accounts, corresponding to users with specified (but synthetic) demographics and preferences (such as a marketing manager, an elderly conservative, etc.). Each of these simulated agents then interacts with social media (liking articles on \emph{Facebook}, \emph{Google+}, etc.), after which their \emph{Google News} recommendations are compared.

The main conclusion of this second study is simply that implicit personalization has little effect on the recommended results (though there is evidence that some results are indeed personalized).

Ultimately while both \cite{bakshy2015exposure} and \cite{haim2018burst} argue against a `filter bubble' as such, both point to potential issues of bias in recommendations; 
%\todo_{finish this argument}



% Blockbuster culture’s next rise or fall:
% The impact of recommender systems on sales diversity (already covered)


Breaking the filter bubble: democracy and design
(maybe)

% Cover:
% Exploring the filter bubble: the effect of using recommender systems on content diversity \cite{nguyen2014exploring}



\section{Fairness}

%Paper: Beyond Parity: Fairness Objectives for Collaborative Filtering (http://papers.nips.cc/paper/6885-beyond-parity-fairness-objectives-for-collaborative-filtering.pdf)

\emph{Fairness} in machine learning is often defined in terms of predictions and protected characteristics. For example, when building a classifier to aid in hiring decisions, we might be interested in ensuring that men and women are ranked as `qualified' at approximately the same rate. Or, a system for predicting recidivism should not be biased against individuals of a certain race \cite{chouldechova2017fair}. 

Typically, we might 
%want the predictions of 
desire that the outputs of
our classifier $f(x_i)$ 
%to be independent of a 
do not depend on some
protected feature $x_{i,f}$ (indicating race, gender, etc.).
%
Some common definitions include, for example, \emph{demographic parity}, which states that the probability of a positive prediction (e.g.~being ranked as `qualified') should be the same whether one has the protected feature or not:
\begin{equation}
    P(f(x_i) = 1 | x_{i,f} = 1) = P(f(x_i) = 1 | x_{i,f} = 0).
    %\todo{Check definition against some paper}
\end{equation}
The related notion of \emph{equal opportunity} allows for the possiiblity that the target variable depends on the protected feature,
%\emph{among qualified candidates} (i.e.,~$y_i = 1$)
and states that \emph{among the qualified $(y_i = 1)$ or unqualified $(y_i = 0)$
%\todo_{y_i = 0 or y_i = 1}
individuals}, the probability of a positive prediction should be the same whether or not one has the protected features:
%Equal opportunity:
%\todo{Finish descriptions of both}
\begin{equation}
    P(f(x_i) = 1 | x_{i,f} = 1, y_i = y) = P(f(x_i) = 1 | x_{i,f} = 0, y_i = y).
\end{equation}
These are just two examples out of dozens of possible notions of fairness that may be of interest, see e.g.~\cite{mehrabi2019survey} for a comprehensive survey.

A classifier may violate the above rules for a variety of reasons. For example, the training data may exhibit historical bias against a certain group; or, a classifier trained on highly imbalanced data may simply make inaccurate (or imbalanced) predictions for groups that are poorly represented in the data. Several machine learning techniques have been proposed to mitigate unfairness in such scenarios, for example by pre-processing the biased data \cite{kamiran2009classifying}, or by altering the classifier itself \cite{zafar2017fairness}.
%, or post-process the outputs \cite{me}.

%\cite{pedreshi2008discrimination}

In the contexts of recommendations and personalized predictions, one may have slightly different definitions and goals in terms of building a `fair' model. \cite{yao2017beyond} attempts to adapt notions of fairness to personalized recommendation contexts. The paper considers a running example of recommendation in the context of education, where course evaluations in Computer Science (for example) may primarily represent the preferences of the predominantly male population; models trained on such data (or even simple statistics compiled from the data).

\cite{yao2017beyond} introduce several metrics of fairness with respect to the outputs of a recommender system, and show that these metrics can be straightforwardly incorporated into the training objective. Their metrics are defined by dividing users into groups $g_u$ and items into `genres' or attributes $h_i$; user groups are assumed to be binary, though in the studied case of gender $g_u$ can simply be divided into the over-represented group (male) and the under-represented group (non-male).

% Briefly, the fairness metrics they define include:
% \begin{description}
  \emph{Value Unfairness} measures the extent to which one group tends to have their ratings over- or under-predicted compared to the other:
  \begin{equation}
      U_{\text{val}} = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \Bigl( \underbrace{\mathbb{E}_g[y]_i}_{\hspace{-10mm}\mathrlap{\text{expected \emph{prediction} for group $g$ on item $i$}}} - \overbrace{\mathbb{E}_g[r]_i}^{\hspace{-10mm}\mathrlap{\text{average \emph{rating} for group $g$ on item $i$}}} \Bigr) - \Bigl( \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i\Bigr) \Bigr|.
      \label{eq:valueUnfairness}
  \end{equation}
  Note that since both sides take expectations (or averages), the measure is invariant to size differences between the two groups.
  
  Value unfairness could occur in a latent factor model (like that of \Cref{eq:?}) if, for example, predicitons are dominated via the bias terms $\beta_i$; in a model in which one group was over-represented, the bias terms may essentially reflect the preferences only of the over-represented group.
  %\todo{Tenses in above sound funny}
% \end{description}

\emph{Absolute Unfairness} replaces the differences in expectation from \Cref{eq:valueUnfairness} with absolute values:
\begin{equation}
      U_{\text{abs}} = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \Bigl| \mathbb{E}_g[y]_i - \mathbb{E}_g[r]_i \Bigr| - \Bigl| \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i\Bigr| \Bigr|.
      \label{eq:absoluteUnfairness}
  \end{equation}
  Following this change, absolute unfairness now captures the extent to which one group has their ratings \emph{mispredicted} in an absolute sense more than the other. This essentially measures a difference in the system's \emph{utility} between the two groups, in the sense that if one group routinely receives recommendations with high error then the system is unlikely to be useful to them.
  %\todo{poorly phrased}
  
  Next, \cite{yao2017beyond} define \emph{under-} and \emph{over-estimation} fairness to assess the model's tendendy of the model to either under or overpredict the true ratings:
  \begin{eqnarray}
      U_{\text{under}} & = & \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \max\lbrace 0,  \mathbb{E}_g[r]_i - \mathbb{E}_g[y]_i \rbrace - \max\lbrace \mathbb{E}_{\neg g}[r]_i - \mathbb{E}_{\neg g}[y]_i \rbrace \Bigr|,\\
      U_{\text{over}} & = & \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \max\lbrace 0,  \mathbb{E}_g[y]_i - \mathbb{E}_g[r]_i \rbrace - \max\lbrace \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i \rbrace \Bigr|.
      \label{eq:overunder}
\end{eqnarray}
%\todo_{doesn't render right, comma is blocked}
These definitions are somewhat analogous to related concepts we saw when evaluating ranking models in \Cref{sec:evaluatingClassifiers}; consistently underpredicting is analogous to having low recall (failing to retrieve relevant items), whereas overpredicting is analogous to having low precision (retrieving items that are not relevant). Both can potentially reduce the utility of the recommender system for one of the groups.
%\todo{Precision and recall don't feel quite right...}

Ultimately, \cite{yao2017beyond} shows that each of the above metrics can be incorporated into recommender systems of the form given in
\Cref{eq:MFObjective}.
That is, they can be combined via a trade-off term so that the model is accurate while minimizing unfairness, e.g.:
\begin{equation}
    \frac{1}{\mathcal{T}} \sum_{(u,i) \in \mathcal{T}} \underbrace{(\alpha + \beta_i + \beta_u + \gamma_i + \gamma_u - R_{u,i})^2}_{\text{accuracy}} + \lambda \underbrace{U_{\text{abs}}}_{\mathclap{\text{(absolute) fairness}}}.
    \label{eq:fairnessTradeOff}
    %\todo{Might fix alignment of underbraces}
\end{equation}
Optimization remains straightforward, as each fairness metric is differentiable with respect to the model parameters.\footnote{Or at least, has a subderivative.} The main finding of the paper is then that fairness metrics can optimized while paying only a minimal price in terms of overall model accuracy.
%\todo_{should extend the above argument to demonstrate that these types of unfairness do actually exist in data!}

%http://www.ec.tuwien.ac.at/~dimitris/research/recsys-fairness.html



\subsection{Multisided Fairness}
\label{sec:multisided}

A separate attempt to introduce fairness metrics is described in \cite{burke2017multisided}. In comparison to the fairness metrics defined above, the main difference is to consider fairness from both the perspective of users of the system (`consumers') as well as content providers (`producers'). As a motivating example, they consider a hypothetical recommendation scenario on the microfinancing website \emph{Kiva.org}, where there may be an desire to ensure that proposals
%\todo{is that the right word?}
from different businesses receive somewhat balanced representation among recommendations. More broadly, this is an instance of recommendation in a `matchmaking' setting where both sides (in this case, users and businesses) are being matched to each other; in such cases, fairness should not be defined in terms of one `side' only, but should consider the needs of both types of stakeholders. Other examples are cited including online advertising, the sharing economy, or online dating \cite{pizzato2010recon}.

To achieve this notion of fairness, they consider fairness separately from the perspective of `consumers' and `producers,' which they term $C$- and $P$-fairness. Following these definitions, the fairness metrics we studied above
%\todo{check}
would be examples of $C$-fairness. \cite{burke2017multisided} notes that $C$- and $P$-fairness are not merely symmetric definitions, and that $P$-fairness may have requirements not encountered when studying $C$-fairness, such as in the examples above. For example, in a product recommendation setting, if we wanted to encourage sales diversity (as we considered in \Cref{sec:salesDiversityAbove}, and will revisit in \Cref{sec:salesDiversityBelow}), the producers are \emph{passive} in the sense that they are not actively seeking out recommendations in the system.

Finally, \cite{burke2017multisided} considers settings of $\mathit{CP}$-fairness, where fairness must be considered from the perspectives of both sides simultaneously. We'll revisit examples of $P$-fairness and $\mathit{CP}$-fairness as we examine case studies of gender bias in \Cref{sec:genderCaseStudies}.

%Citation: Sales diversity is useful to \cite{fleder2009blockbuster}
%Have this above somewhere too

% Multisided Fairness for Recommendation (https://arxiv.org/abs/1707.00093)

Paper: Balanced Neighborhoods for Multi-sided Fairness in Recommendation (maybe)

\subsection{Group-Aware Recommendation} 

Group-aware Recommendation?
Paper: Top-N Group Recommendations with Fairness

Fairness-Aware Group Recommendation with Pareto-Efficiency


\section{Case Studies on Gender Bias in Recommendation}
\label{sec:genderCaseStudies}
%\todo{Could be a section with subsections, rather than a subsection with paragraphs}

Just as \cite{yao2017beyond} used gender imbalance as a motivating example to study fairness and bias with regard to under-represented groups in recommender systems, several studies have investigated specific scenarios where recommenders exhibit significant bias, or have reduced utility for a specific gender.

\subsection{Data Resampling and Popularity Bias}

\cite{ekstrand2018all} studies a similar problem as \cite{yao2017beyond} (\Cref{sec:fairnessRec?}),
also noting that there is a substantial \emph{utility gap} between the majority versus under-represented groups (i.e.,~closest to \emph{absolute unfairness} as in \Cref{eq:absoluteUnfairness}). Bias is reported with respect to both gender and age attributes, both of which are self-reported by uses in datasets of movies and songs (from \emph{MovieLens} \cite{harper2015movielens} and \emph{Last.FM} \cite{celma2009music}).

Unlike \cite{yao2017beyond}, which corrects for this type of bias using a joint objective that balances overall utility with unfairness (\Cref{eq:fairnessTradeOff}), \cite{ekstrand2018all} uses a \emph{data resampling} approach to correct for bias. This type of approach is borrowed from \cite{kamiran2009classifying} where it was used in the context of fair classification. The basic idea is to resample the data so as to achieve equal representation among groups; practically speaking this is fairly similar to the reweighting schemes we explored in \Cref{sec:trainingBalancedClassifier}.

\cite{ekstrand2018all} also raises the potential issue of \emph{popularity bias} in recommender systems (also discussed in \cite{bellogin2011precision}), in which algorithms that work well for popular items will generally be favored over algorithms which \emph{personalize} better, but whose performance is worse for popular items.
%\todo{Clunky phrasing above}
To address this they introduce evaluation metrics that control for the effect of popularity, so that algorithms can be compared according to their degree of personalization rather than their tendency to select popular items.
%\todo{Clunky phrasing here too}

\subsection{Bias and Author Gender in Book Recommendations}
\label{sec:genderBook}

%Paper: Exploring Author Gender in Book Rating and Recommendation 

\cite{ekstrand2018exploring} 
%\todo{Is this the same as Exploring Author Gender in Book Rating and Recommendation, Michael D. Ekstrand and Daniel Kluver? Latter only has two authors}
explores bias from the perspective of book \emph{authors}. This is somewhat analogous to the idea of $P$-fairness from \Cref{sec:multisided}, given that we are interested in how recommendations could be biased against `producers' (in this case, authors of a certain gender).

Data is collected from
\emph{BookCrossing} \cite{ziegler2005improving},
\emph{Amazon} \cite{mcauley2015image},
and \emph{GoodReads} \cite{wan2018item}. An interesting component of the study is how these datasets can be augmented to incorporate the gender of each author, which is not a feature immediately available in any of the above datasets; author gender information is compiled from external sources, which is matched to records in each of the datasets.

The main research questions in \cite{ekstrand2018exploring} start by analyzing the overall gender distribution of authors in the datasets, as compared to the gender distribution among reading histories of individual users. Beyond this, they seek to study how gender bias is `propagated' by recommendation algorithms, i.e.,~the extent to which users who exhibit a moderate tendency toward authors of a certain gender will tend to have recommendations in which that gender is more extremely over-represented. Finally, they analyze the extent to which these issues can be mitigated algorithmically.

Ultimately, the study concludes that authors in all three datasets (at least those whose identities could be resolved) are predominently male. In terms of rating histories by users, the distribution is less skewed. 
In terms of recommendation algorithms, results are quite mixed, with certain algorithms and datasets leading to more or less skewed recommendations, or otherwise recommendations that mimic users' own gender preferences.

Finally, the authors find that gender imbalance in recommendations can be mitigated easily via simple re-ranking strategies, with minimal impact on performance (the strategy they use is similar to the one of \cite{ziegler2005improving}, as we studied in \Cref{sec:?}).
This analysis bears some similarity to our study of filter bubbles (\Cref{sec:?}), or the techniques used to calibrate recommendations (in this case to match a desired gender distribution rather than a genre distribution) from \Cref{sec:?}.
%or extremification (\Cref{sec:?})


%\todo{Make sure I use this citation the first time I introduce the GoodReads Data in chapter 1}

\subsection{Gender Bias in Marketing}

\cite{wan2020addressing} investigates bias in terms of how products are \emph{marketed}. For example, a user may be more (or less) inclined to purchase a clothing item if it is modeled by somebody sharing their gender, weight, age, skin-tone, etc. In some instances, these features may be directly relevant to the suitability of the item, but in others they may not be. If users are disinclined to interact with items simply because their own identity is not represented, this reduces the utility of the system to the users, represents a missed opportunity in terms of sales, and raises broader issues of representation in marketing. `Fairness' from this perspective is 
%somewhat between $C$- and $P$-fairness
an instance of $\mathit{CP}$-fairness
from \Cref{sec:multisided}, as both producers and consumers 
%stand to be harmed.
face consequences from unfair treatment.

Like the above
%\todo{check}
study, \cite{wan2020addressing} starts by assessing the extent to which these types of bias can be found in historical purchase data. They consider two settings: clothing, using a dataset from \emph{ModCloth},
%\todo{italics}
and electronics, using data from \emph{Amazon}. On \emph{ModCloth}, they are interested in whether users are less inclined to buy items if the model has a different body type than the user (e.g.~the user is plus-size but the model is not, even though the clothing is available in plus-sizes). On \emph{Amazon}, they are interested in whether ostensibly `genderless' products have different sales patterns among male and female users.

Again, the study faces difficult issues of augmenting the data, since gender and size attributes of users are not readily available. \emph{ModCloth} specifies the size of the models in marketing images, and user sizes are inferred from their historical tendency to purchase only items of a certain size. Data augmentation is more difficult on \emph{Amazon}: gender attributes in marketing images must be inferred using computer vision techniques; gender attributes of users are inferred from their purchases in the \emph{clothing} category.\footnote{Of course, clothing purchases are a rough proxy for gender identity, and users whose purchases span both gender categories are not considered.}

Indeed, the study determines that there is significant correlation between users' attributes and their purchase patterns (e.g.~male users tend to purchase electronics items marketed by male models).
%
Of course, as with gender in book recommendations
above
%\todo{check}
(\Cref{sec:genderBook}), it is hard to disentangle `bias' or `unfairness' from users' intrinsic preferences or legitimate marketing choices (e.g.~the reason that women tend to buy women's watches may be largely practical). However the goal is to determine whether bias is \emph{amplified} by recommender systems, and whether this effect can be mitigated.

The specific question that is asked is whether recommendation \emph{errors} are correlated with market segments and marketing images. This bears passing similarity to the notion of \emph{absolute unfairness} as in \Cref{eq:absoluteUnfairness}, though that measure considers unfairness only from the perspective of the \emph{user's} identity, whereas the question asked here concerns both user and item `identity' simultaneously. Specifically, four possible types of error are investigated:
\begin{equation}
\begin{tabular}{cc}
$\text{Product Image}\left\lbrace
\begin{tabular}{c}
Female\\
Male
\end{tabular}\right.$ \hspace{-6mm} &
$\left[ \begin{tabular}{cc}
$\bar{e}_{F,F}$ & $\bar{e}_{M,F}$\\
$\bar{e}_{F,M}$ & $\bar{e}_{M,M}$
\end{tabular} \right]$\\
 & $\underbrace{\begin{tabular}{cc}
 Female & Male
 \end{tabular}}_{\text{\normalsize User Identity}}$
 \end{tabular}.
\end{equation}
Under a null model, the errors should not be correlated with market segments (this can be measured via a specific statistical test).

Finding that these errors are indeed significantly correlated with market segments, \cite{wan2020addressing} seeks to address this via a loss which balances model error and error correlation:
\begin{equation}
    \overbrace{\sum_{u,i} (f(u,i) - r_{u,i})^2}^{\text{prediction error}} + \alpha \underbrace{\mathcal{L}_{\mathit{corr}}}_{\mathclap{\text{error parity on market segments}}}.
    %\todo{make not look ugly}
\end{equation}
Again, this joint loss can be optimized much like the one in \Cref{eq:fairnessTradeOff}, satisfying the fairness objective with minimal loss in prediction accuracy.


% \section{Bias and the ‘Long-Tail’ of Personalization}

%Paper:All The Cool Kids, How Do They Fit In?
%Moved this to fairness section



\section{Justification and Trust}

Building User Trust in Recommendations via Fairness and Explanations

Designing and evaluating explanations for recommender systems


Amazon papers:
?

Netflix:
Big \& Personal: data and models behind Netflix recommendations (talks about explainability, but not very good)


%\section{Case Studies from Industry}



%Paper: Algorithmic Effects on the Diversity of Consumption on Spotify 

%\todo{Need to organize these papers. Should there be a separate `filter bubble' section, or a separate 'diversity' section?}



\begin{problemlist}
\item Max Marginal Relevance on some recommendation problem
%\todo{this}
\item Some other re-ranking task
\item Study beer dataset (or some other dataset) with respect to gender bias. Compute the various unfairness metrics on the data.
\end{problemlist}


%\todo{Diversity? Justifiability / explainability}

%\backmatter
% % if you only have one appendix, use \oneappendix instead of \appendix
%   \appendix
%   \endappendix
% insert a blank line to the toc list
%   \addtocontents{toc}{\vspace{\baselineskip}}
%   \theendnotes

% bibliography style files
% if you are using harvard, the style is specified with the \usepackage command
% \bibliographystyle{vancouver}% uncomment for vancouver style
% \bibliographystyle{IEEEtran} % uncomment for IEEE style

% \renewcommand{\refname}{Bibliography}% if you prefer this heading

% \theendnotes

%TODO: This doesn't appear in the references properly. I tried to follow the guide but it just appears in the last chapter
\bibliography{refs}
%\label{refs}

%   \cleardoublepage

% indexes

% for a single index
\printindex

% for multiple indexes using multind.sty
%   \printindex{authors}{Author index}
%   \printindex{subject}{Subject index}

% for multiple indexes using index.sty
% \printindex[aut]
% \printindex


\end{document}
\endinput
