% authored_guide.tex
% 2011/06/23, v3.1 gamma
%
% Adapted by Diana Gillooly and David Tranah
% from Ali Woollatt's original documentation for cambridge7A

\RequirePackage{listings}

\NeedsTeXFormat{LaTeX2e}[1996/06/01]

\documentclass{cambridge7A}
% \documentclass[spanningrule]{../cambridge7A}% option

%\done{Don't know if this works? -- yes. \part causes problems so used \part*. \parts don't show up in ToC though}
\usepackage[hidelinks,linkcolor=black]{hyperref}

% \usepackage{todonotes}

\usepackage{lmodern}

\makeatletter
\let\commentfullflexible\lst@column@fullflexible
\makeatother

\lstset{
%   basicstyle=\small\ttfamily,
  basicstyle=\fontsize{8}{8}\selectfont\ttfamily,
%   columns=fullflexible,
% commentstyle=\rmfamily\commentfullflexible,
  breaklines=true,
  frame = single,
  numbers = left,
  numberstyle = \small,
%   framexleftmargin=0pt,
  language=Python,
  xleftmargin=15pt,
  xrightmargin=5pt,
  aboveskip=1.75\medskipamount,
  upquote=true
}

\setlength{\fboxsep}{5pt}

\newcounter{projectCounter}

%\done{Sidecap not part of this style? Keeping for the moment}
\usepackage[rightcaption,raggedright]{sidecap}% for side captions
\usepackage{framed}         % for floatingboxes
\usepackage{soul}           % for letterspacing in theorem-style headings


  \usepackage{natbib}
% \usepackage[numbers]{natbib}% option

  \usepackage[figuresright]{rotating}
  \usepackage{floatpag}
  \usepackage{mathtools}
  \rotfloatpagestyle{empty}
  
  \usepackage{booktabs}
  
  
  \usepackage{xcolor}


% \usepackage{amsmath}% if you are using this package,
                      % it must be loaded before amsthm.sty
  \usepackage{amsmath}
  \usepackage{amsthm}
  \usepackage{amsfonts}
  \usepackage{graphicx}
  
  \usepackage{wrapfig}
  
\usepackage{tabularx}


\usepackage{cleveref}

\crefname{equation}{eq.}{eqs.}
\crefname{figure}{fig.}{figs.}
\crefname{tabular}{tab.}{tabs.}
\crefname{section}{sec.}{secs.}
\crefname{chapter}{chap.}{chaps.}

\Crefname{equation}{Equation}{Equations}
\Crefname{figure}{Figure}{Figures}
\Crefname{tabular}{Table}{Tables}
\Crefname{section}{Section}{Sections}
\Crefname{chapter}{Chapter}{Chapters}

\usepackage{tikz}
\usetikzlibrary{positioning,chains,fit,shapes,calc,arrows.meta}
\usetikzlibrary{decorations.pathreplacing}
\usetikzlibrary{fadings}


\usepackage{enumitem}
\setlist[description]{leftmargin=2\parindent}

 \usepackage{txfonts}
 \usepackage{siunitx}
% \usepackage[scaled=0.9]{couriers}% use if you're using \tt fonts

% indexes
% uncomment the relevant set of commands

% for a single index
\usepackage{makeidx}
\makeindex

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}


% for multiple indexes using multind.sty
%   \usepackage{multind}\ProvidesPackage{multind}
%   \makeindex{authors}
%   \makeindex{subject}

% for glossary entries
  %\usepackage[style=list]{glossary}
  \usepackage[number=none]{glossary}
\makeglossary

% theorem definitions
% see chapter 3 for details
%   \theoremstyle{plain}% default
%   \newtheorem{theorem}{Theorem}[chapter]
%   \newtheorem{lemma}[theorem]{Lemma}
%   \newtheorem{proposition}[theorem]{Proposition}
%   \newtheorem{corollary}[theorem]{Corollary}
%   \newtheorem{conjecture}[theorem]{Conjecture}

%   \newtheorem*{theorem*}{Theorem}
%   \newtheorem*{lemma*}{Lemma}
%   \newtheorem*{proposition*}{Proposition}
%   \newtheorem*{corollary*}{Corollary}
%   \newtheorem*{conjecture*}{Conjecture}

%   \theoremstyle{definition}
%   \newtheorem{definition}[theorem]{Definition}
%   \newtheorem{example}[theorem]{Example}
%   \newtheorem{prob}[theorem]{Problem}
%   \newtheorem{remark}[theorem]{Remark}
%   \newtheorem{notation}[theorem]{Notation}
%   \newtheorem{exer}[theorem]{Exercise}

%   \newtheorem*{definition*}{Definition}
%   \newtheorem*{example*}{Example}
%   \newtheorem*{prob*}{Problem}
%   \newtheorem*{remark*}{Remark}
%   \newtheorem*{notation*}{Notation}
%   \newtheorem*{exer*}{Exercise}



% \hyphenation{docu-ment baseline-skip polar}

% for this documentation, table of contents lists up to subsection level
  \setcounter{tocdepth}{2}

  \newcommand\cambridge{cambridge7A}

% remove the dot and change default for enumerated lists
  \def\makeRRlabeldot#1{\hss\llap{#1}}
  \renewcommand\theenumi{{\rm (\roman{enumi})}}
  \renewcommand\theenumii{{\rm (\alph{enumii})}}
  \renewcommand\theenumiii{{\rm (\arabic{enumiii})}}
  \renewcommand\theenumiv{{\rm (\Alph{enumiv})}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% \includeonly{06authored}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% \makeatletter
% \def\smallunderbrace#1{\mathop{\vtop{\m@th\ialign{##\crcr
%   $\hfil\displaystyle{#1}\hfil$\crcr
%   \noalign{\kern3\p@\nointerlineskip}%
%   \tiny\upbracefill\crcr\noalign{\kern3\p@}}}}\limits}
% \makeatother

\makeatletter
\providecommand*{\toclevel@schapter}{0}
\makeatother

% \newcommand{\empt}[2]{$#1^{\langle #2 \rangle}$}
\newcommand{\empt}[2]{$#1_{#2}^l$}

% \vbadness=10000
\vbadness=2000

\begin{document}


\sidecaptionvpos{figure}{c}


  \title{Personalized Machine Learning}
  \author{Julian McAuley}


%   \bookabstract{This is the guide for authors who are preparing written,
%     rather than edited, books.}
%   \bookkeywords{\LaTeX; authored books; CUP style; cambridge7A.cls.}

  \frontmatter
  \maketitle
  \tableofcontents
  %\done{Dunno if this is part of 7A. Leaving in for now}
  \listoffigures
  \listoftables
% \listofcontributors

%   \include{authored_preface}
  
  
  %\done{residual definitions not always in same order}
  
  %\done{Dunno if this or glossary? Keep for now}
  \chapter*{Notation}
  
  \section*{Common Mathematical Symbols}

%\begin{tabular}{@{}lX}
\begin{tabularx}{\linewidth}{lX}
\multicolumn{2}{c}{\textbf{Machine learning:}}\\[2pt]
 $y$ & vector of labels\\
 $X$ & matrix of features\\
 $x_i$ & feature vector for the $i^{\text{th}}$ sample\\
 $f(x_i)$ & model prediction for the $i^{\text{th}}$ sample\\
 $r_i$ & residual (error) associated with the $i^{\text{th}}$ prediction, $r_i = (y_i - f(x_i))$\\
 $\theta$ & vector of model parameters\\
 $\sigma$ & sigmoid function $\sigma(x) = \frac{1}{1 + e^{-x}}$\\
%  $\Omega$ & (generic) regularizer (usually one of the two below)\\
 $\| x \|_p$ & $p$-norm, $\| x \|_p = (\sum_i |x_i|^p)^{1/p}$\\
 $\ell_1$; $\ell_2$ & regularizers $\| \theta \|_1$ and $\| \theta \|_2$\\
%  $\ell_2$ & \\
 $\lambda$ & regularization hyperparameter\\
 $\mathcal L$; $\ell$ & likelihood and log-likelihood \\[2pt]
\multicolumn{2}{c}{\textbf{Users and items}:}\\[2pt]
 $u \in U$ & user $u$ in user set $U$\\
 $i \in I$ & item $i$ in item set $I$\\
 $I_u$ & set of items rated (or interacted with) by user $u$\\
 $U_i$ & set of users who have rated (or interacted with) item $i$\\
 $|U|$; $|I|$ & number of users and number of items\\
%  $|I|$ & number of items\\
 $R_{u,i}$ & measurement (e.g.~a rating) associated with an interaction between user $u$ and item $i$\\
 $x_{u,i}$ & model estimate of the compatibility between user $u$ and item $i$\\[2pt]
 \multicolumn{2}{c}{\textbf{Recommender systems}:}\\[2pt]
%  $\ell$ & \\
%  $\mathchardef\mhyphen="2D \mathit{tf\mhyphen idf}$\\
%  $\mathcal W$ & dictionary (a set of words)\\
 $\beta_u$ & bias term associated with user $u$\\
 $\beta_i$ & bias term associated with item $i$\\
 $\gamma_u$ & vector of parameters describing a single user $u$\\
 $\gamma_i$ & vector of parameters describing a single item $i$\\
 $\gamma_U$; $\gamma_I$ & parameters for all users $U$ or all items $I$\\
%  $\gamma_U$ & parameters for all users $U$\\
 $K$ & feature dimensionality (or number of latent factors)\\
  %; also used in precision@$K$ and recall@$K$\\
%  $R^2$ & something\\
%  \multicolumn{2}{c}{Other mathematics:}\\
%  $\delta$ & indicator function\\
\end{tabularx}
%\done{Write}

% \section*{Definitions}

% Hyperparameters

% Regularizer

\section*{Common Abbreviations}

%\done{make table three columns. No}
\begin{tabular}{@{}ll}
 AUC & Area Under the ROC Curve (\cref{eq:bprOpt1})\\
 BER & Balanced Error Rate (\cref{eq:BER})\\
 BPR & Bayesian Personalized Ranking (\cref{subsec:BPR})\\
 CNN & Convolutional Neural Network (\cref{subsec:CNNRNN})\\
 FVU & Fraction of Variance Unexplained (\cref{eq:FVU})\\
 FN/FNR & False Negatives / False Negative Rate (\cref{sec:type1type2})\\
 FP/FPR & False Positives / False Positive Rate (\cref{sec:type1type2})\\
 GAN & Generative Adversarial Network (\cref{sec:personalGans})\\
 LSTM & Long Short-Term Memory Model (\cref{sec:rnn})\\
 MAE & Mean Absolute Error (\cref{eq:MAE})\\
 MLE & Maximum Likelihood Estimation (\cref{subsec:MLE})\\
 MLP & Multilayer Perceptron (\cref{subsec:MLPRec})\\
 MMR & Maximal Marginal Relevance (\cref{sec:MMR})\\
 MRR & Mean Reciprocal Rank (\cref{subsec:MReciprocalR})\\
 MSE & Mean Squared Error (\cref{sec:MSE})\\
 NDCG & Normalized Discounted Cumulative Gain (\cref{subsec:cumGain})\\
%  FME & Factorized Markov Embeddings (\cref{sec:PRME})\\
%  SVD & Singular Value Decomposition (\cref{sec:?})\\
 RNN & Recurrent Neural Network (\cref{sec:rnn})\\
 ROC & Receiver Operating Characteristic (\cref{sec:errorRanking})\\
 SVM & Support Vector Machine (\cref{subsec:otherClass})\\
 TF-IDF & Term Frequency and Inverse Document Frequency (\cref{eq:tfidf})\\
 TN/TNR & True Negatives / True Negative Rate (\cref{sec:type1type2})\\
 TP/TPR & True Positives / True Positive Rate (\cref{sec:type1type2})\\
 \end{tabular}

  \mainmatter
  
  
% \part*{Introduction To Personalized Machine Learning}
%\done{Change all these names. Done but could still move them}

\chapter{Introduction}
\label{chap:intro}

%Traditional models of

\emph{Machine learning} encompass a broad range of problems ranging from detecting objects in images, finding documents relevant to a given query, or predicting the next element in a sequence, among countless others. 
%Roughly speaking, the field of 
%Traditionally,
%machine learning
%machine learning
Traditional 
approaches to these problems operate by collecting large, labeled datasets for training, uncovering informative features, and 
%uncovering complex 
mining complex patterns that explain the
association between features and labels.
%by mining patterns in data in order to discover some underlying `truth' about a document, image, or sequence.
Typically, labels are regarded as an underlying `truth' that should be predicted as accurately as possible.

Increasingly, though, 
%there is an increasing need to develop machine learning applications 
there is a need to apply machine learning in settings where the `correct' outcome is subjective, or otherwise depends on the context and characteristics of individual users.
%
%
As we browse online for movies to watch, products to buy, or romantic partners to connect with, we are likely engaging with these new forms of \emph{personalized} machine learning:
%that is 
Results 
%of such algorithms 
%the results 
are tailored to us specifically, based on the types of movies, products, or partners that \emph{we specifically} are likely to engage with.

%More complex applications: Heart rate profiles, anaesthetic volume, surgery scheduling

Much like traditional machine learning algorithms, \emph{personalized} machine learning algorithms are at their heart essentially forms of pattern discovery.
%At their heart, 
That is, predictions are made \emph{for you} by analyzing the behavior of people \emph{similar to you}. A recommendation such as `people who liked this also liked' 
%(\cref{fig_?}) 
is perhaps the most simple example of this type of personalized pattern discovery:\footnote{Though strictly speaking maybe not one that we'd call `machine learning.'} based on the contextual attribute of a user 
liking 
%interacting with
a particular item, recommendations are extracted based on users who share 
%the same attribute. 
this common preference.
At the other end of the spectrum are complex deep learning approaches that learn `black box' representations of users in order to make predictions, though these too at their heart rely on the intuition that `similar' users (in terms of some complex representation) will have similar interaction patterns.



\section{Purpose of This Book}

%In this book 
We seek to introduce \emph{Personalized Machine Learning} by exploring 
%the 
%types
a family
of approaches used to solve the above problems, and construct a narrative around the 
%underlying 
common
methods and design principles involved.
%
%As we explore in the next chapter, 
We show that
even in applications as diverse as song recommendation, heart-rate profiling, or fashion design, there is a common set of techniques around which personalized machine learning systems are built.


%\done{rearrange. Fine}

%Beyond introducing the basic techniques, 
By introducing this underlying set of principles,
%The purpose of \emph{personalized machine learning} 
the book is intended
is to teach readers how modern machine learning techniques can be improved by incorporating ideas from personalization and user modeling, and to guide readers in building machine learning systems where accurately modeling the users involved is key to success.

There is currently an abundance of models, datasets, and applications that seek to capture human dynamics or interactions. Examples pervade in diverse areas including web mining, recommender systems, fashion, dialog, and personalized health, 
%etc.
among others.
%\done{could update to reflect final topics. Fine}
%
As such, there is an emerging set of 
%common 
techniques that are used to capture the dynamics of `users' in each of these settings. This book is designed to act as a reference point to 
%explore and explain these techniques.
explain these techniques, and explore their common elements.
%
As a starting point, we will begin the book (\cref{chap:basicsReg,chap:classPipe}) with a primer of machine learning (and especially supervised learning) that will bring readers up-to-speed on the basic techniques required later. Although this introductory material is likely familiar to many readers, we have a particular focus on user-oriented datasets, and show that even with `standard' machine learning techniques, there is considerable scope for building personalized systems through careful feature engineering strategies that capture relevant user characteristics.

Following this, our main introduction to personalized machine learning will be to explore 
%personalized recommendation
recommender systems
(\cref{chap:recsysIntro,chap:modelbasedRecsys}).
Recommendation technology has traditionally relied on personalization and user modeling, whether through simple similarity functions among users (`people like you also bought' etc.)~or through more modern approaches
%including approaches based on neural networks.
involving temporal pattern mining or neural networks.

More recently, the need to account for personalization and to model users has 
%encroached on 
spread into
a variety of 
%other 
new
areas of machine learning. Following our study of recommender systems, exploring personalization and user modeling in these new areas---and 
%more broadly 
giving readers the tools they need to design personalized approaches 
%to them---
in new settings---is the main goal of this book.
%\done{badly worded}

% Examples include:
% \begin{description}
% \item[Sequence Modeling and Time-Series Analysis]
% %\done{dropped}
% \item[Natural Language Processing] Modern NLP methods focus on interactive systems for language synthesis, dialog, question answering, etc. Within these applications there is a need to understand people's personalized writing styles, subjectivity, personal context (etc.). Understanding the variation
% among users is key to success in these applications.
% \item[Computer Vision] A few `traditional' computer vision problems depend on models of individual users (e.g.~face recognition), though we are more interested in modern applications, such as personalized models of visual preferences, applications in fashion, and personalized GANs.
% Such systems can be used to build recommender systems in which users' aesthetic sense plays a key role, to
% generate fashionable outfits, or to design new items tailored to individual users' preferences.
% %\done{reorder to match chapters. Dropped}
% %\item[Online Advertising]
% \item[Fairness and Ethics] Finally, there is growing recognition of the risks of blindly applying personalization and recommendation technologies without regard for the consequences of doing so. Examples include filter bubbles, `concentration' effects (e.g.~due to a lack of diversity of recommended content), or systematic biases against users from underrepresented groups. In response, there is a growing body of work that tries to understand and correct these types of bias
% \end{description}


% Sequence Modeling Personalized models of sequence data appear when mod-
% eling (e.g.) activity traces, heart rate data, sleep patterns etc.

% Application domains of interest that will serve as focus areas for the book
% include:

% e-Commerce which includes many of the ‘traditional’ problems in recommen-
% dation, but also newer areas such as personalized fashion.

% Healthcare When modeling data such as heart rate sequences, clinical notes,
% (etc.) it is critical to capture the variation among individual users in order
% to make meaningful predictions.
% Web mining Models of behavior on the web, whether to predict future actions,

% model user attrition, model bias and extremism, (etc.) rely on disentan-
% gling global (or ‘background’) effects from the characteristics and varia-
% tions among individuals. This topic is closely connected to the emerging
% 
% area of computational social science.

% \section{Audience}
% The book is primarily intended for readers who already have some
% familiarity with machine learning, though basics will be revisited.
% A typical reader might be a student, researcher, or practitioner with some exposure to ML fundamentals, but who is looking to explore how their applications of interest can be improved by modeling the users involved.

% We 
% %\done{We or I? We}
% hope that any readers working in the areas described above would benefit from the
% presentation and organization of this book. For example, a practitioner working
% with clinical text, and wishing to model their data with machine learning, might
% find their needs poorly served by existing texts. With the increasing need to
% develop personalized machine learning models, it is useful and timely to have a
% textbook with this particular focus.

\section{For Learners: What is Covered, and What Isn't}
\label{sec:whatcovered}

Although this book is primarily intended as a guide to the specific topics of personalization, recommendation, and user modeling (etc.), it should also serve as a relatively gentle introduction to the topic of machine learning in general. 
%Indeed, 
%\done{Consider whether to use "I". No}
%courses I've taught on recommender systems have routinely served as students' first exposure to the broad topic of machine learning.
%Such material is 
Topics such as web mining and recommender systems serve as an
%ideal as a 
ideal
starting point for learners seeking a more `application oriented' view of machine learning compared to what is typically covered in introductory machine learning texts. 

Throughout the book, we focus on building examples on top of large, real-world datasets, and exploring techniques that are practical to implement in projects and exercises. Our particular focus guides us toward (and away from) certain topics, as we describe below.
%, and developing algorithms that are representative of deployed approaches.

% \subsection{Expected Knowledge}
% \label{sec:expectedKnowledge}

% \paragraph{Expected Knowledge}
% \index{json} %\done{get this to work. I think is fine}
% While suitable as a first book on machine learning, the material in this book is intended for readers with some background in computer science, especially some familiarity with statistics, probability, linear algebra, and some experience working with structured datasets (e.g.~\emph{json}, \index{csv}\emph{csv}).
%\done{style}
%
%\done{datasets in index. Dropped}
%
%\done{connecting sentence. Dropped}
% \begin{itemize}
%     \item Programming in \emph{Python}, which is used for all code examples.
%     \item Basic string and data processing, file I/O etc.
%     \item A basic knowledge of linear algebra, probability, etc.
% \end{itemize}

%, and we assume basic knowledge

%Links to resources page
%moved
%To help readers with exercises and projects, and to collect resources such as datasets and additional reading materials, we have prepared an online supplement:

% \begin{center}
% \url{http://cseweb.ucsd.edu/~jmcauley/pml/index.html}
% \end{center}



% For readers less familiar with the above, we give links on our online resources page (\cref{sec:onlineResources}).

%\done{not useful section, could drop. Dropped}

%\subsection{What is \emph{Not} Covered}
%\subsection{What is Covered, and what Isn't}

% \begin{description}
    \paragraph{Regressors, classifiers, and the learning pipeline}
    %\done{Consistent use of title case throughout book}
    We 
    %seek to 
    give a detailed introduction of the end-to-end machine learning process in \Cref{chap:basicsReg,chap:classPipe}, which (while condensed) should be suitable for learners with no background in machine learning. 
    When introducing basic machine learning concepts in \Cref{chap:basicsReg,chap:classPipe}, we limit ourselves to linear regression and linear classification (logistic regression), since these serve as building blocks for the methods we develop later.
    %later methods we develop. 
    Consequently, 
    we ignore dozens of alternative regression and classification methods that are often the core of standard machine learning texts (though we briefly discuss the merits of alternatives in \Cref{subsec:otherClass}).
    %\item[Dimensionality reduction]
    
    
    \paragraph{User representations and dimensionality reduction}
    %We avoid a detailed treatment of dimensionality reduction and clustering algorithms, 
    %both 
    %staples of many introductory texts on machine learning. We note however that many of the techniques we explore when learning user representations are 
    %essentially 
    %at their heart forms of dimensionality reduction. %
    Many of the techniques we explore when learning user representations are 
    essentially 
    %at their heart 
    forms of manifold learning (or dimensionality reduction), and borrow ideas from related topics such as matrix factorization (\cref{subsec:matrixFactor}). While readers should have some basic familiarity with linear algebra,
    %However 
    we 
    %simply 
    %choose to 
    carefully
    avoid a linear algebra-heavy presentation of `traditional' dimensionality reduction techniques:
    %which 
    in terms of actual implementation these have little in common with the methods we develop
    (though we discuss the connection to e.g.~the singular value decomposition in \Cref{subsec:matrixFactor}).
    %(principal component analysis, singular value decomposition, etc.),~
    %\done{not great. Fine}
    
    \paragraph{Deep learning}
    %Although we cover various 
    Any discussion of `modern' machine learning approaches necessitates a fairly broad discussion of deep learning. For example we discuss
    %deep learning techniques (e.g.~
    multilayer perceptron-based recommendation in \Cref{subsec:MLPRec}, sequence models based on 
    recurrent neural networks
    %RNNs 
    in \Cref{chap:temporalSequential}, and models of visual preferences based on 
    convolutional neural networks
    %CNNs
    in \Cref{chap:visual}.
    However in
    %by 
    doing so we are merely scratching the surface of deep learning-based personalization, and will largely refer readers elsewhere for in-depth discussion of specific architectures, or for a first-principles presentation of deep learning methods.
    %However, 
    %Ultimately,
    %we note that deep learning approaches are merely one specific solution to the same problems that we'll primarily 
    %visit 
    %explore
    %using more traditional techniques.
    %\done{funny sentence above}
    %\done{More? Dropped}
    %We survey deep learning-based personalization techniques in \Cref{sec:deepRec}.
    %\item[Reinforcement and online learning]
    
    \paragraph{Offline versus online learning}
    We largely limit ourselves to traditional offline, supervised learning problems, i.e.,~uncovering patterns and making predictions from historical collections of training data.
    Generally we prefer this 
    %category of methods 
    setting
    since it allows us focus on methods that we can develop on top of real-world, publicly available datasets.
    %In practice when deploying predictive models, 
    Of course, in practice, when deploying predictive models,
    data may be obtained in a streaming setting and updates must be made in real time. This type of training regime is known as \emph{online learning}, which we briefly cover in \Cref{sec:onlineUpdates}; 
    %We also 
    we also
    avoid discussion of (e.g.)~reinforcement learning algorithms,
    %which are used in settings including personalized
    %though touch upon their use for online advertising in \Cref{sec:advertising}.
    though mention their use briefly in settings such as conversational recommendation (e.g.~\cref{subsec:convRec}).
    %\done{do we or did I delete?}
    


    
    %\paragraph{User interfaces and human computer interaction}
    \paragraph{Bias, consequences, and user considerations}
    %Finally, we note that our study of personalization is 
    By design, our study of personalization is
    largely confined to machine learning approaches. That is, we are generally concerned with building predictive systems that can estimate---as accurately as possible---how a particular user will respond to a given stimulus. By doing so, we can estimate preferences, predict future activities, retrieve relevant items, etc. 
    
    Of course, we are mindful of the
dangers associated with `black-box' approaches to machine learning,
and want to avoid the pitfalls of
blindly optimizing model accuracy, such as filter bubbles, unwanted biases, or simply a degraded user experience. In \Cref{chap:consequence} we discuss these issues, as well as potential approaches to address them. 

Again 
%though 
our discussion is mostly
limited to machine learning solutions, i.e.,~we investigate \emph{algorithmic} approaches to correct for biases, increase recommendation diversity, etc. 
We note that algorithmic solutions are only part of the picture, and that while having better algorithms is critical, it is also critical that those algorithms are appropriately \emph{used}. %Such questions fall under the umbrella of human computer interaction, and user interface design. Although these topics are outside the scope of this book, we discuss them briefly at the end of
%\done{Either make this exist or delete. Dropped}
% \Cref{chap:consequence}.
    %
    %This 
    Our presentation
    is complementary to a large body of work that explores personalization from the perspective of human computer interaction, or user interface design, where the primary concern is maximizing the quality of the user experience (ease of finding information, satisfaction, long-term engagement, etc.). %\item[Deployment]
    %Likewise we largely avoid the `systems building' side of personalized machine learning, such as concerns around deploying machine learning models on distributed servers (etc.),~though we discuss high-level libraries and implementation best-practices throughout the book.
    
            % \item[Fairness]
    %Of course, there are 

%However our discussion is still 


\paragraph{Implementation and libraries}
%
All code examples are presented in \emph{Python}. While we assume a working familiarity with data processing, matrix libraries (etc.), further links on our online resources page (\cref{sec:onlineResources}) will help users with less familiarity.
%we give links on our online resources page (\cref{sec:onlineResources}).
When discussing deep learning approaches, and more generally when fitting complex models, we base our implementations on \emph{Tensorflow}, though these examples can easily be interchanged with alternate libraries (PyTorch, Theano, etc.).

While we focus on implementation,
%Likewise 
we largely avoid `systems building' aspects of personalized machine learning, such as concerns around deploying machine learning models on distributed servers (etc.),~though we discuss high-level libraries and implementation best-practices throughout the book.
    

    % \item[Implementation]
    
% \end{description}

%\done{paragraphs not connected}



\section{For Instructors: Course and Content Outline}

%\emph{Personalized Machine Learning}
%\done{looks kinda goofy to use book title in sentence}
This book
is inspired by my own experience teaching classes on recommender systems and web mining at UC San Diego. Courses on these topics have proved extremely popular, and are often chosen as learners' first exposure to machine learning.
%courses.

One reason this topic acts as a good first contact with the machine learning curriculum is that it has a somewhat lower bar for entry than many machine learning courses, including (for example) courses on deep learning, or even many `introductory' machine learning classes. Partly this is due to the material being less dependent on deep and complex theory, and partly it is due to the ability to quickly build working solutions that are fairly representative of the state-of-the-art, rather than mere proofs-of-concept. As such, a focus of this book is to quickly build working solutions, and covering a wide breadth of approaches, rather than diving too deep into the theory behind any one approach. 
%I believe this will be 
This approach can be
useful in helping learners
%\done{always use word "learners" or "students". Looks consistent, student only used when needed}
to understand the practical considerations behind building predictive systems based on user data, and is complementary to the more theoretical treatment given in most introductory texts.

Another feature that has made this material popular among learners is the ability to work quickly with large, real-world datasets. The ability to work with collections of user data from \emph{Amazon}, \emph{Google}, \emph{Steam} (etc.),~on applications that are representative of real use-cases, has proved immensely valuable for students building their project portfolios or preparing for interviews. As such, each chapter is paired with project suggestions, each of which would be suitable as a major class project. These projects aim to synthesize the material from each chapter, with 
%a particular focus 
more focus
on 
system building 
%implementation
considerations, design choices, and thorough model evaluation.

%\done{could drop paragraphs above. Fine}

\subsection{Course Plan and Overview}

%\done{redundant? Fine}
The content in this text is aimed at developing a quarter- or semester-long course, for students with some background in linear algebra, probability, and data processing. 
After revising basic material in 
%Chapters \Cref{chap:basics} and \Cref{chap:recsys}
\Cref{chap:basicsReg,chap:classPipe},
\Cref{chap:recsysIntro,chap:modelbasedRecsys}
cover the core material upon which the remainder of the book builds.
%Chapters
\Crefrange{chap:temporalSequential}{chap:visual}
%\done{check. Okay}
are somewhat more orthogonal, such that components can be selected and combined as time or student background allows. A final chapter on 
bias, fairness, and the consequences of personalization
%\done{check above}
(\cref{chap:consequence}) provides an opportunity to revisit earlier material through a new lens.

%\done{Say something about projects and homeworks here}
%Projects, homeworks
Each chapter is paired with homework and a project. Again the focus on these components is mainly on developing practical implementations, working with real data, and understanding the design choices involved, rather than testing theoretical concepts.
Below we briefly summarize the material from each chapter:

%\begin{description}
  \paragraph{Machine Learning Primer} (\cref{chap:basicsReg,chap:classPipe}) 2-3 weeks.
  %a selection of datasets that include user interaction data, in order to visit (or revisit)
  Introduces the foundational concepts of machine learning, feature design, and evaluation, via a selection of datasets that capture user interactions. Exercises 
  %(\cref{sec:basicsExercises}) 
  range from simple data manipulation to building a working machine learning pipeline (training, validation, etc.). Exercises
  %\Cref{chap:basics} 
  are mainly concerned with feature design, including projects (Projects~\ref{proj:1} and \ref{proj:classif})
  %\done{Could make this work with Cref if I can be bothered? No}
  that involve experimenting with activity data involving temporal and geographical dynamics.
  %\done{Is this accurate?}

  
  \paragraph{Recommender Systems} (\cref{chap:recsysIntro,chap:modelbasedRecsys}) 2-3 weeks. 
  %In %\Cref{chap:recsys} 
  %We introduce 
  Introduces
  the core set of techniques used for recommendation. Traditional heuristics are presented in \Cref{chap:recsysIntro} followed by machine learning approaches in \Cref{chap:modelbasedRecsys}. Recommender systems are used to develop the concept of a \emph{user manifold} which is used throughout the following chapters to capture variation among users in 
  %various other 
  several
  settings (\cref{sec:techniques}).
  %\done{not great explanation. "various other settings" sounds generic}
  Exercises 
  are mainly 
  %concerned with 
  focused on
  the basics of building practical recommendation approaches, 
  and projects (Projects~\ref{proj:1111} and \ref{proj:2}) are concerned with building an end-to-end recommendation pipeline for a book recommendation scenario.
  %\done{Is that what's actually in the project}
    %\done{second project? (this and above section}
  
  \paragraph{Content and Structure in Recommender Systems} (\cref{chap:contentStructure}) 1 week. Explores how to incorporate features (i.e.,~side information) into personalization (mostly recommendation) approaches, and explores personalization in settings with additional structure, such as socially-aware recommendation and settings
  %news recommendation, etc.
  involving price dynamics. A particular focus is given to leveraging side-information in \emph{cold-start} scenarios, where interaction histories are not yet available (\cref{sec:coldStart}).
  %\done{confirm that's what's actually covered}
  Some of these content-aware approaches (such as factorization machines) are revisited later the book when 
  %incorporating othe
  developing more complex models based on (e.g.)~temporal or sequential dynamics. A project (Project~\ref{proj:coldStart}) consists of developing recommender systems for use in cold-start settings.
  %(i.e.,~capable of adapting to new users or new items).
  
  \paragraph{Temporal and Sequential Models} (\cref{chap:temporalSequential}) 1-2 weeks. We revise some of the basic approaches to temporal and sequential modeling, such as autoregression and Markov chains, and later develop more complex personalized approaches based on recurrent neural networks.
  %later explores how to personalize such models, 
  The \emph{Netflix Prize} (\cref{sec:caseStudyNetflixTemporal}) is presented as a case study to explore the basic design principles of temporal modeling.
  %ranging from simple
  %and explores 
  %Finally, we explore
  %\done{confirm that that's true}
  A project (Project~\ref{proj:tempBusiness}) compares various approaches to temporal recommendation.
  %\done{tense of these sentences not very consistent. looks okay}
  
  \paragraph{Personalized Models of Text} (\cref{chap:text}) 1 week. After revising some of the basic predictive models of text (such as bag-of-words representations), we explore how text can be used to understand the dimensions of preferences.
  We revisit sequential modeling by exploring techniques that borrow from natural language to model interaction sequences. We also visit methods for text \emph{generation}, which can be personalized in settings ranging from conversation to justification of machine predictions. A project (Project~\ref{proj:text}) consists of building personalized systems for document retrieval.
  
%   \paragraph{Online Advertising}
  %\done{Does this chapter exist? No}
  
  \paragraph{Personalized Models of Visual Data} (\cref{chap:visual}) 1 week. Explores applications involving visual data, ranging from personalized image search, to applications in fashion and design. A project (Project~\ref{proj:visualProj})
  %\done{don't write exercises and a project and then only reference the project}
  consist of building visually-aware recommendation systems for applications in fashion.
  
  \paragraph{The Consequences of Personalized Machine Learning} (\cref{chap:consequence}) 1 week. The final chapter
  %\done{Not the final chapter, possibly rewrite. Is final chapter}
  %looks at the ethics and 
  explores the consequences and pitfalls
  %the various pitfalls 
  of developing personalized machine learning systems. Examples include filter bubbles, extremification, and issues of bias and fairness. The chapter has a significant focus on applied case-studies, and allows us to revisit several of the topics from previous chapters through a new lens. A project (Project~\ref{proj:fairBeer}) consist of improving recommendation approaches in terms of gender parity and other fairness objectives.
%\end{description}
%\done{Make sure items match contents}
%\done{Repetitive, isn't there another section saying basically the same thing? Is okay}
%\done{Could just expand to several paragraphs and not make it a "course outline" type thing. Is fine}

% \paragraph{Related Topics and Further Reading}
% %\done{DELETE Chapter probably not called that}
% (\cref{chap:further}) 1 week. The last chapter points to further directions that are closely related to personalization, but do not quite fit elsewhere. This chapter includes discussion of topics that are closely related to personalized machine learning, such as ad recommendation, but which rely on techniques quite different from those covered in the book (such reinforcement learning), as well as topics for further reading that go beyond machine learning approaches, such as user interface design and deployment considerations.
% %\done{DELETE Rewrite once the topics are in the chapter}
% The material in this chapter is perhaps ideal as `bonus content,' from which a few interesting topics may be drawn, allowing some flexibility into lecture scheduling.


\section{Online Resources}
\label{sec:onlineResources}

%To supplement the material covered in this text, an online supplement is available to augment the material covered here with working code and examples.
To help readers with exercises, projects, and to collect resources 
%such as 
including
datasets and additional reading materials, 
%we have prepared an online supplement:
an online supplement is available to augment the material covered here with working code and examples:
%\done{could say more}
%
%These resources are collected on 
\begin{center}
\url{https://cseweb.ucsd.edu/~jmcauley/pml/}
%\done{make sure only https and that links work}
\end{center}

The online supplement includes:
\begin{itemize}
    \item Code examples covering the material in each chapter. These cover complete worked examples from which the code samples presented in each chapter are drawn. Additional code samples are included that 
    %reproduce 
    correspond to
    various figures and examples presented throughout the book.
    \item Links to all datasets used in the book (as well as various other personalization datasets).
    \item Links to additional reading, mostly focused on introductory material useful to learners less familiar with some of the background material described in \Cref{sec:whatcovered}.
\end{itemize}
%\todo{make sure above exists and covers all relevant items}

% \subsection{Datasets}

%\done{Table of datasets. Skip}

\section{About the Author}



\begin{wrapfigure}{r}{0.4\linewidth}
\includegraphics[width=0.99\linewidth]{fig/portrait_bw.jpg}
\caption{The Author.}
\end{wrapfigure}

%Julian McAuley has 
I have
been a
%has been an Assistant 
Professor at UC San Diego since 2014,
following postgraduate training at Stanford University, and undergraduate and graduate training in Australia.
%, and an Associate Professor since 2019. Previously he was a postdoctoral scholar at Stanford University, after completing my undergraduate and graduate training at the University of New South Wales and the Australian National University.
%\done{boring!}
%
\emph{Personalized Machine Learning} is the main theme of my research lab at UCSD. Our lab's research has pioneered the use of images and text in recommendation settings (e.g.~\cite{mcauley2015image,mcauley2013hidden}), with applications including fashion design, personalized question answering, and interactive dialog systems. Our lab has also studied personalization outside of typical recommendation settings, such as developing personalized models of heart-rate profiles \citep{ni2019modeling}, and systems for generating personalized recipes \citep{majumder2019generating}.

Our lab regularly collaborates with industry to develop state-of-the-art systems for personalized machine learning. We've worked on problems including visually-aware recommendation with \emph{Adobe} and \emph{Pinterest}, understanding user budgets and personalized price dynamics with \emph{Etsy} and \emph{Microsoft}, and question-answering and dialog systems with \emph{Microsoft} and \emph{Amazon}. We'll explore several of these approaches through case-studies throughout the book.

\section{Personalization in Everyday Life}
%\done{Chapter has little value, could fold into previous chapter}

Other than introducing the techniques underlying personalized machine learning systems, one of our goals in this book is to explore the wide range of practical applications where personalization is applied, to explore the history of the topic, and eventually to explore the associated risks and
%broad 
consequences.
%and 
%ethical concerns 
%associated with personalized machine learning.

Personalized machine learning is increasingly becoming pervasive 
%in our daily lives 
to the point that most of us are likely to interact with personalized machine learning systems every day.  Systems that generate playlists based on our listening habits, mark e-mails as `important,' suggest products or advertisements based on our recent activities, rank our newsfeeds, or suggest new connections on social media, all personalize their predictions or outputs in some way. Techniques 
%to solve these 
range from simple heuristics (e.g.~we're likely to become friends with somebody if we already share mutual friends), to complex algorithms that account for temporal patterns, or incorporate natural language and visual signals.

Below we'll study a few common (and less common) scenarios for in which personalization plays a key role, many of which will form the basis of case-studies throughout this book.
%\done{personalized recommendation is pervasive...}

% \section{The Importance of Personalization}
%\done{need to update all these section levels. Dropped}

%\subsection{E-Commerce and Advertising}
\subsection{Recommendation}

Many of the examples we cover in this book will relate to \emph{recommender systems}, and more broadly to modeling users' interactions with data collected from the web.
%on e-Commerce platforms, using data
%largely 
Part of the reason for this focus is opportunistic: user interaction datasets are widely available, allowing us to build models on top of real data.
%throughout the book.
%
%They

Pedagogically, recommender systems are also appealing as an introduction to personalized machine learning as they allow us to quickly implement working systems that are close to the state-of-the-art. As we'll see, even widely-deployed systems turn out to be surprisingly straightforward, relying on simple heuristics and 
%building on
standard data structures (\cref{sec:amazonCaseStudy}).
%\done{could be longer. No}

Ultimately though, our main reason for studying recommender systems is because they are a fundamental tool for modeling \emph{interactions between users and items}. %As such, 
The basic techniques developed when building recommender systems can be applied in a variety of other situations where we want to predict how a user will respond to some stimulus. 
%We explore a variety of related settings below.
Many of the settings we describe later build on this general theme.

% \subsection{Social and News-Media}
%\subsection{User Productivity}

%\section{Personalization Beyond E-commerce}
% \subsection{Personalization Beyond Recommendation}

%Fundamentally, the value of \emph{personalized machine learning} is to model datasets where \emph{variation among individuals} captures a large fraction of the variability in a dataset.
%\done{already said something like this? Seems okay}
%\done{above sentence doesn't really fit. Okay for now}
%
%A high degree of 
Recommender systems represent perhaps the purest 
settings where \emph{variation among individuals} captures a large fraction of the variability in a dataset.
%form of this type of variation, 
%where our goal is 
To build recommender systems we must
%to 
understand the underlying \emph{preferences} of users and \emph{properties} of items that explain why an item might be purchased by one user and not another.
%
%In recommendation scenarios, 
Users might vary due to subjective preferences, budgets, or demographic factors; 
%item prices or availability may change over time; 
both users and items might change over time due to social, temporal, or contextual factors (etc.).

%Our presentation of personalized machine learning builds 
%Variability among 


%Beyond recommendation, 
Building on the techniques we develop for recommendation, we argue that there are countless settings where 
%correctly modeling 
capturing
variation among individuals is key to making meaningful predictions.
%we argue that there are several
%individuals manifests itself in a wide variety of settings. 
%
In settings like personalized health, users may vary in terms of their physical characteristics, medical histories, or risk factors; or in settings involving natural language (or dialog) users may vary in terms of their writing styles, personalities, or their specific context.

Below we describe a few such examples, partly to highlight the wide range of settings where personalization is critical, but also to demonstrate the common set of ideas involved in modeling them.
%\done{not the best written. Is fine}

\subsection{Personalized Health}
%\done{Could drop to last chapter and flesh out a bit, or maybe not. No way}

% \emph{Personalized health}

% Note that most of these tasks are ones for which \emph{non}-personalized models would not be effective
%\done{put somewhere}

Beyond `obvious' applications in electronic commerce or social media, personalization is increasingly playing a role in high-stakes and socially-important problems. \emph{Personalized health} is a key emerging domain for personalization: like recommendation, problems in health have the key characteristic that predictions are highly contextual, and exhibit significant variation among individuals. Critically, when estimating symptoms, responses to medication, or heart-rate profiles, it would be impossible to make useful predictions \emph{without} personalization.
%in such settings \emph{non}-personalized models

Estimating what symptoms a patient will exhibit on their next hospital visit is a canonical task in personalized health, with applications in (e.g.)~preventative treatment. This task closely resembles the settings we explore when developing recommender systems, given the goal to estimate patients' interactions with certain stimuli (symptoms) over time \citep{yang2014finding}. As such, techniques for such tasks borrow ideas from recommender systems, especially temporal and sequential recommendation, as we develop in \Cref{chap:temporalSequential}.
%\done{maybe cite this again when I talk about beer paper in temporal chapter. No}

Beyond estimating patient symptoms, personalized machine learning techniques can be adapted to related tasks 
%such as 
ranging from
estimating the duration of surgical procedures
%(in order to more efficiently utilize hospital resources). Accurate prediction for such a task depends on both patient and surgeon characteristics. Successful models for this task have been built using traditional regression techniques, along with carefully-engineered patient and surgeon features
\citep{ng2017predicting},
%, resembling the types of personalization we explore in `standard' regression and classification settings in \Cref{chap:basicsReg,chap:classPipe}.
%
%Beyond regression, classification, and ranking tasks, a variety of problems in personalized health depend on more complex predictions, e.g.~forecasting sequential data. For example, techniques to 
modeling the progression of heart-rate sequences in response to physical stimuli
%Heart-rate modeling
\citep{ni2019modeling},
%or to 
or estimating
%estimate 
the distribution kinetics of drugs (such as anesthetics)
%Drug dose prediction
\citep{ingrande2020performance}.
%,
%requires complex sequential models. Modern approaches to these types of sequential modeling tasks are often based on recurrent neural networks, which are capable of learning complex semantics from sequential data; we explore these types of personalized sequential models in \Cref{sec:rnn}.
%Techniques to solve such problems range from simple regression
Modeling such problems requires understanding the characteristics of patients or physicians (and the interactions between them). Techniques range from simple regression (e.g.~to predict surgery duration) to recurrent neural networks (e.g.~to forecast heart-rate profiles).

%Clinical note generation
Many problems in personalized health also depend upon natural language data, for example modeling the characteristics of clinical notes,
%which again exhibit substantial variation as a function of the patient or physician. More complex tasks may even consist of generating notes automatically based on (e.g.)~a radiology image
or generating reports based on radiology images
\citep{ni2020learning}. Such applications build on techniques for personalized natural language processing and generation, as we develop in \Cref{chap:text}.

%\done{mention how above techniques represent the different "types" of personalized learning}

The techniques above 
%highlight 
span
the different `types' of personalized learning systems (see \cref{sec:techniques}): some systems leverage traditional machine learning techniques, in which `personalization' merely means extracting features that capture the relevant properties about users (or patients, physicians, etc.);~others use complex deep-learning approaches, in which the underlying dimensions that capture patterns in 
%user 
behavior are harder to interpret. 
%A common feature is the need to understand 
%We discuss this further in \Cref{sec:techniques}.

\subsection{Computational Social Science}

Often the goal of modeling user data is not merely to \emph{predict} future events or interactions, but to \emph{understand} the underlying dynamics at play. Using machine learning and data-driven approaches to understand the underlying dynamics of human behavior from large datasets is one of main goals of \emph{computational social science}.

Likewise, for many of the models we develop, our goals are as much about building more accurate predictors as they are about understanding social or behavioral dynamics. When we develop regressors to predict content success on \emph{reddit} (\cref{subsec:casestudyReddit}), our main goal is to disentangle what factors lead to success, such as community dynamics, titles, submission times, etc. 
%Likewise when studying price dynamics (\cref{subsec:?}), our goal is as much about understanding how users respond to price changes, 
%
%\done{could add price, even just a sentence. Tried, didn't like it}
%
Or, 
when building recommender systems
%, and especially content leveraging rich side-information such as temporal (\cref{chap:temporalSequential}) and textual (\cref{chap:text}) signals, 
our goals are to understand and interpret the underlying preference dimensions that guide users' decisions, and what causes those preferences to change over time, including how users acquire tastes, develop nostalgia for old items, or simply respond to changes in a user interface.




%reddit

%\begin{description}
% \item[Price sensitivity]
% \item[Spotify maybe]
%\end{description}

% Diversity

%Extreme content

Finally, as we begin to explore the ethical consequences of personalization (which we introduce 
%below),
in \Cref{sec:introEthics}), 
we'll underline 
%emphasize
the point that accurate prediction is rarely a desirable goal in and of itself. In \Cref{chap:consequence} we'll examine the long-term effects on users who interact with personalized systems: this includes studying
what factors drive users to extreme content
%, the importance of diversity, algorithmic bias
the roles 
how to algorithmically mitigate such undesirable outcomes.


\subsection{Language Generation, Personalized Dialog, and Interactive Agents}

Finally,
%\done{is it finally? Yes}
given the new modalities 
via which people interact with predictive systems, there are new demands for personalization.

%Beyond dialog, 
For example, 
personalization is critical in a broad range of settings involving natural language. User-generated language data exhibits substantial variability due to differences in writing style, subjectivity, etc. When dealing with such data, non-personalized models may struggle with this nuance.
%
For example, automated systems for dialog, whether in task-oriented settings or for open-domain `chit-chat,' can benefit from personalization, in order to generate responses that are more personalized or empathetic to tone or context of individual users
%
\citep{majumder2020like}. 
%

We'll see several instances of personalized language modeling throughout the book:
language models are increasingly important to explain or interpret machine predictions (\cref{subsec:textExplain}), to facilitate new modalities of interaction with predictive systems (such as conversation, \Cref{subsec:convRec}), and to develop new kinds of assistive tools, e.g.~to help users respond to e-mail (\cref{sec:caseStudySmart}).

%\cite{mao2020speech}
%probably not relevant





\section{Techniques for Personalization}
\label{sec:techniques}

As mentioned in \Cref{chap:intro}, one of the goals of this book is to establish a common narrative around the tools and techniques used to design personalized machine learning systems. Although we've shown that such systems are applied in domains as diverse as online commerce to personalized health, we find that the techniques used to implement these models follow a few common paradigms.

\subsection{User Representations as Manifolds}

One of the main ideas we'll revisit throughout this book---and which allows us to adapt ideas from recommender systems to other types of machine learning---is that of a \emph{user manifold}. That is, most of the personalized methods we'll explore will involve \emph{representations} of users that describe the common patterns of variation in their activities and interactions.

In the case of recommender systems, this `user manifold' will be a vector that describes the principal dimensions that explain variance among user preferences (\cref{fig:recsysIdea}). For example, we might discover that the principal 
%\done{Check principle / principal throughout}  
dimensions that explain variance in preferences in a movie recommendation setting
center around certain genres, actors, or special effects. 
%In another setting, 
%In a text generation setting (\cref{sec:personalTextGeneration})
%\done{not great}
%user manifolds might describe patterns of variation in users' writing styles; when estimating users' heart-rate profiles (\cref{sec:caseStudyHR}), user manifolds might describe users' fitness levels or other patterns of variation in heart-rate dynamics;
%or in a personalized fashion setting (\cref{sec:visualRec}), user manifolds might describe users' preferences toward specific visual characteristics.
%In other settings, user manifolds might describe characteristics ranging from 
%\done{says same thing as below. Removed}
%
%\done{Make sure no \Cref (lowercase c) exists}
%
%
Throughout the book, we'll revisit the idea of user manifolds, as a general-purpose means of capturing common patterns of variation among users. 
Some examples include:
\begin{itemize}
    \item In \Cref{chap:modelbasedRecsys}, we'll use low-dimensional user representations to describe the dimensions of preferences and activities, which can be used to recommend items that users are likely to interact with.
    \item In \Cref{chap:text}, user representations can describe the topics users tend to discuss (e.g.~when writing reviews), or individual characteristics of their writing styles.
    \item In \Cref{chap:visual}, user representations will describe the visual dimensions that users are interested in, allowing us to rank, recommend, or generate images in a personalized way.
    \item Throughout various case studies, user representations will capture characteristics ranging from dietary preferences (\cref{sec:recipeCaseStudy}), fitness profiles (\cref{sec:caseStudyHR}), social trust (\cref{subsec:soRec}), or fashion choices (\cref{sec:fashionCaseStudies}).
    %\done{fashionCaseStudies isn't the perfect ref. Good enough}
\end{itemize}


\begin{figure}
\centering
\includegraphics[scale=.75]{fig/recsysIdea.pdf}
\caption[Users as low-dimensional manifolds.]{The basic idea behind recommender systems, and various other types of personalized machine learning, is to represent users by \emph{low-dimensional manifolds} that describe the patterns of variance among their interactions. 
In a recommendation setting, a low-dimensional user vector might describe my \emph{preferences} while a low dimensional item vector describes that item's \emph{properties}; compatible users and items have vectors that point in the same direction (\protect\cref{chap:modelbasedRecsys}).
\label{fig:recsysIdea}}
\end{figure}

\subsection{Contextual Personalization and Model-Based Personalization}
\label{sec:contextVsPerson}

Although this book will predominantly cover methods that explicitly model user terms (as above), we will also cover a variety of models that deliberately avoid doing so.

Starting with simple 
%models 
approaches
such as `people who bought X also bought Y,'
%\done{what do I call this elsewhwere? Nothing}
many classical approaches for (e.g.)~recommendation \emph{leverage user data}, but do not include explicit parameters (i.e.,~a `model') associated with a user. However, such models are still \emph{personalized}, in the sense that different predictions will be made for each individual based on how they interact with the system. Simple machine learning techniques, such as those we develop in \Cref{chap:basicsReg,chap:classPipe}, where users are represented by a few carefully-engineered features, also follow this paradigm. 

We'll distinguish between these two classes of approach using the terms \emph{model-based} and \emph{contextual} personalization. \emph{Model-based} approaches learn an explicit set of parameters associated with each user, such as the `user manifolds' described above (and in \cref{fig:recsysIdea}); these models are typically intended to capture 
the predominant patterns of variation among users in a system, usually in terms of a low-dimensional vector.
In contrast, \emph{contextual} (also sometimes called `memory-based,' as in \Cref{chap:modelbasedRecsys}) approaches extract features from users' histories of recent interactions.

There are several settings in which contextual personalization may be preferable to explicitly modeling a user. 
%For example, 
When developing simple recommender systems in \Cref{chap:recsysIntro}, and even more trivial personalized models in \Cref{chap:basicsReg,chap:classPipe}, we see that personalization can often be achieved with simple heuristics, or hand-crafted features or similarity measures. Such approaches may be desirable for a number of reasons: simple models may be more interpretable (and therefore preferable to expose to a user compared to `black-box' predictions); or, we may lack adequate training data to learn complex representations from scratch.
%\done{Doesn't finish as punchily as it could. Fine}


\section{The Ethics and Consequences of Personalization}
\label{sec:introEthics}
%\done{rename}

Along with the increasing ubiquity of personalized machine learning systems, there is a growing awareness of the risks associated with personalization. Some of these issues have reached mainstream awareness, such as the idea that personalized recommendations can trap users in `filter bubbles,' while other issues are considerably more subtle.
%
%
For instance, considering the specific case of recommender systems, a na\"ively-implemented model can introduce issues including:
\begin{description}
    \item[Filter bubbles] Roughly speaking, recommendation algorithms rely on identifying specific item characteristics that are preferred by each user, and recommending items that most closely represent those characteristics. Without care, even a user with broad interests may be recommended only a narrow set of items that closely mimic their prior interactions.
    \item[Extremification] Likewise, a system that identifies features that a user is interested in may identify items that are most representative of those features, e.g.~a user who likes action movies may be recommended movies with \emph{a lot} of action; in contexts such as social media and news recommendation this can lead to users being exposed to increasingly extreme content (the relationship between this and the previous issue is explained in \Cref{chap:consequence}).
    \item[Concentration] Similar to the previous phenomenon, a user who has diverse interests may receive recommendations that only follow their most predominant interest (\cref{sec:filterBubble}). In aggregate, this may lead to to a small set of items being over-represented among users' recommendations. 
    \item[Bias] Given that recommenders (and many other personalized models) ultimately work by identifying common patterns of user behavior, users in the `long-tail' whose preferences don't follow the predominant trends may receive sub-par recommendations.
\end{description}

Along with a rising awareness of these issues has come a set of techniques designed to mitigate them. These techniques borrow ideas from the broader field of fair and unbiased machine learning, whereby learning algorithms are adapted so as not to propagate (or not to exacerbate) biases in training data, though the fairness goals are often quite different. 
Diversification techniques can be used to ensure that predictions or recommendations balance relevance with novelty, diversity, or serendipity;
%One set of techniques seeks 
related techniques seek
%\done{Clunky wording}
to better `calibrate' personalized machine learning systems by ensuring that predicted outputs are balanced in terms of categories, features, or the distribution over recommended items (\cref{sec:diversification}). 
%\done{correct section? Good enough} 
Such techniques can mitigate filter bubbles by ensuring that model outputs aren't highly concentrated around a few items, and more qualitatively can increase the overall novelty or `interestingness' of model outputs. Other techniques follow more directly from fair and unbiased machine learning, ensuring that the performance of personalized models is not degraded for users belonging to underrepresented groups, or who have niche preferences (\cref{sec:fairness}).

%\done{moved this section, check that it works. Looks fine}

\part{Machine Learning Primer}

%\part*{Fundamentals of Personalized Machine Learning}



%\chapter{Machine Learning Primer}
\chapter{Regression and Feature Engineering}
\label{chap:basicsReg}

In this chapter, we'll cover the fundamental principles of machine learning (and in particular \emph{supervised} learning), that will serve as a foundation for the remaining material in this book.

%Our main goal in this chapter is to develop the essential building blocks of 
In the following we'll cover essential building blocks including:
\begin{itemize}
    \item Strategies for feature extraction and transformation, including real-valued and categorical data, and temporal signals (\cref{sec:featureEngineering}).
    \item 
    %Relationship between probability and stuff
    The general strategy of
    associating probabilities with model outcomes, and more broadly
    the relationship between fitting a model and likelihood maximization (\cref{subsec:MLE}).
    \item Gradient-based approaches to model fitting (\cref{sec:gradient}), 
    %including the use of 
    and (in \cref{chap:classPipe}) their implementation via
    high-level languages such as \emph{Tensorflow} (\cref{subsec:regressionTensorflow}).
    % \item 
    %Learning pipeline
    % The end-to-end machine learning pipeline, including strategies for model selection, and how to ensure that our model will generalize to unseen data (\cref{sec:learningPipeline}).
    \item 
    %Stratigies for model evaluation
    How to deal with outliers, imbalanced datasets, and general strategies for model evaluation (\cref{sec:evaluateRegression}).
    %,sec:evaluatingClassifiers}).
\end{itemize}

Although we'll only briefly touch upon \emph{personalization}
in this chapter, 
our examples will focus on the same types of user-oriented data that we'll visit in later chapters. In particular, we'll focus on datasets covering topics such as recommendation, sentiment, and predictive tasks involving (e.g.)~demographic characteristics.

As such, the view we'll take on `personalization' in this chapter will consist of \emph{extracting features from user data} in order to make predictions using traditional machine learning frameworks. Later, we'll draw a distinction between this type of method---where we extract \emph{features} about users---and methods where we explicitly \emph{model} each user. This will drive our discussion of \emph{contextual} versus \emph{model-based} personalization (as we introduced in \Cref{sec:techniques}), though we'll discuss this distinction more precisely in 
%\Cref{chap:recsys} (\cref{sec:modelbased}). 
\Cref{chap:recsysIntro,chap:modelbasedRecsys}.
However, as we'll see in this chapter (and in various examples throughout the book), even traditional machine learning techniques, paired with appropriate feature extraction strategies, can lead to surprisingly effective models for personalized prediction.
%\done{not the tightest writing. Fine}
%\done{When do I actually clarify the distinction between types of personalization? }

\subsubsection{Supervised learning}
%\done{Is paragraph the correct level?}

All of the techniques presented in this chapter---and
most of the personalization techniques we'll explore throughout this book---are forms of \emph{supervised learning}.
%Supervised learning techniques assume that there is a 
Supervised learning techniques assume that our prediction tasks (or our datasets) can be separated into two components:
\begin{description}
\item[labels] (denoted $y$) that we would like to predict, and;
\item[features] (denoted $X$) which we believe will help us to predict those labels.\footnote{Generally we'll use $X$ when referring to a feature matrix, versus $x$ or $x_i$ to refer to a vector of features associated with a single observation.}
\end{description}
For example, given a sentiment analysis\index{Sentiment Analysis} task (\cref{chap:text}), our data might be (the text of) reviews from \emph{Amazon}\index{Amazon} or \emph{Yelp}\index{Yelp}, and our labels would be the ratings associated with those reviews.

Given this distinction between features and labels in a dataset, the goal of a \emph{supervised learning} algorithm is to infer the underlying function
\begin{equation}
    f(x) \rightarrow y
\end{equation}
that explains the relationship between the features and the labels. Usually, this function will be \emph{parameterized} by model parameters $\theta$, i.e.,
\begin{equation}
    f_\theta(x) \rightarrow y.
    \label{eq:firstEquationTheta}
\end{equation}
For example, in this chapter $\theta$ might describe which features are positively or negatively correlated (or uncorrelated) with the labels; later $\theta$ might capture the preferences of a particular user in a recommender system (\cref{chap:modelbasedRecsys}). \Cref{box:supervised} explains how this type of supervised approach relates to other types of learning.

\begin{figure}
\caption{Supervised, unsupervised, and semi-supervised learning.}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
% \hbox{
\emph{Supervised Learning} approaches are those that seek to directly learn the relationship between the observed data $X$ and the labels $y$. 
%Examples include regression and classification, as covered in this chapter.
Nearly all of the models in this book are forms of supervised learning, starting with regression and classification in this chapter, and continuing into later chapters as we build models to predict user activities.

In contrast, \emph{unsupervised learning} approaches seek to find patterns in the data $X$, but are not specifically concerned with predicting any label; examples include techniques for clustering and dimensionality reduction. 
%\done{Covered in some chapter? No}

Finally, \emph{semi-supervised learning} approaches are somewhere in between, usually leveraging large datasets of \emph{unlabeled} data to improve the performance of supervised models with a small number of labels.
\end{minipage}
}
\label{box:supervised}
\end{figure}

Throughout this chapter, we will assume that we are given \emph{labels} in the form of a vector $y$, and features in the form of a matrix $X$, so that each $y_i$ is the label associated with the $i^{\text{th}}$ observation, and $x_i$ is a vector of features associated with that observation.
%\done{data point is not a word}

%\done{summarize what we'll cover}

%\section{Regression}
%\done{Section structure? Regression could be its own chapter. Probably not}

%We'll cover two forms of supervised learning in this chapter
The two categories of supervised learning we'll cover in this and the next chapter include:
\begin{itemize}
    \item \emph{Regression}, in which 
    %the labels $y$ are real-valued, and . 
    our goal is to predict \emph{real-valued} labels $y$ as closely as possible (\cref{sec:linearReg}).
    When building personalized models in later chapter, such targets may include ratings, sentiment, the number of votes a social media post receives, or a patient's heart-rate.
    \item \emph{Classification}, in which $y$ is an element of a discrete set (\cref{chap:classPipe}). In later chapters these will correspond to outcomes such as whether a user clicks on or purchases an item. We'll also see how such approaches can be adapted to learn rankings over items (\cref{sec:errorRanking}).
\end{itemize}


\section{Linear Regression}
\label{sec:linearReg}

Perhaps the simplest association we could assume between our features $X$ and our labels $y$ would be a \emph{linear}\index{Linear model} relationship, i.e.,~that the relationship between $X$ and $y$ is defined as
\begin{equation}
  y = X\theta.
  \label{eq:linearTheta}
\end{equation}
Using our notation from \Cref{eq:firstEquationTheta}:
\begin{equation}
    f_\theta(X) = X\theta,
\end{equation}
or equivalently for a single observation $x$ (a row of $X$)
\begin{equation}
    f_\theta(x) = x \cdot \theta = \sum_i x_i \theta_i.
\end{equation}
%In the above, $X$ is a matrix describing features extracted from the data, $y$ is our vector of labels, and 
Here
$\theta$ is our set of model \emph{parameters}: a vector of unknowns which describe which features are \emph{relevant} to predicting the labels.
%\done{almost repeated sentence. Could drop this one or the one above. Removed}

Ignoring strict notation for now, a trivial example might consist of predicting a review's rating as a function of its length. To do so, let's consider a small dataset of 100 $(\text{length}, \text{rating})$ pairs from \emph{Goodreads} Fantasy novels \citep{wan2018item}.
%\done{Reference the dataset / exercise properly. Doesn't matter}
\Cref{fig:ratingLength} plots the relationship between review length (in characters) and the rating.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLength.pdf}
\caption[Ratings versus review length (\emph{Goodreads}).]{Ratings compared to review length (in characters), based on 100 reviews of fantasy novels from \emph{Goodreads}. \label{fig:ratingLength}}
\end{SCfigure}

From \Cref{fig:ratingLength}, there appears to be a (rough) association between ratings and review length, i.e.,~more positive reviews tend to be longer. A very simple model might attempt to describe that relationship with a line, i.e.,
\begin{equation}
  \text{rating} \simeq \theta_0 + \theta_1 \times (\text{review\ length}).
  \label{eq:sysOfEq0}
\end{equation}
Note that the above is just the standard equation for a line ($y = mx + b$), where $\theta_1$ is a slope and $\theta_0$ is an intercept.

If we can identify a line that approximately describes this relationship, we can use it to estimate a rating from a given review, even though we may never have seen a review of some specific length before. In this sense, the line is a simple \emph{model} of the data, as it allows us to predict labels from previously unseen features. To do so we formalize the problem of finding a \emph{line of best fit}.

Specifically, we are interested in identifying the values of $\theta_0$ and $\theta_1$ that
%correspond to a \emph{line of best fit}, i.e.,~so that the line 
most closely match the trend in \Cref{fig:ratingLength}.
%
To solve for $\theta = [\theta_0, \theta_1]$, we can write out the problem as a system of equations in matrix form:
\begin{equation}
  y \simeq X \cdot \theta,
  \label{eq:sysOfEq1}
\end{equation}
where $y$ is our vector of observed ratings and $X$ is our matrix of observed features (in this case the reviews lengths).\footnote{We write $y \simeq X \cdot \theta$ in \protect\Cref{eq:sysOfEq1} since the equation is an \emph{approximation} (i.e.,~we cannot precisely solve for $\theta$; however we will typically write $y = X \cdot \theta$ when defining model equations.} For the first few samples of our \emph{Goodreads} data we have:
\begin{equation}
  \underbrace{\left[ 
  \begin{tabular}{c} 5 \\ 5 \\ 5 \\ 4 \\ 3 \\ 5 \\ \vdots \end{tabular}
  \right]}_{\mathclap{y}}
  \simeq
  \underbrace{\left[
  \begin{tabular}{cc} 1 & 2086 \\ 1 & 1521 \\ 1 & 1519 \\ 1 & 1791 \\ 1 & 1762 \\ 1 & 470 \\ \multicolumn{2}{c}{\vdots} \end{tabular}
  \right]}_{\mathclap{X}} \cdot \underbrace{\left[ \begin{tabular}{c} $\theta_0$ \\ $\theta_1$ \end{tabular} \right]}_{\mathclap{\theta}}.
  \label{eq:sysOfEq2}
\end{equation}
It is useful to compare \Cref{eq:sysOfEq0,eq:sysOfEq2} to understand how the matrix expression above expands to include the slope ($\theta_1 \times (\text{review\ length})$) and intercept ($\theta_0$) terms. We explain this construction more precisely in \Cref{box:feature1}.

\begin{figure}
\caption{Why is there a column of `1's in the feature matrix?}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}

The first column of the feature matrix $X$ in \Cref{eq:sysOfEq2}, and in most feature matrices throughout this chapter, is a column of ones. To explain why we always have this feature, it is useful to expand the inner product $[1, \text{length}] \cdot [\theta_0, \theta_1]$ (e.g.~as in \Cref{eq:sysOfEq2}) to confirm that it expands to the equation for a line $\theta_0 + \theta_1 \times \text{length}$. Without the constant term in our feature matrix, we would be implicitly assuming that the fitted line passes through $(0,0)$.
\end{minipage}
%\done{Only moderately useful explanation}
}
\label{box:feature1}
%\done{not referenced}
\end{figure}
%\done{Are there enough boxes to keep them or delete? 12 seems enough}
%\done{caption placement in boxes. Above}

We would like to solve \Cref{eq:sysOfEq2} for $\theta$. Na\"ively, we might attempt to multiply both sides of the equation $y = X \cdot \theta$ by $X^{-1}$; however the inverse is not well-defined, since $X$ is not a square matrix.

To obtain a square matrix, we (left) multiply both sides by $X^T$:
\begin{equation}
    X^T y \simeq X^TX \theta,
\end{equation}
resulting in a square (in this case $2 \times 2$) matrix $X^TX$. We can now multiply both sides by the inverse of this matrix:
\begin{equation}
    (X^TX)^{-1}X^T y \simeq (X^TX)^{-1}(X^TX)\theta, \quad \text{or\ simply} \quad \theta = (X^TX)^{-1}X^T y.
    \label{eq:pseudoInv}
\end{equation}
The quantity $(X^TX)^{-1}X^T$ is known as the \emph{pseudoinverse} of $X$.

Computing $\theta = (X^TX)^{-1}X^Ty$ for our 100 ratings from \emph{Goodreads} yields
%\done{make sure numbers match those in figure}
\begin{equation}
    \theta = \left[ 
    \begin{tabular}{c} 
    $3.983$ \\ 
    $1.193 \times 10^{-4}$
    \end{tabular}
    \right],
    \label{eq:thetaTwo}
\end{equation}
corresponding to the line
\begin{equation}
    \text{rating} = 3.983 + 1.193 \times 10^{-4} (\text{review\ length}).
    \label{eq:reviewLength1}
\end{equation}
This line reflects a positive (albeit slight) trend between review length and ratings: for every additional character in a review, our estimate of the rating increases very slightly (by $1.193 \times 10^{-4}$ points). This line of best fit is depicted in \Cref{fig:ratingLengthBestFit}.
%(we will revisit how to interpret the parameters of linear models in \Cref{sec:interpretLinear}).

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLengthBestFit.pdf}
\caption{Line of best fit between ratings and review length (Goodreads). \label{fig:ratingLengthBestFit}}
%\done{fix alignment of text in all SCfigures. Centered, looks okay for the moment}
\end{SCfigure}

\paragraph{More complex models}
%
The above reasoning generalizes to fitting more complex models than a simple line, for example we could imagine that a rating could be related to both the length of the review and the number of comments the review received:
\begin{equation}
    \text{rating} = \theta_0 + \theta_1 \times (\text{review\ length}) + \theta_2 \times (\text{n\_comments}).
    \label{eq:threeParameters}
\end{equation}
The above process---finding a line of best fit that best approximates the relationship between our observed features $X$ and labels $y$---describes the basic concept of \emph{linear regression}.

\paragraph{Adding more dimensions}
%
Just as \Cref{eq:sysOfEq0} corresponds to fitting a line in two dimensions, \Cref{eq:threeParameters} now corresponds to fitting a plane in three. But ultimately the procedure for fitting this model remains the same. We simply have an additional column in our feature matrix:
\begin{equation}
    X = \left[
  \begin{tabular}{ccc} 1 & 2086 & 1 \\ 1 & 1521 & 1 \\ 1 & 1519 & 5 \\ 1 & 1791 & 1 \\ 1 & 1762 & 0 \\ \multicolumn{3}{c}{\vdots} \end{tabular}
  \right].
\end{equation}
Solving $\theta = (X^TX)^{-1}X^Ty$ yields
\begin{equation}
    \theta = \left[
    \begin{tabular}{c}
         3.954 \\
         $7.243 \times 10^{-5}$\\
         0.108
    \end{tabular} \right]
    \begin{tabular}{l}
      intercept\\
      slope for length\\
      slope for number of comments
    \end{tabular}.
    \label{eq:thetaThree}
\end{equation}
Interestingly, when we add this additional parameter $\theta_2$, the values of $\theta_1$ and $\theta_0$ are different from those of the model we previously fit (compare \Cref{eq:thetaThree,eq:thetaTwo}). Critically, the slope associated with the length term ($\theta_1$) is reduced in our new model. 
We discuss how to interpret these parameters in \Cref{sec:interpretLinear}.




\subsection{Regression in \emph{sklearn}}

\index{sklearn}
Various libraries support the basic machine learning techniques described in this chapter, and indeed they can be implemented relatively straightforwardly via standard linear algebra operations. Here we describe the implementation in \emph{scikit-learn},
%\done{style}
though other implementations follow similar interfaces.

\index{json}
First we load our dataset; here we read our sample (in this case a toy dataset of 100 reviews) in \emph{json} format,\footnote{\emph{Json} is a structured data format, made up of key-value pairs (where values can in turn be lists or other \emph{json} objects). See \url{https://www.json.org/}.}
%\done{italics}
which results in a list of 100 dictionaries:
%\done{json described anywhere? Added footnote}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
data = []
for l in open('fantasy_100.json'): # 100 reviews of fantasy novels from Goodreads
    d = json.loads(l)
    data.append(d)
\end{lstlisting}
\end{minipage}
%\done{Data or dataset? Looks fine}

Next we extract labels and features from the dataset. In this case we train a predictor to estimate ratings as a function of review length, as in \Cref{eq:sysOfEq0}:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
ratings = [d['rating'] for d in data] # The output we want to predict
lengths = [len(d['review_text']) for d in data] # The feature used for prediction
\end{lstlisting}
\end{minipage}

To regress on this data we must first construct our matrix of features $X$ and our vector of labels $y$; note the inclusion of a constant feature in our feature matrix:\footnote{Although in fact this can be excluded and $\theta_0$ fit by the library by setting \texttt{fit\_intercept=True}; here we include it manually.}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
X = numpy.matrix([[1,l] for l in lengths])
y = numpy.matrix(ratings).T
\end{lstlisting}
\end{minipage}

\index{sklearn}
From here regressing is simply a matter of passing our features and labels to the appropriate model from \emph{sklearn}.
%\done{italics}
%\done{sklearn or scikit-learn. sklearn}
Having done so we extract the coefficients $\theta$:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
model = sklearn.linear_model.LinearRegression(fit_intercept=False)
model.fit(X,y)
theta = model.coef_
\end{lstlisting}
\end{minipage}

Finally, we confirm manually that the pseudoinverse from \Cref{eq:pseudoInv} yields the same result:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
numpy.linalg.inv(X.T*X)*X.T*y
\end{lstlisting}
\end{minipage}

%(in both cases we find that $\theta = ?$).
In both cases we find that $\theta = (3.983, 1.193 \times 10^{-4})$, as in \Cref{fig:ratingLengthBestFit}.
%\done{fill in correct value of theta}

\section{Evaluating Regression Models}
\label{sec:evaluateRegression}


When developing the linear models above, we were somewhat imprecise about what is meant by a `line of best fit' (or generally a model of best fit). Indeed, the pseudoinverse is not a `solution' to the system of equations given in \Cref{eq:sysOfEq2}, but is merely an approximation (naturally, the line of best fit does not pass through all points exactly).

Here, we would like to be more precise about what it means for a model to be `good.' This is a key issue when fitting and evaluating any machine learning model: one needs a way of quantifying how closely a model fits the given data. Given a desired measure of success, we can compare alternative models against this measure, and design optimization schemes that optimize the desired measure directly.

\index{Mean Squared Error}
\subsection{The Mean Squared Error}
\label{sec:MSE}

A commonly used evaluation criterion when evaluating regression algorithms is called the Mean Squared Error, or MSE. The Mean Squared Error between a model $f_\theta(X)$ and a set of labels $y$ is defined as 
%\done{make sure notation is consistent with rest of chapter}
\begin{equation}
    \text{MSE}(y, f_\theta(X)) = \frac{1}{|y|} \sum_{i=1}^{|y|} (f_\theta(x_i) - y_i)^2,
    \label{eq:MSE}
\end{equation}
in other words, the average squared difference between the model's predictions and the labels. Often reported is also the \emph{Root} Mean Squared Error (RMSE), i.e.,~$\sqrt{\text{MSE}(y, f_\theta(X))}$; the RMSE is sometimes preferable as it is consistent in scale with the original labels.
%\done{check this}

With some effort, it can be shown that the linear model $f_\theta(X)$ that minimizes the MSE compared to the labels $y$ is given by using the pseudoinverse as in \Cref{eq:pseudoInv}. We leave this as an exercise (Exercise~\ref{ex:hardPseudoInv}).
%\done{write this exercise and reference}

\index{Mean Squared Error}
\subsection{Why the Mean Squared Error?}
\label{sec:whyMSE}

Although the Mean Squared Error has a convenient relationship with the pseudoinverse, it may otherwise seem a somewhat arbitrary choice of error measure. For instance, it may seem more obvious at first to compute an error measure such as the Mean Absolute Error (or MAE):
\begin{equation}
    \text{MAE}(y, f_\theta(X)) = \frac{1}{|y|} \sum_{i=1}^{|y|} | f_\theta(x_i) - y_i |
    \label{eq:MAE}
\end{equation}
Or, why not count the number of times the model is wrong by more than one star? Or why not measure the mean \emph{cubed} error?

To defend the Mean Squared Error as a reasonable choice, we need to characterize what types of errors are more `likely' than others. Essentially, the Mean Squared Error 
%is assigning 
assigns
very small penalties to small errors, and \emph{very} large penalties to large errors. This is in contrast to, say, the Mean Absolute Error, which assigns penalties 
%exactly proportional 
precisely in proportion
to how large the error is. What the Mean Squared Error therefore seems to be assuming is that small errors are \emph{common} and large errors are particularly uncommon.

What we are talking about informally above is a notion of how errors are \emph{distributed} under some model. Formally, we say that the labels are equal to our model's predictions, plus some error:
\begin{equation}
    y = \underbrace{f_\theta(X)}_{\text{prediction}} + \underbrace{\epsilon}_{\text{error}},
\end{equation}
and that our error follows some probability distribution. Our argument above said that small errors are common and large errors are very rare. This suggests that errors may be distributed following a \emph{bell curve}, which we could capture with a Gaussian (or `Normal') distribution:
\begin{equation}
    \epsilon \sim \mathcal N(0, \sigma^2).
\end{equation}
The density function for a (zero mean) Gaussian distribution is given by
\begin{equation}
    f'(x') = \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{x'}{\sigma}\right)^2}
\end{equation}
(we use the notation $f'$ and $x'$ to avoid confusion with $f$ and $x$ elsewhere). So, the probability density for an error 
%as large as 
of size
$y_i - f_\theta(x)$ is given by
\begin{equation}
    \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y - f_\theta(x)}{\sigma}\right)^2}.
    \label{eq:errorDensity}
\end{equation}
This density function is depicted in \Cref{fig:gaussianDensity}.
%\done{squash plot so that it's same as others? It's fine, at least one other is the same}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/gaussianError.pdf}
\caption{Gaussian error density. \label{fig:gaussianDensity}}
\end{SCfigure}

\subsection{Maximum Likelihood Estimation of Model Parameters}
\label{subsec:MLE}
%\done{subsection or subsubsection? organization could be better. or could add some gluing text above. Fine}

%Now that we have this type of density function in mind, we can begin to reason more formally about what it means for a particular model to be a `good' fit to the data.
Having defined the density function above, we can now reason more formally about about what it means for a particular model to be a `good' fit to the data. In other words, we would like to ask how \emph{likely} a particular model is in terms of a given error distribution. %For example, are the errors we observe consistent with a bell curve shape? 
%\done{cut this text up a lot, check. Looks fine}

Specifically, the density function in \Cref{eq:errorDensity} gives us a means of assigning a probability (or likelihood) to a particular set of labels $y$, given features $X$ and a model $\theta$, under some particular error distribution (in this case a Gaussian):
%\done{mouthful. Fine}
\begin{equation}
    \mathcal L_\theta(X | y) = \prod_{i = 1}^{|y|} \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}.
    \label{eq:likelihood1}
\end{equation}
Essentially, we want to choose $\theta$ so as to maximize this likelihood.
% Something else here?
Our goal intuitively is to choose a value of $\theta$ that is consistent with this error distribution, i.e.,~a model that makes many small errors and few large ones.

Precisely, we would like to find
%\begin{equation}
    $\argmax_{\theta} \mathcal L_\theta(X | y)$.
%\end{equation}
This procedure (finding a model $\theta$ that maximizes the likelihood under some error distribution is known as \emph{Maximum Likelihood Estimation}.
We solve by taking logarithms and removing irrelevant terms ($\pi$, $\sigma$):
\begin{align}
  \argmax_\theta \mathcal L_\theta(X | y) & = \argmax_\theta \ell_\theta(X | y)
  \label{eq:MSEMLEderiv0}
  \\
  & = \argmax_\theta\log \prod_{i = 1}^{|y|} \frac{1}{\sigma\sqrt{2\pi}}e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}\\
  & = \argmax_\theta \sum_i \log e^{-\frac{1}{2}\left(\frac{y_i - f_\theta(x_i)}{\sigma}\right)^2}\\
  & = \argmax_\theta -\sum_i (y_i - f_\theta(x_i))^2\\
  & = \argmin_\theta \sum_i (y_i - f_\theta(x_i))^2\\
  & = \argmin_\theta \frac{1}{|y|} \sum_i (y_i - f_\theta(x_i))^2
  \label{eq:MSEMLEderiv}
\end{align}
Note crucially in the above equation that \emph{the maximum likelihood solution for $\theta$ under our Gaussian error model is precisely the Mean Squared Error}. This demonstrates the relationship between the Mean Squared Error and Maximum Likelihood Estimation (which we summarize in 
%\Cref{box:MSEMLE}).
\Cref{box:MSEMLE}).
%\done{text here, and in the box, is a bit clunky, try rewriting}

\begin{figure}
\caption{The MSE and the MLE.}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
The argument we made in \Cref{sec:whyMSE} explained our motivation behind the choice of the Mean Squared Error (MSE): by choosing the MSE as our error metric, we are implicitly assuming that our model's errors follow a Gaussian distribution. This assumption is explained by the fact that minimizing the MSE maximizes the likelihood of the observed errors under a Gaussian error model.
\end{minipage}
}
\label{box:MSEMLE}
\end{figure}

The above arguments may seem like just a mathematical curiosity, and indeed in practice we will often minimize the Mean Squared Error without scrutinizing the decision to do so. But this relationship between error functions and probabilities will come up regularly when we develop models for classification (\cref{chap:classPipe}), sequence mining (\cref{chap:temporalSequential}), and recommender systems (\cref{chap:modelbasedRecsys}). To summarize a few key points:
\begin{enumerate}
    \item When we optimize a certain error criterion, we are often making implicit assumptions about how errors are distributed.
    \item Sometimes, a model will poorly fit a dataset because these assumptions are violated. Understanding the underlying assumptions gives us a chance to diagnose them and attempt to correct it (\cref{subsec:errorsArent}).
    %\done{below is paragraph, so can't be referenced. Is subsection}
    \item In many of the models we fit later (including when we develop classifiers in \Cref{chap:classPipe}), 
    %\done{which? Done} 
    we will use this style of probabilistic language, i.e., we will talk about some observed data having high \emph{likelihood} under some model. Fitting such models will use this same strategy of selecting a model which maximizes the corresponding likelihood.
\end{enumerate}

\subsection[The $R^2$ Coefficient]{The $\boldsymbol{R}^{\boldsymbol{2}}$ Coefficient}
\label{subsec:r2coeff}

Having motivated our choice of the Mean Squared Error at some length, it is worth asking how low the MSE should be before we consider our model to be `good enough'?

This quantity turns out not to be well-defined: the Mean Squared Error will depend on the scale and variability of our data, and the difficulty of our task. For example, predicted ratings on a 5-point scale would likely have lower MSEs than predicted ratings on a 100-point scale; on the other hand, this might not be the case if ratings on a 100-point scale were highly concentrated (e.g.~nearly all ratings were in the 92-95 range). Finally, the MSE in either setting could be higher simply due to a lack of available features that allow us to predict ratings accurately.

As such, we would like a calibrated measurement of model error. As we just argued, the MSE is related to the \emph{variance} of the data: this relationship is easy to see as follows:
%\done{Why is this MSE(y|X) and not MSE(f|y,X)? Done}
\begin{align}
    \bar{y} & = \frac{1}{|y|} \sum_i y_i\\
    \mathrm{var}(y) & = \frac{1}{|y|} \sum_i (y_i - \bar{y})^2\\
    %\mathrm{MSE}(f_\theta;y,X) & = \frac{1}{N} \sum_i (y_i - f(x_i))^2
    \mathrm{MSE}(y,f_\theta(X)) & = \frac{1}{|y|} \sum_i (y_i - f(x_i))^2
\end{align}
In other words, the Mean Squared Error would be equal to the variance if we had a trivial predictor that always estimated $f(x_i) = \bar{y}$.\footnote{Note that this is the best we could do if using a trivial predictor of the form $f(x_i) = \theta_0$ (Exercise~\protect\ref{ex:MSEtrivial}).} Thus the variance might be used as a way of normalizing the Mean Squared Error:
\begin{equation}
    %\mathrm{FVU}(f_\theta;y,X) = \frac{\mathrm{MSE}(f_\theta;y,X)}{\mathrm{var}(y)}
    \mathrm{FVU}(y,f_\theta(X)) = \frac{\mathrm{MSE}(f,f_\theta(X))}{\mathrm{var}(y)}
    \label{eq:FVU}
\end{equation}
This quantity, known as the \emph{Fraction of Variance Unexplained} essentially measures the extent to which the model \emph{explains variability} in the data, as compared to a predictor which always predicts the mean (i.e.,~one which explains no variability at all).

This quantity will now take a value between 0 and 1: 0 being a perfect classifier (MSE of zero), and 1 being a trivial classifier.\footnote{The FVU \emph{could} be greater than 1, if our classifier were \emph{worse} than a trivial one.}

Often, one reports the \emph{$R^2$ coefficient}, which is simply one minus the FVU:
\begin{equation}
    R^2 = 1 - \frac{\mathrm{MSE}(y,f_\theta(X))}{\mathrm{var}(y)},
\end{equation}
which now takes a value of 1 for a perfect predictor, and 0 for a trivial predictor. The name `$R^2$' comes from a different way of deriving the same quantity, in terms of the correlation between the predictions and the labels.\footnote{We omit this alternate derivation for now, but revisit the idea of correlation briefly in \protect\Cref{sec:pearsonSim}.}



\subsection{What to do if Errors \emph{Aren't} Normally Distributed?}
\label{subsec:errorsArent}

\index{Mean Squared Error}
Our arguments 
%we made 
above characterized the relationship between the Mean Squared Error and the normal (Gaussian) distribution. In summary, the MSE 
%\done{consistently abbreviate} 
is a reasonable choice so long as our model errors are expected to be centered around zero, and not to have large outliers.

But what can we do if these assumptions do not hold? First, we consider how to validate the assumptions in the first place. Recall that our 
%assumptions assume 
basic assumption asserts
that the residuals
\begin{equation}
    r_i = y_i - f_\theta(x_i)
    \label{eq:qq}
\end{equation}
follow a normal distribution. To begin with, a simple plot may reveal whether the residuals follow the desired overall trend.

\Cref{fig:qq} (left) shows a histogram of residuals $r_i$ for a simple prediction task, in which we estimate review lengths as a function of user gender (covered later in \Cref{sec:binaryFeat}).
Although the plot has a slight bell shape, it deviates from the normal distribution in several key ways, for instance:
\begin{itemize}
    \item The residuals do not appear to be centered around zero. In fact the \emph{average} residual is zero,\footnote{In fact, the average residual of this type of linear regression model is \emph{always} zero (see Exercise~\protect\ref{ex:residual0}).} though the largest bins in the histogram are somewhat below zero.
    \item The are some large outliers (i.e.,~extremely long reviews whose length was underpredicted).
    \item There are no small outliers, and there is almost no `left tail,' i.e.,~the model never significantly overpredicts.
\end{itemize}

\begin{figure}
\begin{center}
\includegraphics[scale=0.75]{fig/qqhist.pdf}
\includegraphics[scale=0.75]{fig/qq.pdf}
\end{center}
\caption[Residuals versus theoretical quantiles under a normal distribution.]{Histogram of observed residuals (left), and residuals compared to theoretical quantiles under a normal distribution. \label{fig:qq}}
\end{figure}

Although the histogram in \Cref{fig:qq} allows us to quickly assess whether the residuals follow a normal distribution, this can be visualized more precisely by comparing the theoretical quantiles of a normal distribution to the observed residuals, as in \Cref{fig:qq}, right.\footnote{This type of diagnostic plot can be generated easily with a library function, e.g.~this one was generated with \texttt{scipy.stats.probplot}.} The plot essentially compares the (sorted) residuals to those we would expect if we were to sample the same number of values from a normal distribution: if our residuals followed a normal distribution, plotting these quantities against each other would result in a straight line. Again, the plot basically reveals that there is an unusual outlier, and that residuals are missing the left tail (i.e., overpredictions) that would be expected. Note that this same type of diagnostic tool can be used to compare our residuals against any hypothetical distribution in the same way.

While the above is merely a diagnostic for determining \emph{whether} the residuals followed a normal distribution, the more difficult question is how these discrepancies can be corrected. Some general guidelines are as follows:
\begin{description}
    \item[Remove outliers] The normal distribution (and thus the MSE) is especially sensitive to outliers due to how it penalizes large errors. To the extent that extremely long reviews do not conform to the usual behavior of the data, we could simply discard them before training.
    \item[Choose an error model less sensitive to outliers] The Mean Absolute Error (for example) assigns a smaller penalty to large mispredictions, so outliers will have a smaller effect on the model.
    \item[Choose a skewed distribution] In this example we are predicting a length, which by definition is bounded below (at length zero) but not above. Thus there will be a long tail of underpredictions, but not large overpredictions. We might account for this by modeling the data using a skewed probability distribution (such as a Gamma distribution).
    \item[Fit a better model] Note that the diagnostic in \Cref{fig:qq} is a function of the \emph{errors}, rather than the original data. Thus, for example, if we had a feature that allowed us to correctly predict the length of the unusually long review, the errors may 
    %again 
    become more consistent with a normal distribution.
\end{description}

Again, 
%most of the time 
the Mean Squared Error is generally a safe and reasonable choice, and can be used without too much scrutiny.
%\done{Feel like I wrote this text elsewhere. Fine}
Nevertheless it is useful to have a sense of its underlying assumptions so that one can detect when they have been violated.

\section{Feature Engineering}
\label{sec:featureEngineering}

Along with the simple linear function relating features to labels as in \Cref{eq:linearTheta} come significant limitations in terms of what kinds of relationships can be modeled with linear regression techniques. 
%For instance, when trying to model features like gender (categorical), temporal information (periodic)
When modeling asymptotic, periodic, or other non-linear relationships between features and labels, it is not yet clear how this can be accomplished given the limitations of this type of model.

As we shall see, complex relationships can be handled within the framework of linear models, so long as we exercise care by appropriately transforming our features (and labels). In practice, the success or failure of our models will often depend on carefully processing our data to help the model uncover the most salient relationships.  This process of \emph{feature engineering} proves critical even when developing deep learning models based on images or text: in spite of the vague promise of learning complex non-linear relationships automatically, extracting meaningful signals from data is often a matter of careful engineering, rather than selecting a more complex model.

\subsection{Simple Feature Transformations}
\label{sec:simpleTransforms}

The first model we fit in \Cref{eq:sysOfEq0} revealed a positive association between review length and ratings. However, fitting the data with a line (\cref{fig:ratingLengthBestFit}) does not seem to fit the data very accurately. Fitting the data with a line seems limiting, given that the trend may be better captured by a polynomial or asymptotic function (since the rating cannot grow above five stars).

Na\"ively, we might think that this is a fundamental limitation of linear models. Note however that the assumption of linearity in $\theta$ (\cref{eq:linearTheta}) does not prevent us from fitting (for example) a polynomial function. The polynomial equation
\begin{equation}
  \text{rating} = \theta_0 + \theta_1 \times (\text{review\ length}) + \theta_2 \times (\text{review\ length})^2
  \label{eq:sysOfEqPoly}
\end{equation}
is linear in $\theta$, even though we have transformed the input features in $X$.

This idea can be applied straightforwardly to fit polynomial functions, as shown in \Cref{fig:cubicFit}.\footnote{Actually, these curves were generated using the feature $\frac{\text{length}}{1000}$, as the matrix inverse $(X^TX)^{-1}$ becomes numerically unstable given large values of $(\text{length})^3$.}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/ratingLengthBestFitQuad.pdf}
\caption{Quadratic and cubic polynomials of best fit. \label{fig:cubicFit}}
\end{SCfigure}

\subsection{Binary and Categorical Features: One-Hot Encodings}
\label{sec:binaryFeat}

So far we have dealt with regression problems where we have both real-valued inputs (features $X$), and real-valued outputs (labels $y$). What can we do in cases where features are binary or categorical?

As an example, let's consider whether the length of a user's review can be predicted by (or more simply, is related to) their gender. To do so, we'll look at a different dataset (of a few hundred beer reviews \citep{mcauley2012learning}) that includes the gender of its users.

That is, we'd like a model of the form:
\begin{equation}
    \text{length} = \theta_0 + \theta_1 \times \text{gender}.
    \label{eq:genderModel}
\end{equation}
Obviously, gender (represented in this dataset as a string) is not a numerical quantity, so we need some appropriate \emph{encoding} of the gender variable.

For the moment, let's treat gender as a binary variable. We'll relax this assumption in a moment to allow for a non-binary gender variable (and allow for the possibility that the gender is missing, as it can be in this dataset), but for the moment let's encode the gender variable as:
\begin{equation}
    \text{Male} = 0; \quad \text{Female} = 1.
    \label{eq:genderEncoding1}
\end{equation}
Alternately, this is just a binary indicator specifying whether this user is female. This encoding, although only one of a few we might have used, allows us to fit a linear model and estimate the values of $\theta_0$ and $\theta_1$. The model we fit (after removing users who did not specify a gender) is
\begin{equation}
    \text{length (in words)} = 127.07 + 8.76 \times ( \text{user is female} ).
\end{equation}
With a little thought, we can interpret the model parameters as indicating that, on average, females write slightly longer reviews (by 8.76 words) compared to males. Note that 127.07 is not the population average, but rather the average for males (whose gender feature is zero).
%; we will revisit how to interpret the parameters of these models in \Cref{sec:interpretLinear}.

A scatter plot of the data (i.e., the encoded gender attribute and the review lengths), as well as the line of best fit above is depicted in \Cref{fig:genderLength}. Note that although we have fit the data with a line (\cref{fig:genderLength}, left), the actual feature values only occupy two points (0 and 1); thus the fit is perhaps better represented with a bar plot (\cref{fig:genderLength}, right). 
%Note the fact that the model would predict (for example) a length of 1,000 words if the gender feature took a value of 100 does not invalidate it, as such values are never observed.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/genderVlength.pdf}
\includegraphics[scale=0.75]{fig/genderVlengthBar.pdf}
\caption[Gender versus review length (beer data).]{Gender versus review length (beer data). Visualized via a line of best fit (left) and a bar plot (right). \label{fig:genderLength}}
\end{SCfigure}

\subsubsection{Categorical features}
%\done{Replace various paragraphs with subsubsections}

%\done{Probably this should be a subsection. It's fine}
In practice, the gender attribute may assume more than binary labels in some datasets. To accommodate this, we might na\"ively imagine extending our encoding from \Cref{eq:genderEncoding1} to include additional values:
%\done{Four category gender figure shouldn't have axis label `gender?'}
\begin{equation}
\begin{tabular}{ll}
    Male = & 0;\\
    Female = & 1;\\
    Other = & 2;\\
    Not specified = &3;\\
    etc.
    \end{tabular}
    \label{eq:oneHot1}
\end{equation}
Again we fit the same model as in \Cref{eq:genderModel}. Doing so we might obtain a fitted model like the one in \Cref{fig:genderFit4} (left).

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/genderVlengthBar4.pdf}
\includegraphics[scale=0.75]{fig/genderVlengthBar4oneHot.pdf}
\caption[Sequential versus one-hot encodings.]{Categorical features with a na\"ive sequential encoding (left), and a one-hot encoding (right). \label{fig:genderFit4}}
\end{SCfigure}

Note that the model fit in \Cref{fig:genderFit4} (left) implicitly makes some dubious assumptions. For example, because the model is linear, it assumes that the difference between `male' and `female' lengths is \emph{the same as} the difference between `female' and `other' lengths.\footnote{That is, males receive the prediction $\theta_0$; females receive $\theta_0 + \theta_1$; other receives $\theta_0 + 2\theta_1$, etc.}

This assumption is not supported by the data, and in fact would be different if we simply reordered our indices in \Cref{eq:oneHot1}. Rather, we would like to associate different predictions to members of each group, as in \Cref{fig:genderFit4} (right). This can be achieved via a different encoding:
\begin{equation}
    \begin{tabular}{lll}
      Male = & $[0,0,0]$\\
      Female = & $[0,0,1]$\\
      Other = & $[0,1,0]$\\
      Not specified = & $[1,0,0]$\\
      etc.
    \end{tabular}
    \label{eq:oneHot2}
\end{equation}
%Using the above encoding, 

We can quickly confirm that the model would make predictions as follows:
\begin{equation}
    \begin{tabular}{ll}
    Male: & $y = \theta_0$\\
    Female: & $y = \theta_0 + \theta_1$\\
    Other & $y = \theta_0 + \theta_2$\\
    Not specified & $y = \theta_0 + \theta_3$\\
    etc.
    \end{tabular}
\end{equation}
That is, $\theta_0$ is the prediction for males, $\theta_1$ is the \emph{difference} between females and males, etc. Note that we now have four parameters to estimate four values, as opposed to two parameters as in \Cref{eq:oneHot1}. As such, the model has sufficient flexibility to make different estimates for each group, as in \Cref{fig:genderFit4} (right).

This type of encoding, in which we have a separate feature dimension for each category, is called a \emph{one-hot} encoding. 
%\done{is this the first time I mention? Yes}

Note that to represent four categories in \Cref{eq:oneHot2} we only used \emph{three}-dimensional features (or in general, for $N$ categories, we could use an $(N-1)$-dimensional encoding. Possibly this seems slightly confusing compared to using a four-dimensional feature vector (e.g.~$\text{Male} = [0,0,0,1]$, etc.). Two reasons for using an $(N-1)$ dimensional feature vector are as follows:
\begin{enumerate}
  \item Doing so is not necessary; together with $\theta_0$, the representation in \Cref{eq:oneHot2} uses four parameters to predict four values, so adding an additional dimension would add no more expressive power to the model and would be redundant.
  \item Doing so could possibly be harmful. While adding redundant features seems harmless, in practice doing so means the system 
  %of equations 
  in \Cref{eq:sysOfEq1} would no longer have a unique solution, as the matrix $X^TX$ would be uninvertable.
  %(\cref{exercise_?}).
\end{enumerate}

Similarly, a \emph{multi-hot} encoding can be used in cases where an instance can belong to multiple categories simultaneously, for example for a `race' feature, a user may associate with multiple racial groups
%\done{crappy example. it'll do}
(note that this is equivalent to a concatenation of several binary features).

\subsection{Missing Features}
\label{sec:missingFeat}

Often datasets will have features that are missing, for example the underlying data used for the example in \Cref{sec:binaryFeat} consisted of a gender attribute that many users may leave unspecified.

When dealing with binary or categorical features we dealt with these missing values quite straightforwardly---we simply treated `missing' as an additional category.

But if a continuous feature, such as a user's age or income, were missing, we must think harder about how to handle it. Trivially, we might simply discard instances with missing features, though this strategy will harm model performance if it means discarding a substantial fraction of our data.

Alternately we might replace the missing entries by the average (or mode) value for that feature; this strategy is known as \emph{feature imputation}. This may be more effective than discarding the feature, but may also introduce some bias, as (for example) users who choose to leave a feature unspecified may be quite different from the average or mode.

To avoid the above issues, we would like a strategy that uses features when they are available, but makes separate predictions for those users when they are not. This can be achieved via the following strategy: for any feature $x$ which is sometimes missing, replace it by two features $x'$ and $x''$ as follows:
\begin{equation}
    x' = \left \lbrace
    \begin{tabular}{ll}
      1 & if feature is missing\\
      0 & otherwise
    \end{tabular} \right. , \quad
    x'' = \left\lbrace
    \begin{tabular}{ll}
      0 & if feature is missing\\
      $x$ & otherwise
    \end{tabular} \right. .
\end{equation}
Following this parameters can be fit within a model as usual:
\begin{equation}
    y = \theta_0 + \theta_1 x' + \theta_2 x''.
\end{equation}
The above representation may seem somewhat arbitrary, but makes sense once we expand the expression for missing and non-missing features. E.g.~when a feature is available predictions are made according to
\begin{equation}
    y = \theta_0 + \theta_2 x,
\end{equation}
whereas when a feature is missing predictions are made according to
\begin{equation}
    y = \theta_0 + \theta_1.
\end{equation}
This achieves the desired effect: when the feature is available we predict as normal, and when the feature is unavailable we predict using a learned value ($\theta_1$). Note that this strategy is very similar to feature imputation, but rather than using a heuristic imputation strategy, the model will directly learn what is the best prediction to impute.

\subsection{Temporal Features}
\label{sec:temporalFeatures}

Temporal features may make excellent predictors in various settings. Outcomes such as ratings, clicks, purchases (etc.)~are often influenced by factors such as the day of the week, the season, or long-term trends that span several years.

Let's explore an example in which we try to predict the rating of a book on \emph{Goodreads} based on the day of the week that it was entered. Average ratings for each weekday\footnote{Again based on a small sample of reviews from the Fantasy genre.} are shown in \Cref{fig:temporalFit}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/timePlot.pdf}
\caption{Ratings as a function of the weekday, and line of best fit. \label{fig:temporalFit}}
\end{SCfigure}

As before, we might try to describe this relationship using a line, i.e.,~to fit a model of the form 
%\done{make sure use of $=$ and $\simeq$ is consistent throughout this section. Looks okay} 
\begin{equation}
    \text{rating} = \theta_0 + \theta_1 \times (\text{day of week}).
    \label{eq:fitDoW}
\end{equation}
For this equation to make sense, we need to map the day of the week to a numeric quantity. A trivial encoding might assign numbers sequentially, e.g.
\begin{equation}
    \text{Sunday} = 1; \quad \text{Monday} = 2; \quad \text{Wednesday} = 3; \quad \text{etc.}.
\end{equation}
Fitting \Cref{eq:fitDoW} using this representation yields the line of best fit depicted in \Cref{fig:temporalFit}, which reveals a slight upward trend as the days of the week progress.

The linear trend in \Cref{fig:temporalFit} seems a fairly poor fit to the data; we might think about fitting more complex functions like polynomials to better capture the observed data. But consider that our model is essentially periodic: Sunday (represented by a 1) follows Saturday (represented by a 7), though we could just as easily have represented Wednesday as 1 and Tuesday as 7. These choices seem arbitrary, but they impact our model in unexpected ways.

The above point is perhaps clearer if we visualize our model's predictions over a period of two weeks, as in \Cref{fig:periodic}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/timePlotPeriodic.pdf}
\caption[Attempt to fit periodic data with a linear model.]{If we consider that our weekly measurements are periodic, we realize that fitting periodic data with a linear trend seems unrealistic. \label{fig:periodic}}
\end{SCfigure}

We'll revisit the critical role of temporal dynamics in \Cref{chap:temporalSequential}, in particular within the context of recommendation
%the context of recommendation 
in \Cref{sec:caseStudyNetflixTemporal}.
%\done{check this, add others}

\subsection{Transformation of Output Variables}
\label{subsec:transformOutput}

Finally, just as we saw how to transform features in \Cref{sec:simpleTransforms}, we can also transform our \emph{output} variables.

For example, let's consider fitting a model to determine whether resubmitted posts on \emph{reddit} \citep{lakkaraju2013s}
%\done{Cite datasets somehow? Probably not}
receive lower numbers of upvotes, i.e.,%
%long reviews on \emph{Steam} tend to be associated with games with a long play time, i.e.,
% \begin{equation}
%     \text{time played (hours)} = \theta_0 + \theta_1 (\text{review length}).
% \end{equation}
\begin{equation}
    \text{upvotes} = \theta_0 + \theta_1 \times (\text{submission number})
    \label{eq:reddit1}
\end{equation}
(where the `submission number' is `1' for an original submission, `2' for the first resubmission, etc.).
This model, along with the observations on which it is based, are shown in \Cref{fig:redditResub} (left).

% \begin{figure}
%     \centering
%     \includegraphics[scale=0.75]{fig/lengthPlayed.pdf}
%     \includegraphics[scale=0.75]{fig/lengthPlayedLog.pdf}
%     \caption{Review length versus hours played (left), and logarithm of hours played (right), from \emph{Steam} video game reviews. Lines of best fit for both plots are included.}
%     \label{fig:lengthPlayed}
% \end{figure}

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/redditUp.pdf}
    \includegraphics[scale=0.75]{fig/redditUpLog.pdf}
    \caption[Upvotes versus submission number on \emph{reddit}.]{Number of upvotes versus submission number on \emph{reddit}. The left plot shows the original data (with averaged upvote counts), the right plot shows the logarithm of the number of upvotes. Lines of best fit for both plots are included.}
    \label{fig:redditResub}
\end{figure}

Although the line of best fit indicates a slight downward trend, it does not appear to correspond closely to the overall shape of the data. 
%Part of the issue is that the data contains numerous \emph{outliers} (i.e.,~games with extremely large `time played' values), that the model is unable to reliably predict; as we discussed in \Cref{sec:whyMSE}, such large outliers do not fit the assumptions made when taking our line of best fit, and ultimately spoil the model's ability to capture a meaningful trend.
%\done{Write this, or maybe not. Dropped above, rewrote below}
Eye-balling the data in \Cref{fig:redditResub}, we might hypothesize
%Possibly we might imagine 
that the data follows an exponentially decreasing trend, e.g.~every time you resubmit a post, you can expect to receive half as many upvotes.

Again, one might assume that this type of trend is something that cannot be captured by a linear model. But in fact we can possibly address this by transforming the output variable $y$. For example, consider fitting
\begin{equation}
    \log_2(\text{upvotes}) = \theta'_0 + \theta'_1 (\text{submission number})
    \label{eq:reddit2}
\end{equation}
%(note that we add a constant to ensure that the argument of the logarithm is always positive).
Now, a unit change in the prediction corresponds to a post receiving \emph{twice} as many upvotes.
%a game being played \emph{twice} as long. 
While this is still a linear model, the model corresponds to fitting
\begin{equation}
    \text{upvotes} = 2^{\theta'_0 + \theta'_1 (\text{submission number})}.
    \label{eq:reddit3}
\end{equation}
The transformed data and line of best fit are shown in \Cref{fig:redditResub} (right). 

Arguably, this second line better captures the overall trend, and does not have the same issues with outliers. 
%
%In fact, i
If we transform the fitted values from \Cref{eq:reddit2} back to their original scale via \Cref{eq:reddit3}, the transformed values actually have a Mean Squared Error about 10\% lower than the model from \Cref{eq:reddit1}, indicating that the transformed data more closely follows a linear trend compared to the untransformed data.
%\done{mouthful}
%compute the Mean Squared Error of the models in \Cref{eq:reddit1} and \Cref{eq:reddit2}, the model from \Cref{eq:reddit3} has 



\section{Interpreting the Parameters of Linear Models}
\label{sec:interpretLinear}

When analyzing the linear models developed so far, we have already talked about interpreting their parameters in terms of general trends, correlation, differences between groups, etc.

While is tempting to casually interpret the meaning of various features, we must be careful and precise when doing so.

First, we should be precise about the interpretation of our slope and intercept terms. For example, when we modeled ratings as a function of review length (\cref{eq:reviewLength1}), we stated that under our model, ratings increased fractionally ($1.193\times 10^{-4}$) for every character of a review.

This interpretation makes sense given a model containing only a single features, but as soon as we incorporate multiple features we must be more careful. Consider e.g.~the model from \Cref{eq:thetaThree}, in which we included both the length and number of comments as predictors. We could no longer state that under this model, the rating increases (by $7.243 \times 10^{-5}$) for every character in the review. Precisely, we must interpret the parameters as follows: \emph{Our prediction of the rating increases by $7.243\times 10^{-5}$ for every character in the review, assuming the other features remain unchanged}. This definition is stated precisely in \Cref{box:interpretLinear}.

\begin{figure}
\caption{Interpreting the parameters of linear models. \label{box:interpretLinear}}
%\done{Capitalize captions in boxes or not? Not}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
Given a linear model $y = X\theta$ we should interpret a parameter $\theta_k$ as follows:

\begin{center}
\parbox{0.9\linewidth}{
For every unit change in $x_{ik}$, our prediction of the output $y_i$ would increase by $\theta_k$, \emph{if all other feature values remain fixed}
}
\end{center}

It is important to note that we are talking about the model's \emph{prediction} (rather than an actual change in the label), which could change if different features were included. And we must include the condition that other features remain constant, without which we would fail to account for the potential correlations among different features.
\end{minipage}
}
\end{figure}
%\done{none of these boxes fit...}
%\done{formatting of text in box}

Critically, features like review length and number of comments may be highly \emph{correlated} (e.g.~we may rarely see longer reviews without also seeing more comments). For example, when incorporating features based on polynomial functions (as in \cref{eq:sysOfEqPoly}), or when dealing with one-hot encodings (as in \cref{eq:oneHot1}), a feature \emph{cannot} change without the other features changing.

%\done{clean up writing a little. Good enough}
Second, we should be clear when interpreting parameters that we are talking about \emph{predictions} under a particular model rather than actual changes in the output variable $y_i$. These predictions can change as we include additional features; a feature that had previously been predictive may become less so in the presence of another (as we saw in \Cref{eq:thetaThree}). Likewise, we should be careful not to conclude that (e.g.)~length is \emph{not} related to the output variable, simply because another correlated feature 
%is \emph{better} correlated.
has a stronger relationship.

Finally, we should be careful not to make statements about the \emph{causal} effect of features on the output variable. Our line of best fit does not state that long reviews `cause' positive opinions any more than it states that positive opinions cause long reviews.

% Second, we should note that the 
% Consider a trivial example in which we predict a student's GPA as a function of their commute time:
% \begin{equation}
%     \mathrm{GPA} = 3.1 - 0.15 \times (\text{commute time to school in hours}).
%     %\done{Could try a real example. Deleted}
% \end{equation}
% From this one might casually conclude that long commutes contribute to lower GPAs. However if we fit a model that includes a third feature, hours employed, we might instead fit
% \begin{equation}
%     \mathrm{GPA} = 3.2 - 0.02 \times (\text{commute time}) - 0.08 \times (\text{hours worked})
% \end{equation}


\section{Fitting Models with Gradient Descent}
\label{sec:gradient}

So far, when solving regression problems,
%in the previous section, 
we looked for \emph{closed form} solutions. That is, we set up a system of equations (\cref{eq:linearTheta}) in $\theta$, and attempted to solve them for $\theta$ (albeit approximately via the pseudoinverse).

As we begin to fit more complex models
%\done{where does regularization go?} 
(including in \Cref{chap:classPipe}), a closed-form solution may no longer be available.

\emph{Gradient descent} is an approach to search for the minimum value of a function, by iteratively finding better solutions based on an initial starting point. The process (depicted in \Cref{fig:gradient}) operates as follows: 
%\done{algorithmic environment. Would look weird to use only once}
\begin{enumerate}
    \item Start with an initial guess for $\theta$
    \item Compute the derivative $\frac{\partial}{\partial \theta}f(\theta)$. Here $f(\theta)$ is the MSE (or whatever criterion we are optimizing) under our model $\theta$.
    \label{step2}
    \item Update our estimate of $\theta \coloneqq \theta - \alpha \cdot f'(\theta)$
    \label{step3}
    \item Repeat Steps \ref{step2} and \ref{step3} until convergence.
    \label{step4}
\end{enumerate}
During each iteration, the process now follows the path of steepest descent, and will gradually arrive at a minimum of the function $f_\theta$.\footnote{Assuming the function is `well-behaved,' e.g.~the objective is bounded below, and the function is differentiable everywhere; though these are rarely issues when dealing with 
%error functions specifically.
simple error functions like the MSE.
}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/gradient.pdf}
%\done{Figure has problems (e.g. f(heta)), and annotation looks funny. Fixed}
\caption{Gradient descent demonstration. \label{fig:gradient}}
\end{SCfigure}

The above is a simple description of the procedure that omits many details. 
%We omit specifics, and 
In practice, we will largely rely on high-level libraries to implement gradient-based methods (\cref{subsec:regressionTensorflow}).
%some of the specifics for now and save them until we implement the procedure in practice, 
%\done{pick some way to reference code from the text. Dunno what this is}
Briefly, to implement such techniques `from scratch,'
%though 
some of the main issues include:
\begin{itemize}
    \item Given the starting point in \Cref{fig:gradient}, the algorithm would only achieve a \emph{local} rather than a \emph{global} optimum. To address this we could investigate ways to come up with a better initial `guess' of $\theta$, 
    %or later,
    %\done{mentioned later? No} 
    %we could investigate alternative 
    or
    investigate 
    %optimization approaches 
    variants of gradient descent that are
    less susceptible to local minima.
    \item The step size $\alpha$ (step~\ref{step3} above) must be chosen carefully. If $\alpha$ is too small, the procedure will converge very slowly; if $\alpha$ is too large, the procedure may `overshoot' the minimum value and obtain a \emph{worse} solution during the next iteration. Again, other than carefully tuning this parameter, we could investigate optimization methods not dependent on choosing this rate (see e.g.~quasi-Newton methods such as L-BFGS \citep{liu1989limited}).
    \item `Convergence' as defined in Step~\ref{step4} is not well-defined. We might define convergence in terms of the change in $\theta$ (or $f_\theta(X)$) during two successive iterations, or alternately we may terminate the algorithm once we stop making progress on held-out (validation) data (see \cref{sec:regularization}).
    %\done{validation isn't a section, edit a little}
\end{itemize}

%\done{consider all itemize vs. enumerate. Fine}

\subsection{Linear Regression via Gradient Descent}

To solidify the ideas above, let's consider the specific example of minimizing the Mean Squared Error of a linear model, i.e.,
%\done{Consistent way of writing $(y - x_i \cdot \theta)$, e.g. does $y$ go first. Compare to $R^2$ equation. I think it's fine,}
\begin{equation}
    \frac{1}{|y|} \sum_{i = 1}^{|y|} (x_i \cdot \theta - y_i)^2.
\end{equation}
The
%\done{equation above has both $X$ and $y$. This should be corrected everywhere}
derivative $f'(\theta)$ can be computed as follows:
%\done{Check that I'm writing $X$ when I mean $X$ and $x_i$ when I mean $x_i$. Mostly corrected I think}
\begin{equation}
   \frac{\partial f}{\partial \theta_k} = \frac{1}{|y|} \sum_{i = 1}^{|y|} 2 x_{ik} (x_i \cdot \theta - y_i).
   \label{eq:derivative1}
\end{equation}
Note that the above is a \emph{partial} derivative in $\theta_k$, which must be computed for each feature dimension $k = \lbrace 1 \ldots K \rbrace$.\footnote{The derivative of \protect\Cref{eq:derivative1} is more obvious after expanding $x_i \cdot \theta = \sum_{k=1}^K x_{i,k}\theta_k$.
}
%\done{put in notation table}



\section{Non-linear Regression}
\label{sec:nonlinearNoCode}

So far, we have limited our discussion to models of the form $y = X\theta$, mostly because these offered us a convenient (closed form) solution to finding lines of best fit in terms of $\theta$.

However, this type of model has several limitations that we might wish to overcome, such as:
\begin{itemize}
    \item We cannot incorporate simple constraints on our parameters, such as that a certain parameter should be positive, or that one parameter is larger than another (which might be based on domain knowledge of a certain problem).
    \item Although we can manually engineer non-linear transforms of our features (as we did in \Cref{sec:simpleTransforms}), we cannot have the model learn these non-linear relationships automatically.
    \item The model cannot learn complex \emph{interactions} among features, for example that length is correlated with ratings, but only if the user is female.\footnote{To be precise, the linear model \emph{does} consider relationships among features in the limited sense that parameters for one feature will change in the presence of other correlated features (\protect\Cref{sec:interpretLinear}); and, the model \emph{could} capture relationships between (e.g.)~gender and length if we were to manually engineer a feature describing this relationship. Our point here is about whether the model can learn these relationships \emph{automatically}.}
\end{itemize}

The above goals can potentially be realized if we are allowed to transform model \emph{parameters}: for instance, we could ensure that a particular parameter was always positive by fitting
\begin{equation}
    \theta_k = \log(1 + e^{\theta_k'})
\end{equation}
(this is known as a `softplus' function; note that this function smoothly maps $\theta_k' \in \mathbb R$ to $\theta_k \in (0,\infty)$); or if we wanted one feature to be larger than another (e.g.~$\theta_k > \theta_j$) we could simply add the positive quantity above to another feature:
\begin{equation}
    \theta_k = \theta_j + \log(1 + e^{\theta_k'}).
\end{equation}

%Although it goes beyond our discussion here, 
Roughly speaking, fitting these types of non-linear models (and especially models that deal with complex combinations of parameters) is the basic goal of \emph{deep learning}. 
We will see various examples of non-linear models in later chapters, including models based on deep learning (e.g.~\cref{sec:rnn,sec:personalGans}).
%though for the moment we briefly discuss
%We briefly describe how to implement this type of non-linear optimization
%For the moment we 
In \Cref{chap:classPipe} (\cref{subsec:regressionTensorflow}) we
present the basic approach used to fit these types of models using high-level optimization libraries.


\subsection{Case Study: Image Popularity on Reddit}
\label{subsec:casestudyReddit}

\cite{lakkaraju2013s} used regression algorithms to estimate the success of content (e.g.~number of upvotes) on \emph{reddit}. Other than building an accurate predictor, their main goal is to understand and disentangle \emph{which features are most influential in determining content popularity}.

%\done{connecting sentence}

Presumably, one of the biggest predictors of success is the quality of the content itself. Predicting whether 
%submitted content 
a submission
%(e.g.~an image)
is of high quality (e.g.~whether an image is funny or aesthetically attractive) is presumably incredibly challenging.
To control for this high-variance factor of content quality, 
%the primary source of data studied in 
\cite{lakkaraju2013s} study \emph{resubmissions}, 
%on reddit, 
i.e.,~content (images) that has been submitted multiple times. 
%
%Studying resubmissions allows the study to control for the influence of the content itself
This way, if one submission is more successful than another (of the same image), the difference in success cannot be attributed to the content itself, and must arise due to other factors such as the title of the submission or the community it was submitted to.

%Community effects:
Having controlled for the effect of the content itself, the goal is then to distinguish between features that capture the specific dynamics of reddit itself, versus those that arise due to the choice of title (i.e.,~how the content is `marketed'). Various features are extracted that model reddit's community dynamics, such as the following:
%\done{could connect a bit better}
\begin{itemize}
    \item One of the largest predictors of successful content is simply whether it has been submitted before (as we saw in \Cref{fig:redditResub}, which is based on the same dataset); this is captured via an exponentially decaying function.
    \item However, the above effect might be mitigated if enough time has passed between resubmissions (by when the original submission is forgotten, or the community has enough new users); this is captured using a feature based on the \emph{inverse} of the time delta between submissions.
    \item Resubmissions might still be successful if they are resubmitted to largely non-overlapping communities (subreddits).
    \item Submission success may correlate with the time of day. For example, 
    %success may correlate with overall traffic on the site, 
    submissions may be most successful during the highest-traffic times of day,
    or alternately they may be more successful if submitted when there is less competition.
\end{itemize}
%\done{check style on all items: do they finish with a . ? Finish with .} 

Whereas community effects are somewhat reddit-specific, 
%a question of broader interest is to determine the effect 
%\done{effect vs affect, check everywhere} 
%that a particular choice of \emph{title} has on the success of a submission. 
measuring the effect of a particular choice of title can potentially be of broader interest.
Understanding the characteristics of successful titles can have implications when marketing content (such as an advertising campaign) to a new market. 
%(for example).
%\done{repetitive compared to previous paragraph. Edited a little}

%Title effects:
Several features can be
%are 
extracted to capture the dynamics of a submission's title, including:
\begin{itemize}
    \item Titles should differ from those previously used by submissions of the same content.
    \item Titles should align with the expectations of the community the content is submitted to. Interestingly, \cite{lakkaraju2013s} find that there is a `sweet spot,' in the sense that titles should roughly follow the linguistic style of previous successful submissions in the same community, but should not be \emph{too} similar, to the point that they are not novel compared to previous submissions (we'll discuss text similarity measures more in \Cref{chap:text}).
    \item Successful titles might have other features, in terms of length, sentiment, linguistic style, etc.
\end{itemize}

Ultimately, all of the above features are combined into a regression model that estimates the score (number of upvotes minus number of downvotes) that a particular submission will receive.

Due to the way that features are combined, the model is not linear in the parameters, so optimization proceeds by gradient descent (as in \cref{sec:gradient}). The method is evaluated in terms of the $R^2$ coefficient (\cref{subsec:r2coeff}), with experiments revealing that community and textual features both play a key role in prediction. Finally, it is shown that the method can be used `in the wild' to predict the success of actual reddit submissions.




\begin{exercises}
%  \item 
 %A few examples of regression problems and computing MSE to get started. Copy from class.
 \item Using the \emph{GoodReads} data (see e.g.~\Cref{sec:linearReg}), train a simple predictor that estimates ratings from review length, i.e.,
 $$\text{star rating} = \theta_0 + \theta_1 \times (\text{review length in characters}).$$
 Compute the values $\theta_0$ and $\theta_1$, and the Mean Squared Error of your predictor.
 \label{ex:reg1}
 \item Re-train your predictor so as to include a second feature based on the number of comments, i.e.,
 $$\text{star rating} = \theta_0 + \theta_1 \times(\text{length}) + \theta_2 \times(\text{number of comments}).$$
 Compute the coefficients and MSE of the new model. Briefly explain why the coefficient $\theta_1$ in this model is different from the one from Exercise~\ref{ex:reg1}.
 %\item More easy questions
 %\done{seems like enough, could have more classifiers}
 \item Show that $\theta_0 = \bar{y}$ is the best possible solution for a trivial predictor (i.e.,~$y = \theta_0$) in terms of the Mean Squared Error (hint: write down the MSE of this trivial predictor and take its derivative).
 \label{ex:MSEtrivial}
 \item Repeat Exercise~\ref{ex:MSEtrivial}, but this time show that the best trivial predictor in terms of the Mean \emph{Absolute} Error (\cref{eq:MAE}) is given by taking the \emph{median} value of $y$.
 \item In \Crefrange{eq:MSEMLEderiv0}{eq:MSEMLEderiv} we motivated the choice of the MSE by explaining its relationship to a Gaussian error model. Likewise, show that minimizing the MAE is equivalent to maximizing the likelihood if errors follow a Laplace distribution (the Laplace distribution has probability density function $\frac{1}{2b} \exp\left(-\frac{|x - \mu|}{b}\right)$).
 \item 
 %(Hard) 
 In \Cref{eq:pseudoInv} we saw how to compute a line of best fit via the pseudoinverse, $\theta = (X^TX)^{-1}X^Ty$; show that the parameters that minimize the Mean Squared Error are found by taking the pseudoinverse, i.e.,~that $\argmin_\theta \frac{1}{|y|}\sum_{i=1}^{|y|} (x_i\cdot \theta - y_i)^2 = (X^TX)^{-1}X^Ty$ (that is, find the stationary point where $\frac{\partial \text{MSE}}{\partial \theta} = 0$).
 \label{ex:hardPseudoInv}
 \item 
 %(Hard) 
 When minimizing the Mean Squared Error with a linear model as in Exercise~\ref{ex:hardPseudoInv}, show that the residuals $r_i = (y_i - x_i \cdot \theta)$ have average $\bar{r} = 0$.
 \label{ex:residual0}
 \end{exercises}
 

\refstepcounter{projectCounter}
\label{proj:1}
\section*{Project \theprojectCounter: Taxicab Tip Prediction (Part 1)}
\addcontentsline{toc}{section}{Project \theprojectCounter: Taxicab Tip Prediction (Part 1)}
%\done{Do this for all projects}

Throughout the chapter, we've seen various strategies for dealing with features of different types. For our first project, we'll look into building a prediction pipeline to estimate tip amounts from taxicab trips. For this project you might make use of publicly-available data such as 
the \emph{NYC Taxi and Limousine Commission Trip Record Data}.\footnote{\url{https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page}}
%\cite{?}.
%https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page
%\done{more words}

This project is mostly intended to introduce the end-to-end approach of exploring a new dataset, extracting meaningful information from it, and comparing alternative models. We break this down into the following parts:
\begin{enumerate}
    \item First, conduct \emph{exploratory analysis} of the data. Just as we have done throughout the chapter, plot the relationship between the output (tip amount), and various features that you think might be related to this outcome.
    \item Based on the above analysis, consider what features might be useful for prediction. Consider, for example, features associated with the time of the trip, the start and end location, and the duration/distance of the trip.
    \item How should the above features be represented or transformed? For example, how can the timestamp be represented to capture variation at the level of time of day, day of week, or even the time of year (\cref{sec:temporalFeatures}); how might you represent the start and end locations? Are there any useful derived features that are useful for prediction, e.g.~$\text{speed} = \text{distance}/\text{duration}$?
    \item Is it useful to transform the \emph{output} variable (\cref{subsec:transformOutput})? For example, rather than predicting the tip \emph{amount}, it may make more sense to predict the tip \emph{percentage}.
        %\item Are there any alternatives to formulating the task as a regression problem? For example, you could cast the problem as \emph{classification} by estimating whether a tip will be above or below the median; this may be less sensitive to outliers. Discuss the advantages and disadvantages of various formulations as well as what evaluation metrics you might use.
\end{enumerate}

We'll revisit and extend this project in \Cref{chap:classPipe} (Project~\ref{proj:classif}) once we have further developed the learning pipeline, in order to more rigorously investigate and compare our modeling decisions.

%\done{full spell check}


\chapter{Classification and the Learning Pipeline}
\label{chap:classPipe}

%\done{Intro}

%\done{Switch order of classification and pipelines}

%\done{Remove mention of regularization from classification section}

% \section{Classification}
% \label{sec:classificationSec}

So far, we have considered supervised learning tasks in which the output variable $y$ is a real number, i.e.,~$y \in \mathbb R$. Often, we will deal with problems with binary or categorical output variables, for example we might be interested in problems such as:
\begin{itemize}
  \item Will a user click on a product or advertisement? (binary outcome)
  \item What category of object does an image contain? (multiclass)
  \item What product is a user most likely to purchase next? (multiclass)
  \item Which of two products would a user prefer? (binary)
\end{itemize}

% \subsection{Logistic Regression}
% \label{sec:logisticRegression}



%\done{Move the figure below. Not sure what this is?}

%\done{Explain that logistic regression is an example of a linear classifier. Who cares}

%In this section we will introduce a form of 
In this chapter we'll 
explore how to design \emph{classification} algorithms for tasks like those above,
and in particular
explore a
classifier that extends the ideas behind regression from \Cref{chap:basicsReg} to classification problems.

%\done{anything missing? No}
\emph{Logistic Regression} sets up classification using a probabilistic framework,
by transforming the predictions $X \cdot \theta$ that we used when building regressors into \emph{probabilities} associated with observing a particular label $y$.
%the predictions $X \cdot \theta$ that we used when building regressors are transformed into \emph{probabilities} of observing a particular label.
%
%In short, logistic regression associates a probability of the dataset under some model, 
By associating a probability with a particular label, and thereby to all of the labels in a dataset, we can again develop prediction frameworks that are
%which is 
differentiable and can be optimized using gradient-based approaches, much as we saw in 
\Cref{sec:gradient}.
%\Cref{sec:whyMSE,sec:gradient}.
%\done{check. Looks fine}

Ultimately logistic regression is just one of dozens of classification schemes; we describe it here rather than
%other (possibly more popular) 
alternatives (such as Support Vector Machines \citep{cortes1995support}, or Random Forest Classifiers \citep{ho1995random}) mainly because logistic regression more closely matches the approaches we will develop in later chapters.
%\done{more or less repeated text, reference description below. Fine I think}
This same type of modeling approach will be used throughout this book, when building Recommender Systems in \Cref{chap:modelbasedRecsys}, or 
%modeling visual feature 
%or defining model fairness objectives
%in \Cref{chap:ethics}.
generating fashionable outfits in \Cref{chap:visual}, among others. We'll briefly discuss the merits of alternative classification approaches in \Cref{subsec:otherClass}.
%\done{could say more}

After exploring classification techniques in \Cref{sec:logisticRegression}, we'll explore evaluation strategies for classification models in \Cref{sec:evaluatingClassifiers}, much as we did for regression models in \Cref{chap:basicsReg}. 

Finally, 
%in \Cref{sec:learningPipeline} 
we'll explore the idea of the learning \emph{pipeline}. Having developed techniques for regression (\cref{chap:basicsReg}), classification, and evaluation strategies for both, 
in \Cref{sec:learningPipeline}
we will explore how to compare models, how to ensure that our results are significant, and how to ensure that our models generalize well to unseen data. This type of end-to-end strategy for model training will be used whenever we train supervised learning models throughout the remainder of the book.


\section{Logistic Regression}
\label{sec:logisticRegression}

% \subsection{Model Objective}

%and Fitting a logistic regressor}
When 
%fitting 
developing
regular linear regressors in \Cref{chap:basicsReg}, we wanted a model $f_\theta$ whose estimates $f_\theta(x_i)$ were as close as possible to the (real-valued) labels $y_i$. When adapting a linear regression algorithm  to classification, we might instead seek models that associate positive values of $x_i \cdot \theta$ with positive labels ($y_i = 1$), and negative values of $x_i \cdot \theta$ with negative labels ($y_i = 0$).

If we could do so, we could write down the \emph{accuracy} associated with a particular model:
%\done{renders ugly. Looks fine}
\begin{equation}
\frac{1}{|y|}
    \sum_{i=1}^{|y|} \underbrace{\delta(y_i = 0) \delta(x_i\cdot \theta \leq 0)}_{\hspace{-15mm}\mathrlap{\text{label is negative and prediction is negative}}} + \overbrace{\delta(y_i = 1) \delta(x_i\cdot \theta > 0)}^{\mathllap{\text{label is positive and prediction is positive}}\hspace{-15mm}}
    \label{eq:accuracySummation}
\end{equation}
(here $\delta$ is an indicator function that returns $1$ if the argument is true, $0$ otherwise). The equation above, in spite of slightly confusing notation, is merely counting the number of times we correctly predict a positive score for positively labeled instance, and a negative (or zero) score for negatively labeled instance.

We now simply desire from our classifier $\theta$ that it maximizes the accuracy 
%in 
measured by
\Cref{eq:accuracySummation}.
Unfortunately, directly optimizing 
%the accuracy in 
\Cref{eq:accuracySummation}  for $\theta$ is NP-hard (see e.g.~\cite{nguyen2013algorithms}). To get a sense for why it is difficult, consider that the function in \Cref{eq:accuracySummation} is essentially a \emph{step} function (\cref{fig:sigmoid}, left), i.e.,~it is flat (derivative zero) almost everywhere; it is therefore not amenable to techniques like gradient ascent as we saw in \Cref{sec:gradient}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/step.pdf}
\includegraphics[scale=0.75]{fig/sigmoid.pdf}
\caption[The sigmoid function.]{The sigmoid function (right), and the step function (left) that it approximates. \label{fig:sigmoid}}
\end{SCfigure}

So, to optimize the accuracy approximately, we would like a function that is \emph{similar to} \Cref{eq:accuracySummation}, but is more straightforward to optimize.

\emph{Logistic Regression} achieves this goal by converting the outputs of a linear function $x_i \cdot \theta$ to \emph{probabilities} via a smooth function. Our intuition is that large values of $x_i \cdot \theta$ should correspond to high probabilities, and small (i.e.,~large negative) values of $x_i \cdot \theta$ should correspond to small probabilities.

This goal can be achieved via the \emph{sigmoid function}:
\begin{equation}
    \sigma(x) = \frac{1}{1 + e^{-x}}.
\end{equation}
This function, depicted in \Cref{fig:sigmoid}, maps a real value to the interval $(0,1)$, and passes through 0.5 when $x=0$. Thus it can be interpreted as a probability:%
\begin{equation}
p_\theta(y_i = 1|x_i) = \sigma(x_i \cdot \theta) = \frac{1}{1 + e^{-x_i\cdot \theta}}
\end{equation}
Now, as a smooth surrogate for the expression in \Cref{eq:accuracySummation}, we can instead optimize
%\done{Rewrite the previous function so that it uses this notation rather than delta functions. Is fine}
\begin{align}
    \mathcal{L}_\theta(y|X) & = \prod_{y_i = 1} p_\theta(y_i = 1|x_i) \times \prod_{y_i = 0} \left(1 - p_\theta(y_i = 0|x_i)\right)\\
    & = \prod_{y_i = 1} \frac{1}{1 + e^{-x_i\cdot \theta}} \times \prod_{y_i = 0} \frac{e^{-x_i\cdot \theta}}{1 + e^{-x_i\cdot \theta}}.
    \label{eq:likelihoodLogistic}
\end{align}
% Our goal is to maximize this expression for $\theta$, i.e., to find
% \begin{equation}
%     \argmax_\theta L_\theta(y|X)
% \end{equation}
The above expression is a \emph{likelihood function}, much like we saw in \Cref{eq:likelihood1}.
Intuitively, for this expression to be maximized we want positive instances ($y_i = 1$) to be associated with high probabilities, and negative instances ($y_i = 0$) to be associated with low probabilities.

\subsection{Fitting the Logistic Regressor}

Our goal is to maximize the above function, i.e.,~to find $\argmax_\theta L_\theta(y|X)$. Short of a closed form solution, our approach is to take the logarithm $\ell_\theta(y|X)$ (since $\argmax_\theta \mathcal{L}_\theta(y|X) = \argmax_\theta \log(\mathcal{L}_\theta(y|X))$), to compute its gradient, and optimize via gradient ascent (as in \cref{sec:gradient}).
%approach is as follows:
% \begin{itemize}
%     \item Take the logarithm $\ell_\theta(y|X)$, since $\argmax_\theta \mathcal{L}_\theta(y|X) = \argmax_\theta \log(\mathcal{L}_\theta(y|X))$.
%     \item \emph{Subtract} a regularizer $\lambda \| \theta \|_2^2$ (or similar).\footnote{Note that we subtract the regularizer rather than adding it as we did in \Cref{?} since we are maximizing rather than minimizing our objective; since we still want low complexity, we seek to maximize $-\lambda \| \theta \|_2^2$.}
%     \item Compute the gradient of $\ell_\theta(y|X) - \lambda \| \theta \|_2^2$, and optimize via gradient ascent.
% \end{itemize}
We compute the gradient below as follows:
\begin{align}
% \ell_\theta(y|X) & = \sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-x_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-x_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right) - \lambda \| \theta \|_2^2 \label{eq:logisticDeriveWithoutLamb1}\\
%  & = \sum_{i} - \log(1 + e^{-x_i\cdot \theta}) + \sum_{y_i = 0} -x_i \cdot \theta - \lambda \| \theta \|_2^2 \label{eq:logisticDeriveWithoutLamb2}\\
% \frac{\partial \ell}{\partial \theta_k} & = \sum_i x_{ik} \frac{e^{-x_i\cdot \theta}}{1 + e^{-x_i \cdot \theta}} - \sum_{y_i = 0} x_{ik} - \lambda\theta_k\\
% & = \sum_i x_{ik} \left( 1 - \sigma(x_i \cdot \theta) \right) - \sum_{y_i = 0} x_{ik} - \lambda \theta_k
\ell_\theta(y|X) & = \sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-x_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-x_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right) \label{eq:logisticDeriveWithoutLamb1}\\
 & = \sum_{i} - \log(1 + e^{-x_i\cdot \theta}) + \sum_{y_i = 0} -x_i \cdot \theta \label{eq:logisticDeriveWithoutLamb2}\\
\frac{\partial \ell}{\partial \theta_k} & = \sum_i x_{ik} \frac{e^{-x_i\cdot \theta}}{1 + e^{-x_i \cdot \theta}} - \sum_{y_i = 0} x_{ik}\\
& = \sum_i x_{ik} \left( 1 - \sigma(x_i \cdot \theta) \right).
\label{eq:logisticDeriveWithoutLamb4}
\end{align}
Note carefully that the summation indices change between
%\done{Why do the references below point to the wrong thing? Looks right since change to align}
\Cref{eq:logisticDeriveWithoutLamb1,eq:logisticDeriveWithoutLamb2}, since both terms in \Cref{eq:logisticDeriveWithoutLamb1} have the same denominator.
%\done{section is done, think whether paragraph below is needed}

%The basic idea behind logistic regression is that each observation $x_i$ 
%%\done{notation? Gone} that has a positive label ($y_i = 1$) should be associated

\subsection{Summary}
Our development of logistic regression above is representative of the overall approach we'll take later when developing models that estimate interactions, clicks, purchases, etc.: 
\begin{itemize}
    \item Rather than estimating an outcome directly, we associate a \emph{probability} with each outcome.
    %\item 
    Associating 
    %the outcome with probability 
    a probability with the outcome
    allows us to replace discrete (e.g.~$y_i \in \lbrace 0, 1 \rbrace$) 
    %\done{notation. I think is right}
    outcomes with a continuous function ($f(x) \in (0,1)$); this is accomplished via a transformation (such as the sigmoid function) which maps a real-valued output into the desired range.
    \item The model should associate positive (1) labels with high probabilities, and negative labels (0) with low probabilities. Likewise we can associate a probability to the entire dataset by taking a product of probabilities (or a sum of log-probabilities, as in \Cref{eq:logisticDeriveWithoutLamb1}).
    \item Ultimately the procedures above allow us to associate the quality of a model (parameterized by $\theta$)
    %\done{spelling of this word}
    with a continuous function whose value we should try to maximize; we optimize the model via gradient ascent.
\end{itemize}
%\done{consider whether this paragraph is redundant. It's fine}

\section{Other Classification Techniques}
\label{subsec:otherClass}

%In this introductory chapter, 
In our introduction to classification,
we have only discussed a single classification technique: Logistic Regression. Our choice to explore this particular technique was largely a practical one: the idea of associating a probability with a particular outcome (as in \cref{eq:likelihoodLogistic}) and estimating that probability via a differentiable function (to facilitate gradient ascent) will appear repeatedly as we develop more and more complex models. 
%Likewise, the choices of training and regularization strategies were largely guided by what will prove useful later.

However the 
%specific
technique we've explored is only one class of approach to build classifiers.
%, and logistic regression specifically is hardly the ideal classifier for every situation. 
The specific choice to map binary labels to continuous probabilities via a smooth function has hidden assumptions and limitations, meaning that logistic regression is not the ideal classifier for every situation. Below we present 
%several 
a few
alternatives, largely as further reading and to highlight specific situations where logistic regression may not be the preferable choice. 
%While the type of probabilistic model we developed above is a fundamental building block of more complex machine learning models, 

\begin{description}
  \item[Support Vector Machines] While logistic regressors optimize a probability associated with a set of observed labels, they do not explicitly minimize the number of \emph{mistakes} made by the classifier. Support Vector Machines (SVMs)
  \citep{cortes1995support} replace the sigmoid function in \Cref{fig:sigmoid} with an expression that assigns zero cost to correctly classified examples,\footnote{More precisely, correctly classified by some margin.} and a positive cost\footnote{Which is no longer interpretable as a probability.} to incorrectly classified examples (in proportion to the confidence of the prediction $x \cdot \theta$).
  %\done{style}
  %
  This distinction is fairly subtle: while \emph{every} sample will influence the optimal value of $\theta$ for a logistic regressor, the solution found by an SVM is entirely determined by a few samples closest to the classification boundary, or those that are mislabeled. Conceptually it is appealing for a classifier to focus on the most `difficult' samples in this way, though note that in many cases (and notably when building recommender systems) our goal is to optimize ranking performance rather than classification accuracy (as we'll discuss in \Cref{sec:errorRanking}), such that giving special attention to the most ambiguous examples is not necessarily desirable.
  %\done{Best reference? Do I already talk about this somewhere?}
  \item[Decision Trees] Decision trees classify instances based on a sequence of binary decisions, each of which deals with a specific feature. 
  %Following each binary decision, 
  Each node of the tree separates the data based on such a decision, with leaf nodes being responsible for determining an outcome.
  %\done{not great}
  Decision trees straightforwardly facilitate learning non-linear classifiers that capture complex interactions among features, e.g.~we can straightforwardly learn that 
  %for 
  %young people, low price is associated with a positive review, whereas for older people, 
  a low price is associated with a positive review for young people, while a high price is associated with a positive review for older people: such an association is difficult for a linear classifier to learn if neither the `age' nor `price' feature is individually correlated with the outcome.
  Extensions such as \emph{random forests}
  \citep{ho1995random} (an ensemble of decision trees) remain popular forms of classification. 
  %We exclude such techniques here mostly because they have little in common with the approaches we bulid in later chapters.
  %\done{Why is this cited elsewhere? Shouldn't describe twice} 
  \item[Multilayer Perceptrons] So far, we have 
  %only described 
  focused on
  \emph{linear} classifiers, which assume a simple relationship between between features and predictions. Although we argued in \Cref{sec:featureEngineering} that such limitations can be overcome by careful feature engineering, ideally we might like to learn such feature transformations automatically. Uncovering such complex relationships among features, and automatically learning non-linear feature transformations is one of the main goals of \emph{deep learning}. We'll revisit such approaches as we develop more complex models 
  %at various points throughout the book.
  throughout the book.
  %We briefly introduce multilayer perceptrons in \Cref{sec_?} when describing their use within deep learning-based recommendation techniques.
%   \item[Nearest Neighbor Classification]
%   aoeu
  %\done{add? No}
\end{description}
%\done{could include other approaches}

We exclude SVMs and decision trees from the remainder of this book
%such techniques here 
mostly because they have little in common methodologically with the approaches we bulid in later chapters.
We briefly introduce multilayer perceptrons in \Cref{subsec:MLPRec} when describing their use within deep learning-based recommendation techniques. As we try to reiterate throughout the book, multilayer perceptrons
%\done{style}
and various other state-of-the-art models are simply architectural choices that offer alternate ways to optimize the same objectives that we approach through simpler models. Having introduced the overall objectives, and the fundamentals of gradient based optimization approaches, adapting them to alternate architectures is (relatively) straightforward.


\section{Evaluating Classification Models}
\label{sec:evaluatingClassifiers}

So far, when developing classifiers, we have focused on maximizing the alignment between the labels and the model's outputs. E.g.~in the case of logistic regression, we want the predicted probability $p_\theta(y_i = 1 | x_i)$ to be as close as possible to the label $y_i$. Implicitly, when doing so, we are trying to maximize the model's \emph{accuracy}:
\begin{equation}
%\done{f_\theta|y,X is not the notation I use elsewhere. Made everything a two-argument function)
    \text{accuracy}(y,f_\theta(X)) = \frac{1}{|y|} \sum_{i=1}^{|y|} \delta ( f_\theta(x_i) = y_i ),
\end{equation}
where $\delta$ is an indicator function,
%\footnote{
%\done{put this wherever delta is first used, or just in notation page. Is defined above}
%$\delta(x) = 1$ if the argument $x$ is true, $0$ otherwise.}
and $f_\theta(x_i)$ is the binarized output of the model (e.g.~in the case of logistic regression, $f_\theta(x_i) = \delta(x_i \cdot \theta > 0)$).\footnote{Note that this is equivalent to the expression in \protect\Cref{eq:accuracySummation}.} Equivalently we are minimizing the \emph{error}, i.e.,
\begin{equation}
    \text{error}(y,f_\theta(X)) = 1 - \text{accuracy}(y,f_\theta(X)).
\end{equation}
%\done{could drop or inline error definition. It's fine}

To motivate the difficulty of properly evaluating classifiers, consider the following classification task. We saw in \Cref{fig:genderLength} that there was a slight relationship between gender and review length; now, let's see if we can develop a simple classifier that attempts to predict gender based on review length:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = [[1, len(d['review/text'])] for d in data]
y = [d['user/gender'] == 'Female' for d in data]

mod = sklearn.linear_model.LogisticRegression()
mod.fit(X,y)
predictions = mod.predict(X) # Binary vector of predictions
correct = predictions == y # Binary vector indicating which predictions were correct
print(sum(correct) / len(correct))
\end{lstlisting}
\end{minipage}
%\done{This probably shouldn't be the first time code is used in this chapter. It's not}

Surprisingly, the classifier produced by this code is 98.5\% accurate.
%\done{Don't know if I want exclamation marks. I don't}
This result might seem implausible, but turns out to be a limitation of the error measure itself. Counting the number of negative labels in the dataset
%
% \noindent
% \begin{minipage}{\linewidth}
% \begin{lstlisting}
% 1 - (sum(y) / len(y))
% \end{lstlisting}
% \end{minipage}
reveals that the data is 98.5\% male (i.e.,~98.5\% negative labels). Not only does this reveal that the accuracy is unlikely to be an informative metric in this case, but it reveals that \emph{our goal of optimizing the accuracy} caused us to learn a trivial classifier---the model simply predicts zero everywhere.

%\done{replace | with \mid where appropriate}

The above example demonstrates the problem with na\"ively computing (or optimizing) model accuracy. Several situations where we might need more nuanced evaluation measures include:
\begin{itemize}
    \item Datasets whose labels are highly imbalanced, such as the example above.
    \item Situations where different types of errors have different associated costs. E.g.~failing to detect dangerous luggage is a more severe mistake than an erroneous positive identification.
    \item When we use classifiers for search or retrieval (as we will often do when developing recommender systems), we often care about the ability of the model to confidently identify a few positive instances (e.g.~those surfaced on a results page), and are not interested in its overall accuracy.
\end{itemize}

Below we will develop error measures designed to handle each of these scenarios.

%\subsection{Type I and Type II Errors}
\subsection{Balanced Metrics for Classification}
\label{sec:type1type2}

The basic issue with the example presented above was that we allowed one of the two labels to dominate the classifier's objective. Although in some cases we may justifiably want a classifier that focuses more on the dominant label, in the example above we would likely prefer a solution that had reasonable accuracy \emph{per class}.

To achieve this we need evaluation metrics that consider the two classes (positive and negative, or female and male in our example) separately. To do so, we consider each of the four possible outcomes in terms of our prediction and label:
\begin{alignat}{3}
  \mathit{TP} & =\text{True Positives} &&= |\lbrace i \mid y_i \land f_\theta(x_i) \rbrace| \label{eq:TP1}\\
  \mathit{FP} & =\text{False Positives} &&= |\lbrace i \mid \lnot y_i \land f_\theta(x_i) \rbrace| \\
  \mathit{TN} & =\text{True Negatives} &&= |\lbrace i \mid \lnot y_i \land \lnot f_\theta(x_i) \rbrace| \\
  \mathit{FN} & =\text{False Negatives} &&= |\lbrace i \mid y_i \land \lnot f_\theta(x_i) \rbrace|.\label{eq:FN4}
  %\done{No understanding of alignat above, and should fix equation below too}
  %see: https://tex.stackexchange.com/questions/200813/difference-between-align-and-alignat-environments
\end{alignat}
From these, we can define errors (or accuracies) that consider each of the two classes in isolation:\footnote{Various other terms exist for these expressions, e.g.~the terms \emph{sensitivity}, \emph{recall}, \emph{hit rate}, and \emph{true positive rate} are largely interchangeable.}
%\done{Replace by array environment and add more &'s before the last equal, and in equation above to properly align}
\begin{alignat}{4}
 \mathit{TPR} & = \text{True Positive Rate} && = \frac{|\text{true positives}|}{|\text{labeled positive}|} &&= \frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}\\
 \mathit{FPR} & = \text{False Positive Rate} && = \frac{|\text{false positives}|}{|\text{labeled negative}|} &&= \frac{\mathit{FP}}{\mathit{FP} + \mathit{TN}}\\
 \mathit{TNR} & = \text{True Negative Rate} && = \frac{|\text{true negatives}|}{|\text{labeled negative}|} &&= \frac{\mathit{TN}}{\mathit{TN} + \mathit{FP}}\\
 \mathit{FNR} & = \text{False Negative Rate} && = \frac{|\text{false negatives}|}{|\text{labeled positive}|} &&= \frac{\mathit{FN}}{\mathit{FN} + \mathit{TP}}.
 %\done{Probably change the order of the expressions in the denominators to be more consistent}
\end{alignat}
Note that it is trivial to optimize any one of these criteria in isolation (e.g.~we can achieve a True Positive Rate of 1.0 simply by always predicting positive). As such, we would normally optimize a criterion which considers both positive and negative labels together. One such measure is the \emph{Balanced Error Rate}, which simply takes the average of the False Positive and False Negative rates:
\begin{equation}
    % \mathrm{BER}(f_\theta ; y,X) =
    \mathrm{BER}(y,f_\theta(X)) =
    \frac{1}{2}(\mathit{FPR} + \mathit{FNR}) = 1 - \frac{1}{2}(\mathit{TPR} + \mathit{TNR}).
    % \frac{1}{2} \Bigl(
    %     \underbrace{\frac{\mathit{FN}}{\mathit{TP} + \mathit{FN}}}_{\mathit{FNR}} +
    %     \underbrace{\frac{\mathit{FP}}{\mathit{TN} + \mathit{FP}}}_{\mathit{FPR}}
    %     \Bigr)
    %\done{Don't use Bigl, just use \smash and \vphantom. Gone}
    % =
    % 1 - \frac{1}{2} \Bigl(
    %     \underbrace{\frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}}_{\mathit{TPR}} +
    %     \underbrace{\frac{\mathit{TN}}{\mathit{TN} + \mathit{FP}}}_{\mathit{TNR}}
    %     \Bigr)
    \label{eq:BER}
\end{equation}
In our motivating example, this now attributes half of the error to the `Female' (positive) class and half of the error to the `Male' (negative) class.

Note that an appealing quality of the Balanced Error Rate is that (unlike the accuracy) it can no longer be minimized via trivial solutions: always predicting `True,' or always predicting `False,' or predicting at random, will all result in a BER of 0.5.

\subsection{Optimizing the Balanced Error Rate}
\label{subsec:optimizeBalanced}

Having argued
%(\cref{sec:type1type2})
that the Balanced Error Rate may be preferable to the accuracy if we wish to avoid trivial solutions, we next ask how to train a classifier to avoid producing trivial solutions in the first place.

Intuitively, the degenerate solutions we saw in 
\Cref{sec:evaluatingClassifiers} (i.e.,~a classifier which predicted zero everywhere)
arose due to an imbalance in our training data (i.e.,~a high ratio of positive or negative labels). Trivially, we might correct this by \emph{re-sampling} our training data: i.e.,~sampling either a fraction of our 
negative instances, or sampling negative instances (with replacement) until we have an equal number of positive and negative instances.

While the above is a common and reasonably effective strategy, the same goal can be achieved more directly simply by \emph{weighting} the positive and negative instances. Note that in our objective for logistic regression:
\begin{equation}
    \sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-x_i \cdot \theta}}\right) + \sum_{y_1 = 0} \log \left( \frac{e^{-x_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right),
\end{equation}
the two summations (over $y_i = 1$ and $y_i = 0$) essentially reward the model for correctly predicting positive instances 
%(i.e.,~True Positive Rate) 
and negative instances.
%(i.e.,~True Negative Rate). 
The issue with the above objective is that 
%it will be imbalanced, and 
one of the two terms can dominate the expression in the event that positive or negative instances are over-represented in our dataset.

To address this, we can normalize the two expressions by the number of samples in the positive and negative classes:
\begin{equation}
    \frac{|y|}{2|\lbrace i \mid y_i = 1 \rbrace|}\sum_{y_i = 1} \log \left( \frac{1}{1 + e^{-x_i \cdot \theta}}\right) + \frac{|y|}{2|\lbrace i \mid y_i = 0 \rbrace|}\sum_{y_1 = 0} \log \left( \frac{e^{-x_i\cdot \theta}}{1 + e^{-X_I\cdot \theta}} \right).
\end{equation}
%\done{kinda sloppy notation. Fixed}
%\done{writing still a bit sloppy. Fine}
By doing so the left- and right-hand expressions have equal importance, such that \emph{all} positively labeled instances have the same importance as \emph{all} negative instances; in other words the two expressions (after normalization) roughly correspond to the True Positive \emph{Rate} and True Negative \emph{Rate}, as in \Cref{eq:BER}.
Note that in addition to normalizing by the number of samples, both sides are multiplied by $\frac{|y|}{2}$; this is not strictly necessary but is done by convention such that the total `weight' of all instances is still $|y|$.

The above can be accomplished with the \texttt{class\_weight='balanced'}
%\done{do I ever use texttt anywhere? Yes}
option in sklearn as follows:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = [[1, len(d['review/text'])] for d in data]
y = [d['user/gender'] == 'Female' for d in data]

mod = sklearn.linear_model.LogisticRegression(class_weight='balanced')
mod.fit(X,y)
\end{lstlisting}
\end{minipage}

Note that the same idea can be applied to problems including more than two categories, and that one can choose different weighting schemes, e.g.~to assign any desired relative importance to true positives versus true negatives (e.g.~in the baggage-handling scenario we mentioned above).

%\done{Comments in code ugly. Look okay}
%\done{equation references in code different font from comments. Is fine}

% \subsection{Error Metrics for Ranking}
\subsection{Using and Evaluating Classifiers for Ranking}
\label{sec:errorRanking}
%\done{maybe rename? "Using classifiers for ranking"}

% Precision and recall, precision@k

Often, the goal of training a classifier is not merely to generate exhaustive sets of `true' and `false' instances. For example, if we wanted to identify relevant webpages in response to a query, or to recommend items that a user is likely to purchase, in practice it may not matter whether we can identify \emph{all} relevant webpages or products; rather, we might care more about whether we can surface \emph{some} relevant items among the first page of results returned to a user.

Note that the type of classifiers we've developed so far can straightforwardly be used for \emph{ranking}. That is, in addition to outputting a label ($y_i = \delta(x_i \cdot \theta > 0)$ in the case of logistic regression), they can also output \emph{confidence scores} (i.e.,~$x_i \cdot \theta$, or $p_\theta(y_i = 1 | x_i)$).
%\done{check notation}
Thus, in the context of finding relevant webpages or products above, our goal might be to maximize the number of relevant items returned among the few most confident predictions. Furthermore, we might be interested in how the model's accuracy changes as a function of confidence; e.g.~even if the model's accuracy is low overall, is it accurate for the top 1\%, 5\%, or 10\% of most confident predictions?

\subsubsection{Precision and recall}
%\done{Don't like subsubsections. Is fine}

\emph{Precision} and \emph{Recall} assess the quality of a set of retrieved results in terms of two related objectives. Informally, \emph{precision} measures the rate at which those items `retrieved' by the model (i.e.,~those predicted to have a positive label by the classifier) are in fact labeled positively; \emph{recall} measures what fraction of all positively-labeled items our classifier predicted as having a positive label. For example, in a spam filtering setting (where positively-labeled items are spam e-mails), \emph{precision} would measure how often e-mails marked as spam are in fact spam, whereas \emph{recall} would measure what fraction of all spam was filtered.
%\done{is this or something similar repeated somewhere nearby? Don't think so}

Formally precision and recall are defined as follows:
\begin{align}
    \text{Precision} & =  \frac{| \lbrace \text{relevant items} \rbrace \cap \lbrace \text{retrieved items} \rbrace |}{|\lbrace \text{retrieved items} \rbrace|} \label{eq:precRecDefn1}\\
    \text{Recall} & =  \frac{| \lbrace \text{relevant items} \rbrace \cap \lbrace \text{retrieved items} \rbrace |}{|\lbrace \text{relevant items} \rbrace|}
    \label{eq:precRecDefn2}
\end{align}

Alternately it is easy to verify that these expressions can be rewritten in terms of the number of true-positives, false-positives, and false-negatives, as in \Crefrange{eq:TP1}{eq:FN4}:
\begin{align}
  \text{Precision} & = \frac{\mathit{TP}}{\mathit{TP} + \mathit{FP}}\\
  \text{Recall} & = \frac{\mathit{TP}}{\mathit{TP} + \mathit{FN}}
\end{align}

Finally we briefly show how these quantities can be computed for a given predictor (such as the one from the beginning of this section:
%\Cref{code_?}):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
predictions = mod.predict(X) # binary vector of predictions

numerator = sum([(a and b) for (a,b) in zip(predictions,y)])
nRetrieved = sum(predictions)
nRelevant = sum(y)

precision = numerator / nRetrieved
recall = numerator / nRelevant
\end{lstlisting}
\end{minipage}
%\done{better way to write without using zip? It's fine}
%\done{Test code}

\subsubsection{$F_\beta$ score}

Note that neither precision nor recall are particularly meaningful if reported in isolation. For instance, it is trivial to achieve a recall of 1.0 simply by using a classifier that returns `true' for every item (in which case, all relevant documents are returned); such a classifier would of course have low precision. Likewise, a precision of close to 1.0 can often be achieved by returning `true' only for a few items about which we are extremely confident; such a classifier would have low recall.

As such, to evaluate a classifier in terms of precision and recall, we likely want a metric that considers both, or otherwise to place additional constraints on our classifier (as we see below).

The $F_\beta$ score achieves this by taking a weighted average of the two quantities:
\begin{equation}
    F_\beta = (1 + \beta^2) \cdot \frac{\mathrm{precision}\cdot \mathrm{recall}}{\beta^2\text{precision} + \text{recall}}.
    \label{eq:fbeta}
\end{equation}
In the case of $\beta=1$ (which is normally called simply the `$F$-score'), \Cref{eq:fbeta} simply computes the harmonic mean of precision and recall, which is low if either of precision or recall are low.

Otherwise, if $\beta \not= 1$, the $F_\beta$ score reflects a situation where one cares about recall over precision by a factor of $\beta$.\footnote{Without going into detail, this motivation leads to the specific formulation in \protect\Cref{eq:fbeta} \protect\citep{van1979information}.}

There are several situations where one might care about recall more than precision, or \emph{vice versa}. For instance, considering the motivating examples from the start of this section, in a baggage-handling scenario we would likely care primarily about recall, and would be willing to sacrifice precision to achieve it; or, in a search or recommendation setting, we may be happy retrieving only a few items, so long as some are relevant (i.e.,~high precision but low recall).

\subsubsection[Precision and Recall @ $K$]{Precision and Recall @ $\boldsymbol{K}$}

%\done{Rewrite below so it sounds more like precision @ K. Looks fine}
%\done{Also, argument below is ugly and doesn't follow from previous paragraph. Is fine}
%Often, we are interested in evaluating the precision and recall as a function of the model's confidence. 
%We motivated precision and recall by 
One of our motivating examples when defining precision and recall
considered cases where we may only have a fixed budget of results that can be returned to a user.
%That is, what is the model's precision and recall among the top $K$ predictions? 
In particular, we might be interested in evaluating the precision and recall when our classifier returns only its $K$ most confident predictions.
To do so, we begin by sorting the labels $y_i$ according to their associated confidence scores (i.e.,~$x_i \cdot \theta$):
\begin{equation}
%  (0.04961637143573233, True),
%  (0.04265739426913856, True),
%  (0.038081609220486146, True),
%  (0.01647932932266133, True),
%  (0.0027684382608090807, False),
%  (-0.0024453115029425265, True),
%  (-0.005299363587960293, False),
%  (-0.005508568447381734, False),
%  (-0.008051803313750272, True),
%  (-0.010969509446894954, True),
%
{
\setlength{\tabcolsep}{1.8pt}
\small
\begin{tabular}{lcccccccccccc}
confidence $x_i \cdot \theta$: & $\cdots$ & 0.49 & 0.42 & 0.38 & 0.16 & 0.02 & -0.02 & -0.05 & -0.05 & -0.08 & -0.10 & $\cdots$ \\
label $y_i$: & $\cdots$ & True & True & True & True & False & True & False & False & True & True & $\cdots$
\label{eq:sortedByConfidence}
%\done{Why are the cdots not vertically aligned? Looks fine}
\end{tabular}
}
\end{equation}
Such tuples of confidence scores and labels can be generated as follows (in this case for a logistic regressor as in \Cref{sec:evaluatingClassifiers}):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
confidences = mod.decision_function(X) # real vector of confidences

sortedByConfidence = list(zip(confidences,y))
sortedByConfidence.sort(reverse=True) # sorted as in %\Cref{eq:sortedByConfidence}%
\end{lstlisting}
\end{minipage}

Note that when evaluating our model's $K$ most confident predictions, we are no longer interested in whether the actual scores are greater or less than zero (i.e.,~whether the classifier would output `true' or `false'): we are only interested in the \emph{labels} among the top-K predictions.

The \emph{Precision @ $K$} and \emph{Recall @ $K$} now simply measure the precision and recall for a classifier which returns only the $K$ most confident predictions. That is, precision@$K$ measures what fraction of the top-$K$ predictions are actually labeled `true'; recall@$K$ measures the fraction of all relevant documents that are returned among the top $K$. The main difference to note (compared to the definitions in \Cref{eq:precRecDefn1,eq:precRecDefn2}) is that the number of `retrieved' documents is always $K$; that is, the `retrieved' documents are always the $K$ most confident, whether or not the classifier actually predicts a positive label (i.e.,~$x_i \cdot \theta > 0$).

%\done{Add code for precision and recall @ k. Mentioned below}

Unlike precision and recall,
%\done{consistent capitalization}
precision@$K$
%\done{Should write it like this everywhere?}
and recall@$K$ can be reported in isolation as they cannot be optimized by trivial solutions. Precision@10, for example, is an effective measure of a classifier's ability to return reasonable results among a page of 10 retrieved items. 

\subsubsection{ROC and precision/recall curves}

Another holistic measure of a classifier's performance is to report the \emph{relationship} between precision and recall, or between true and false positives.

For example, the relationship between the number of True Positives and False Positives is known as the \emph{Receiver Operating Characteristic} (ROC). It is so named because of its use in evaluating the performance of radar receiver operators: as an operator's threshold for detection decreases, both their true positive and false positive rates (TPR and FPR) will simultaneously increase; thus we might evaluate a classifier by evaluating the TPR and FPR as we change the classifier's detection threshold.
%\done{Double check argument. Looks fine}

The \emph{precision recall curve} is developed following a similar line of reasoning: as we lower a classifier's detection threshold, the precision will decrease while the recall increases; thus we might evaluate a classifier by examining the relationship between precision and recall as the threshold changes.

To generate these curves, we sort the predictions of our classifier by confidence (much as we did in \Cref{eq:sortedByConfidence}), which corresponds to gradually considering lower thresholds; at each step, we compute the precision and recall (i.e.,~we compute the precision and recall@$K$ for each value of $K$). Together these values form the precision recall curve:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
for k in range(1,len(sortedByConfidence)+1):
    retrievedLabels = [x[1] for x in sortedByConfidence[:k]]
    precisionK = sum(retrievedLabels) / len(retrievedLabels)
    recallK = sum(retrievedLabels) / sum(y)
    xPlot.append(recallK)
    yPlot.append(precisionK)
\end{lstlisting}
\end{minipage}

Plotting these $x$ and $y$ coordinates results in the plot in \Cref{fig:PR_ROC} (right); the ROC curve can be generated similarly.


% AUC? Hit rate? Or leave until recommender systems?
% Left until recommender systems


\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/ROC.pdf}
    \includegraphics[scale=0.75]{fig/PRcurve.pdf}
    \caption[Receiver-Operating Characteristic and Precision Recall curves.]{Examples of Receiver-Operating Characteristic (left) and Precision Recall (right) curves.}
    %\done{Why do the plots look funny? Seem to have big fonts. Looks fine now}
    \label{fig:PR_ROC}
\end{figure}


We revisit evaluation techniques based on ranking in \Cref{sec:evaluateRec}, when we explore evaluation strategies for recommender systems.




\section{The Learning Pipeline}
\label{sec:learningPipeline}

By now we have covered many of the individual components that go into building a predictive model: model fitting (for regression models in \Cref{sec:linearReg}), feature engineering (\cref{sec:featureEngineering}), and evaluation (\cref{sec:evaluateRegression}). Bringing these components together still requires filling in some additional details. How can we know whether our model will work well when deployed (i.e.,~on \emph{new} data), and what steps can be taken to ensure this? How can we decide between various alternatives in terms of feature design, and meaningfully compare those alternatives against each other? 
%
Collectively these steps are part of the \emph{pipeline} of machine learning.
%\done{fluff paragraph}

\subsection{Generalization, Overfitting and Underfitting}

So far, when discussing model evaluation in 
\Cref{sec:evaluatingClassifiers} (and earlier in \Cref{sec:evaluateRegression}), we've considered training a model to predict labels $y$ from a dataset $X$; we've then evaluated the model by comparing the predictions $f(x_i)$ to the labels $y_i$. Critically, we're using the \emph{same} data to train the model as we're using to evaluate it.

The risk in doing so is that our model may not \emph{generalize} well to new data. For example, when fitting a model relating review length to ratings (as in \cref{fig:ratingLengthBestFit,fig:cubicFit}), we considered fitting the data with linear, quadratic, and cubic functions. Increasing the degree of the polynomial would continue to lower the errors of the predictor; alternately, we could have modeled review length using a one-hot encoding (so that there was a different predicted value for every length). Such models could fit the data very closely (in terms of their MSE), but it is unclear whether they would capture meaningful trends in the data or simply `memorize' it.
%In spite of their low MSEs (or high $R^2$ coefficients), these models would not seem to meaningfully capture the \emph{trends} in the data; rather, they are effectively `memorizing' meaningless details in the observed data.

To consider an extreme case, imagine fitting a vector $y$ using only \emph{random} features. The code below fits a vector of 50 observations using 1, 10, 25, and 50 random features, and then prints the $R^2$ coefficient of each model:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
y = numpy.random.rand(50)
mod = linear_model.LinearRegression()
for n in [1,10,25,50]:
    X = numpy.random.rand(50,n)
    mod.fit(X,y)
    print(mod.score(X,y))
\end{lstlisting}
\end{minipage}

Here, the $R^2$ coefficients take values of 
% 0.07321723526307
% 0.24827931383412216
% 0.3478616991354071
% 1.0
0.07, 0.25, 0.35, and 1.0---once we include 50 random features, we can fit the data perfectly. Of course, given that our features were \emph{random}, this `fit' is not meaningful, and the model has merely discovered random correlations between the observed data and labels.

The arguments above point to two issues that need to be addressed when training a model:
\begin{enumerate}
    \item We should not evaluate a model on the same data that was used to train it. Rather we should use a held-out dataset (i.e.,~a \emph{test} set).
    \item Features that improve performance on the training data will not necessarily improve performance on the held-out data. 
\end{enumerate}
%\done{Do I ever use enumerates or always itemize? Yes but not here}
Evaluating a model on held-out data gives us a sense of how well we can expect that model to work `in the wild.' 
This held-out data, known as a \emph{test set},
%Thus we will measure
measures
how well our model can be expected to \emph{generalize} to new data. 
%We'll explore strategies for building such held-out data in below.
%We'll explore the roles of test sets
%strategies for building test sets below.
%and strategies for building them below.

\subsubsection{Overfitting}
%\done{Don't like the paragraph name since it matches the subsection name}

Fundamentally, if our model works well on training data but not on held-out data, it must mean that certain characteristics of the training data 
are not representative of the held-out data. This could occur for various reasons. 
One possibility is that our held-out data is drawn from a different distribution from the training data.
For instance, if we had withheld sales data from the most recent month, and trained on data from the previous eleven, 
%it is possible that 
the most recent month of observations may follow a different trend, or occur during a different season, etc.
%
In principle, one might address the above simply by ensuring that the training and test sets
%\done{have I defined test set? No} 
are (non-overlapping) \emph{random samples} of the data, such that both the training and test data will be drawn from the same distribution.\footnote{Though if our goal is to \emph{forecast} next month's sales, using the most recent data as our held out sample may be the most appropriate decision.}

%Addressing the second point above, if a model works well on the training data but not on held-out data, 
Even if the training and held out data are independently drawn samples from the same distribution, we may still observe significantly degraded performance on our held-out data. In such cases we are said to be 
\emph{overfitting}.

A trivial demonstration of overfitting is shown in \Cref{fig:overfit}. Here we show a dataset that follows a line, subject to some random perturbation. While a high-degree polynomial can fit the data very closely, we would not expect this complex function to generalize well to new data. 
%This again demonstrates the principle of
%Occam's Razor, as we saw in \Cref{sec:regularization}, where we would expect the simpler model to better fit new data. 
We are said to have \emph{overfit} when we fit a model that is highly accurate on the training data, but that does not generalize well.
%\done{too repetitive, need to read section together. Okay}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/overfit.pdf}
\caption[Overfitting demonstration.]{Overfitting demonstration. A high-degree polynomial fits the observed data accurately, but is unlikely to generalize well. \label{fig:overfit}}
\end{SCfigure}

Note that we \emph{expect} any model to perform somewhat worse when applied to new data compared to its training performance: in fact this is one of our `theorems' about model performance that we present in \Cref{sec:regularization}. Rather our goal when tuning a model (or selecting among model alternatives)
%the parameter $\lambda$ (\cref{eq:accPlusReg}) %using our validation set 
is to minimize this gap, typically by sacrificing training accuracy in order to improve generalization performance.
%\done{validation sets not defined yet. Removed}
%\done{haven't defined $\lambda$ yet}
%\done{Feel like text is repeated, and maybe even example. Double check}

\subsubsection{Underfitting}
Just as we overfit by fitting a model whose good performance on a training set does not generalize to a held-out set, we \emph{underfit} when our model is insufficiently complex to capture the underlying dynamics in a dataset. Again, this can occur for a variety of reasons. If we select too simple a model, e.g.~a linear function to capture the data in \Cref{fig:cubicFit} (which doesn't seem to follow a linear trend), no choice of parameters will lead to good training \emph{or} held-out performance. 
%\done{style: held-out vs testing, this section. Fine}
%\donedo{not the cleanest. Fine}
%\done{haven't defined lambda yet}

%\done{section looks done}




\subsection{Model Complexity and Regularization}
\label{sec:regularization}

So far we have talked vaguely about what it means for a model to be `too complex' (or too simple) and suggested that we should choose a model that is complex enough to fit the data, but simple enough not to overfit.
%\done{have we actually discussed those things? Yes}
This idea is often referred to as \emph{Occam's Razor}, a philosophical principle which states that among several alternate hypotheses that explain some phenomenon, one should favor the simplest.
%\done{drop occam's razor maybe. It's fine}


%\done{never defined "regularization"}

However for these notions to be useful we must be precise about what it means for a model to be `complex.' We would like to define complexity in terms of the parameters $\theta$, such that given a fixed set of features and labels, we could select the `simplest' $\theta$ that adequately explains (or models) the data.

We'll discuss two candidate notions of `simplicity' as follows:
\begin{enumerate}
    \item A simple model is one that includes only a few terms, i.e.,~in which only a few values $\theta_k$ are non-zero.
    \item A simple model is one in which all terms are about equally important, i.e.,~one in which particularly large values of $\theta_k$ are rare.
\end{enumerate}
% Below we'll argue that these two notions of `complexity' are captured by the following expressions:
These
two potential notions of `complexity' are captured by the following expressions:
%\done{equation doesn't look right}
% \begin{align}
\begin{alignat}{3}
    \Omega_1(\theta) & = \| \theta \|_1 && = \sum_k | \theta_k |, \label{eq:l1Reg}\\
    \Omega_2(\theta) & = \| \theta \|_2^2 && = \sum_k \theta^2_k \label{eq:l2Reg}
\end{alignat}
% \end{align}
i.e.,~the sum of absolute values
and the sum of squares, 
also called the $\ell_1$ and (squared) $\ell_2$ norms of $\theta$. 
We state without proof that these expressions penalize models that have 
many non-zero parameters (\cref{eq:l1Reg})
or large parameters (\cref{eq:l2Reg}), though we further characterize their behavior later.

%Explanation:
%https://stats.stackexchange.com/questions/45643/why-l1-norm-for-sparse-models


%\done{A little more structure, maybe paragraph headings or similar. Added subsubsection below}

\subsubsection{Regularization}
In order to fit a model which simultaneously explains the data but is not overly complex (corresponding to our goal above),
we write down a new objective that combines our original accuracy objective
%from \Cref{eq_?} 
with one of the complexity expressions above (in this case the squared $\ell_2$ norm).
For a regression model we add the regularizer to the expression from \Cref{eq:MSE}:
%\done{include regression and classification here}
%\done{mentio derivatives}
\begin{equation}
    \underbrace{\frac{1}{|y|}\sum_{i = 1}^{|y|}\vphantom{\sum_k} (x_i \cdot \theta - y_i)^2}_{\text{\vphantom{ly}accuracy}} + \lambda \underbrace{\sum_k \theta^2}_{\mathclap{\text{model complexity $\| \theta \|_2^2$}}}.
    \label{eq:accPlusReg}
\end{equation}
For a classification model, we \emph{subtract} the regularizer, since we seek to maximize accuracy rather than minimizing error (so we maximize $-\lambda \| \theta \|_2^2$ rather than minimizing $\lambda \| \theta \|_2^2$):
\begin{equation}
     \sum_{i} - \log(1 + e^{-x_i\cdot \theta}) + \sum_{y_i = 0} -x_i \cdot \theta - \lambda \| \theta \|_2^2. \label{eq:logisticDeriveWithLamb2}
\end{equation}
This procedure---where we add a
%adding a 
penalty term to control model complexity---is known as \emph{regularization}; the parameter $\lambda$, which 
%trades-off accuracy versus model complexity 
controls the extent to which complexity is penalized,
is termed a regularization parameter.
%Or, as above, if we tune $\lambda$ (\cref{eq:accPlusReg}) poorly ($\lambda$ too large in the case of underfitting), we will sacrifice training performance unnecessarily by choosing too simple a model.
%\done{where is hyperparameter defined? If this is first use move here}
% $\lambda \simeq 0$ means that we do not penalize model complexity

Note that we can straightforwardly adapt the derivatives (from \cref{eq:derivative1,eq:logisticDeriveWithoutLamb4}) to include the regularization term, $\lambda \| \theta \|_2^2$ by noting that $\frac{\partial}{\partial \theta_k} \lambda \| \theta \|_2^2 = 2\lambda\theta_k$.
%\Cref{eq:accPlusReg,eq:logisticDeriveWithLamb2}

\subsubsection{Hyperparameters}
Our regularization parameter $\lambda$ in \Cref{eq:accPlusReg} is said to be a model \emph{hyperparameter}.
%is a \emph{hyperparameter} that trades-off accuracy versus model complexity. 
Hyperparameters are model parameters whose values control the model and influence other parameters (in this case $\lambda$ controls the fitted values of $\theta$). More complex models may have several tunable hyperparameters that control various model components. Note that generally we cannot fit hyperparameters in the same way that we fit model parameters (e.g.~if we used our training set to choose $\lambda$ in \Cref{eq:accPlusReg}, we would always choose a model that simply ignored model complexity in favor of accuracy). As such we need a separate strategy to tune model hyperparameters, which we explore when we introduce validation sets below.
%we tuned $\lambda$ using gradient descent (as in \cref{sec_?}) we would always 


Note that generally speaking the term $\theta_0$, i.e.,~the \emph{offset} term, should not be included in the regularizer, i.e.,~our regularizer should be $\sum_{k=1}^K \theta_k^2$ or $\sum_{k=1}^K |\theta_k|$. That is, our underlying assumption that few parameters are non-zero, or that parameters are small, should not apply to the offset term. If we were to include the offset term when regularizing, we would generally select a model which made systematically smaller (in magnitude) predictions.
%\done{cite exercise. No such exercise} 

\subsubsection{Fitting the regularized model (regression)}
%\done{added late, should make sure it jives with surrounding sections. Fine}
%
%https://www.quora.com/What-is-the-Alternating-Least-Squares-method-in-recommendation-systems-And-why-does-this-algorithm-work-intuition-behind-this
When introducing linear models in \Cref{sec:linearReg} we noted that the system in \Cref{eq:sysOfEq1} has a simple closed-form solution based on the pseudoinverse (\cref{eq:pseudoInv}). 
%As we develop 
Briefly we note that this solution can be fairly straightforwardly modified to fit the \emph{regularized} model as follows:
\begin{equation}
%\done{looks wrong, has no lambda. Fixed}
    \theta = (X^TX + \lambda I)^{-1}X^Ty
    \label{eq:fitWithLambda}
\end{equation}
(where $I$ is the identity matrix). Generally, as we develop more complex models, we'll move away from closed-form solutions,
%(beginning in \Cref{chap:classPipe}), 
though this specific solution proves useful when fitting certain types of recommender systems in \Cref{chap:modelbasedRecsys} (\cref{sec:onlineUpdates}).
%\done{exercise on this? No}



\subsubsection{Validation sets}
We now require a protocol for choosing the best value of the trade-off parameter $\lambda$ in \Cref{eq:accPlusReg}. If we were to select $\lambda$ based on the accuracy on the training set, we would always select $\lambda = 0$; 
%and 
ideally, we want to choose 
the value of 
$\lambda$ 
%in \Cref{eq:accPlusReg} 
that will result in the best performance on the held-out test set. However 
%note
%\done{check that we're recalling from somewhere}
%that we cannot 
we should be careful not to
use the test set to compare alternative models: the test set is supposed to represent true held-out performance, and strictly speaking test performance should only be examined \emph{after} we have selected our best model.
%\done{I think}

As such, we need a third data partition 
%of our data 
which can be used to select among alternative models. This \emph{validation set} 
in some sense mimics the test set, in that it is not used to fit the model, but is used to give us an \emph{estimate} of what we expect the test performance to be under a certain model.
%modeling conditions.

%As such, 
A typical pipeline will consist of three partitions of our data, whose roles are summarized as follows:
%\done{need to move probably}
\begin{itemize}
  \item The \textbf{training set} is used to optimize the parameters of a specific model. In the type of models we fit in this book, this usually refers to the parameters that can be fit via gradient ascent/descent (i.e.,~$\theta$ in this chapter).
  \item The \textbf{validation set} 
  %\done{capitalization? Doesn't matter}
  is used to select among model alternatives. 
  %`Model 
  `Alternatives' may simply mean different values of $\lambda$ in \Cref{eq:accPlusReg,eq:logisticDeriveWithLamb2}, but could also mean different feature representation strategies, etc. We discuss a few alternative uses below. Typically we select the model with the highest accuracy/lowest error on the validation set.
  \item The \textbf{test set} is used to evaluate held-out model performance; ideally it should not be used to make any modeling decisions, but should only be used to report performance.
\end{itemize}

The core use of our validation set is to estimate model hyperparameters such as $\lambda$ above. Beyond regularization coefficients, `hyperparameters' more broadly refer to any tunable model components that do not get optimized during the training phase. For example, when building models from text in \Cref{chap:text}, the number of words in our dictionary (from which we build features) would be an example of a hyperparameter, and could be chosen using our validation set to select the model that will generalize the best.

The relationship between 
%these three 
training, validation, and test
sets is shown in \Cref{fig:trainValidTest}, and some overall guidelines for building a validation set are shown in \Cref{box:validationGuidelines}.

\begin{figure}
    \centering
\begin{tikzpicture}
\draw (0,1.6) rectangle (10,1.1) node[pos=.5] {Dataset $\lbrace X; y \rbrace$};
\draw (0,1) rectangle (5,0.5) node[pos=.5] (train) {Training Set};
\node [below=of train,yshift=3em] {Estimate $\theta$};
\draw (5.125,1) rectangle (7.375,0.5) node[pos=.5] (valid) {Validation Set};
\node [below=of valid,yshift=3em] {Choose $\lambda$};
\draw (10,1) rectangle (7.5,0.5) node[pos=.5] (test) {Test Set};
\node [below=of test,yshift=3em] {Evaluation only};

\end{tikzpicture}
    \caption{Basic roles of training, validation, and test sets.}
    \label{fig:trainValidTest}
\end{figure}
%\done{reference figure}



\begin{figure}
\caption{Guidelines for building training, validation, and tests sets.}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
\begin{itemize}
    \item Training, validation, and test sets should (generally speaking) be non-overlapping, random samples of a dataset. Some exceptions apply, for example when fitting temporal models in \Cref{chap:temporalSequential}, we might build a test set out of the \emph{most recent} observations in order to get a sense of how well a model would work \emph{now} rather than how well it would work \emph{on average}.
    \item The size of our training set may be driven by modeling as well as practical concerns. Our training set should be large enough that we can reasonably expect to fit our model on the data (as a guideline, we might hope to have an order of magnitude more training examples than model parameters); likewise if we have a simple model with just a few parameters we need not train on millions of observations.
    \item Likewise the size of our validation and test sets should be large enough that we can be reasonably confident of our results. We briefly touch upon measuring significance in \Cref{subsec:significance}.
\end{itemize}
\end{minipage}
}
\label{box:validationGuidelines} 
\end{figure} 
%\done{I think}
%\done{make sure all figures/tables/boxes referenced}
%\done{maybe make this not a box. Is fine}





%\done{structure in this section. Is okay}

\subsubsection{Why does the $\boldsymbol{\ell}_{\boldsymbol{1}}$ norm induce sparsity?}
%\done{probably put somewhere else. It's okay}

We stated briefly when introducing \Cref{eq:l2Reg,eq:l1Reg} that an $\ell_1$ regularizer will encourage a \emph{sparse} parameter vector $\theta$, while an $\ell_2$ regularizer will result in a more balanced parameter distribution.
%
%\done{funny paragraph name. Fine}
%\done{connecting text / paragraph. Put this paragraph in the right place}
Without formally proving 
%why the $\ell_1$ norm induces sparsity 
this result,
it is instructive to consider some simple geometric intuition as to why this should be the case.
\Cref{fig:l1l2} demonstrates (for a simple two-parameter model) why the $\ell_1$ norm induces sparsity (i.e.,~few non-zero parameters) while the $\ell_2$ norm results in a more uniform parameter distribution. Models with equivalent $\ell_1$ norm lie along diamond-shaped contours (\cref{fig:l1l2}, left) whereas models with equivalent $\ell_2$ norm lie on a circle (\cref{fig:l1l2}, right). Models with an equivalent MSE
%\done{style: mean-squared, mean squared, Mean Squared, etc.}
lie along an ellipse. When balancing the error and the regularizer as in \Cref{eq:accPlusReg}, the best model will correspond to the point where the boundaries intersect. In the case of the $\ell_1$ norm these curves intersect on \emph{on one of the vertices of the diamond}; for the $\ell_2$ norm they do not;
%\done{mention that the ell_2 norm results in more uniform parameters}
the former case corresponds to a model
%whereas for the $\ell_2$ norm 
%corresponding to a model 
with only a few non-zero parameters.
For a more rigorous explanation of this phenomenon see e.g.~\cite{friedman2001elements}.

%\done{mention that validation sets are for hyperparams. Somewhere}

\begin{figure}
    \begin{center}
    \includegraphics[scale=0.75]{fig/normDemo1.pdf}
    \includegraphics[scale=0.75]{fig/normDemo2.pdf}
    \end{center}
    \caption[The regularization effect of $\ell_1$ versus $\ell_2$ norms.]{Demonstration of the regularization effect of the $\ell_1$ (left) versus $\ell_2$ norms. Dashed lines indicate models with equivalent norms ($\ell_1$ or $\ell_2$); solid lines indicate models with equivalent Mean Squared Errors. The selected model in either condition is circled. \label{fig:l1l2}}
    %\done{kinda ugly plots. Could remove boundaries and place \theta_1, \theta_2 along axes. Not critical}
\end{figure}



\subsubsection{`Theorems' regarding training, testing, and validation sets}

To solidify the roles of training, validation, and test sets, below we outline some theorems guiding the relationships among these sets, as the regularization hyperparameter $\lambda$ changes.

Note that these are `theorems' in the sense that they will be true in general, but only in the limit given large enough datasets, and assuming our training, validation, and test sets are drawn from the same distribution (etc.). As such, these theorems should mostly be regarded as guidelines to `sanity check' the correctness of your model pipeline:
%\done{anything more to say? This is fine}
\begin{itemize}
    \item The training error increases as $\lambda$ increases; typically 
    %for large $\lambda$ 
    it will asymptote to some value, e.g.~a linear model might asymptote to the error of a trivial predictor (i.e.,~the variance of the label).
    \item The validation and test errors will be at least as high as the training error; intuitively, the algorithm will not work better on `unseen' data than it did on training data.
    \item When $\lambda$ is too small, a too-complex model achieves low training error, but high validation/test errors. In this case, the model is said to be \emph{overfitting}.
    \item When $\lambda$ is too large, a too-simple model has high training, validation, and test error. In this case, the model is said to be \emph{underfitting}.
    \item Generally, there should be a `sweet spot' between under- and over-fitting, which is determined using our validation set. This point (marked in \Cref{fig:trainValidCurves} with an `x') corresponds to the model we expect to yield the best generalization performance on the test set.
\end{itemize}

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/validation.pdf}
    \caption[Example train, validation, and test curves.]{Example train, validation, and test curves, demonstrating the relationships between each type of error.}
    \label{fig:trainValidCurves}
\end{figure}
%\done{figure referenced? Yes}


\subsection{Guidelines for Model Pipelines}

Having introduced the conceptual details of a model pipeline, it is worth finishing with some practical advice on how to combine the various pieces and how to set the various tunable components of a model pipeline:
\begin{itemize}
    \item If you do \emph{not} see a `sweet spot' between under- and over-fitting (as in the theorems above), it could mean that you have not adequately explored the range of regularization coefficients. For example, if you observe monotonically increasing validation errors, it may mean you have not considered sufficiently small values of $\lambda$. Alternately, given (e.g.)~a simple linear model with only a few parameters, it may simply mean that your model is not capable of fitting (or overfitting to) a particular dataset.
    \item Regularization parameters such as $\lambda$ do not have an absolute scale, and will vary depending on factors ranging from the model's tendency to overfit, to the specific scale of the features $X$ and labels $y$.
    %\done{formatting of x and y}
    As a rough guideline, it is useful to consider setting $\lambda$ by considering several different orders of magnitude (as we do in \Cref{sec:implementPipeline}), before honing in on a narrower range of values.
    \item When implementing iterative models (such as approaches based on gradient descent, as in \Cref{sec:gradient}), the validation set can be used as a condition to cease further iteration. That is, we need not train models until convergence: if we are making no further improvements on the validation set (say, for a predetermined number of iterations), there is little reason to continue optimizing our model on the training set. Ideally, the model parameters $\theta$ might be chosen from whichever iteration yields the best \emph{validation} performance.
\end{itemize}
%\done{more? No}

% \subsubsection{Uses of a validation set}
%\done{Fold this and the above section together. It's fine}



% Our validation set can also be used when training a model. For instance, when we fit models via gradient descent (as in \cref{sec:gradient}), we might periodically evaluate the model's validation performance every few gradient iterations; rather than waiting until the model reaches a particular convergence threshold, we can simply terminate once performance is no longer improving on the validation set.
%\done{Short and not useful section, could be folded into something else -- fold into above as previous todo says. It's fine}




\subsection{Regression and Classification in Tensorflow}
\label{subsec:regressionTensorflow}

%\done{should include classification too}
%\done{finish}

Below we describe an implementation of (regularized) linear regression in \emph{Tensorflow},\footnote{\url{https://www.tensorflow.org/}},
%which will form the basis of more complex gradient-based optimization models we develop later.
forming the basis of the the overall pipeline we'll use to develop more complex models in later chapters.
%
Although widely associated with deep learning, Tensorflow can more simply be thought of as a general-purpose library for gradient-based optimization. Tensorflow computes derivatives \emph{symbolically}, meaning that the programmer must only specify the \emph{objective} to be optimized (e.g.~\cref{eq:accPlusReg}), without having to compute gradients. This makes it easy to quickly experiment with model variants, even for models including complex transformations of model parameters.

We first setup our observed variables ($X$ and $y$) as Tensorflow data constants:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X = tf.constant(X, dtype=tf.float32)
y = tf.constant(y, dtype=tf.float32)
\end{lstlisting}
\end{minipage}

Next we setup our model. Mostly this simply consists of defining the basic model components 
%(corresponding to \Cref{eq_?},\Cref{eq_?},\Cref{eq_?}) 
in terms of Tensorflow primitives:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last,escapechar=\%]
class regressionModel(tf.keras.Model):
    # Initialize with number of parameters and regularization strength
    def __init__(self, M, lamb):
        super(regressionModel, self).__init__()
        self.theta = tf.Variable(tf.constant([0.0]*M, shape=[M,1], dtype=tf.float32))
        self.lamb = lamb

    # Prediction (for a matrix of instances) %(\cref{eq:sysOfEq1})%
    def predict(self, X):
        return tf.matmul(X, self.theta)

    # Mean Squared Error %(\cref{eq:MSE})%
    def MSE(self, X, y):
        return tf.reduce_mean((tf.matmul(X, self.theta) - y)**2)

    # Regularizer %(\cref{eq:l2Reg})%
    def reg(self):
        return self.lamb * tf.reduce_sum(self.theta**2)

    # Loss %(\cref{eq:accPlusReg})%
    def call(self, X, y):
        return self.MSE(X, y) + self.reg()
\end{lstlisting}
\end{minipage}
%\done{reference equations in code}

Next we define an \emph{optimizer} to use (in this case the \emph{Adam} optimizer from \cite{kingma2014adam}), and create an instance of our model. Here we create a model with regularization strength $\lambda = 1$:
%\done{haven't defined regularization yet! Moved}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
optimizer = tf.keras.optimizers.Adam(0.01)
model = regressionModel(len(X[0]), 1)
\end{lstlisting}
\end{minipage}

Finally we run 1,000 iterations of gradient descent. Gradients are computed automatically for the objective defined in \texttt{call()} with respect to the model's variables ($\theta$):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
for iteration in range(1000):
    with tf.GradientTape() as tape:
        loss = model(X,y)
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    print(loss.numpy())
\end{lstlisting}
\end{minipage}

Again, although the above code implements a simple model (which we could already compute in closed form), the value of Tensorflow is that we can easily adapt our model to handle different objectives, including complex, non-linear transformations. For instance, we can easily replace our $\ell_2$ regularizer above with an $\ell_1$ regularizer:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
def reg1(self):
    return self.lamb * tf.reduce_sum(tf.abs(self.theta))
\end{lstlisting}
\end{minipage}

Finally we note that Tensorflow is but one of many popular libraries (see e.g.~Theano, PyTorch, MXNet, etc.),~though all implement the same basic functionality, in terms of performing gradient-based optimization on top of user-defined objectives.

\subsubsection{Classification}

Classification objectives can be built similarly. Given a vector of binary labels $y_i \in \lbrace 0, 1 \rbrace$, our prediction function is replaced by $\sigma(X \cdot \theta)$, and our objective is replaced by that of \Cref{eq:logisticDeriveWithoutLamb2} (along with a negative sign, so that we can still minimize the objective):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
# Probability (for a matrix of instances)
def predict(self, X):
    return tf.math.sigmoid(tf.matmul(X, self.theta))

# Objective as in %\Cref{eq:logisticDeriveWithoutLamb1}%
def obj(self, X, y):
    pred = self.predict(X)
    pos = y*tf.math.log(pred)
    neg = (1.0 - y)*tf.math.log(1.0 - pred)
    return -tf.reduce_mean(pos + neg)
\end{lstlisting}
\end{minipage}
%\done{annotate with equations}
%\done{should I indent code blocks like the one above? No}

%Note that 
In practice one rarely writes out such functions `longhand,' as standard objectives 
%like the above 
are available as \emph{Tensorflow} operations (e.g.~the above is equivalent to a \emph{binary cross-entropy} loss, \texttt{tf.keras.losses.BinaryCrossentropy()}). 

%\done{double check code}
%\done{only useful if there's comparible code elsewhere}
% \noindent
% \begin{minipage}{\linewidth}
% \begin{lstlisting}
% y = tf.constant(y, shape=[len(y),1]) # Convert to column vector
% M = len(X[0]) # Feature dimensionality

% # Regularized MSE, described using tensorflow matrix operations
% def MSE(X, y, theta, lambda):
%   return tf.reduce_mean((tf.matmul(X, theta) - y)**2) +
%          lambda * tf.reduce_sum(theta**2)

% # Initialize theta
% theta = tf.Variable(tf.constant([0.0]*M, shape=[M,1]))

% # Adam Optimizer
% optimizer = tf.train.AdamOptimizer(0.01)
% objective = optimizer.minimize(MSE(X,y,theta,1.0))
% \end{lstlisting}
% \end{minipage}
%\done{Don't think AdamOptimizer uses this syntax anymore}
%\done{Don't think "session" exists either}
%\done{Replace with tf.nn.l2_loss (and elsewhere)}
%\done{Has regularizer but haven't discussed regularization yet}



\section{Implementing the Learning Pipeline}
\label{sec:implementPipeline}
%\done{Why is this here? Could be in regression section. Seems fine here}

Below we briefly show how to practically apply the process from \Cref{sec:learningPipeline} to select a model based on training, validation, and test samples. 

The actual features for this model are based on a sentiment analysis (regression) task from \Cref{chap:text}, in which we predict ratings based on words in a review; for the sake of demonstrating a model pipeline it is useful to consider a problem with high-dimensional features, such that the model is prone to overfitting if not carefully regularized (in this case, we consider 1,000-dimensional features on a dataset with only 5,000 samples).

First, we randomly shuffle the dataset and split it into non-overlapping training, validation, and test samples:
%\done{`portions' is a funny word. Changed to samples}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
random.shuffle(data)
X = [feature(d) for d in data]
y = [d['review/overall'] for d in data]
Ntrain,Nvalid,Ntest = 4000,500,500
Xtrain,ytrain = X[:Ntrain],y[:Ntrain]
Xvalid,yvalid = X[Ntrain:Ntrain+Nvalid],y[Ntrain:Ntrain+Nvalid]
Xtest,ytest = X[Ntrain+Nvalid:],y[Ntrain+Nvalid:]
\end{lstlisting}
\end{minipage}

Next, we consider regularization coefficients $\lambda$ ranging from $\lambda = 10^{-3}$ to $\lambda = 10^{4}$. For each value, we train a model on the training set and evaluate its accuracy on the validation set; during each step, we keep track of the best-performing model in terms of its validation accuracy. The \texttt{Ridge} model below implements regularized linear regression, as in \Cref{eq:accPlusReg}:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
bestModel = None
bestVal = None
for l in [0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]:
    model = sklearn.linear_model.Ridge(l)
    model.fit(Xtrain, ytrain)
    predictValid = model.predict(Xvalid)
    MSEvalid = sum((yvalid - predictValid)**2)/len(yvalid)
    print('l = ' + str(l) + ', validation MSE = ' + str(MSEvalid))
    if bestVal == None or MSEvalid < bestVal:
        bestVal = MSEvalid
        bestModel = model
\end{lstlisting}
\end{minipage}
%\done{need to say what "Ridge" is}

Finally, we evaluate the best-performing model (in terms of validation performance) on the test set. Note that this is the first and only time we use the test set:
%\done{repetitive text. Don't see the problem}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
predictTest = bestModel.predict(Xtest)
MSEtest = sum((ytest - predictTest)**2)/len(ytest)
\end{lstlisting}
\end{minipage}

\Cref{fig:validationInPractice} shows the training, validation, and test performance found during the above steps; note the similarity to the hypothetical curves in \Cref{fig:trainValidCurves}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/validationInPractice.pdf}
\caption{Training, validation, and test error on a real pipeline. \label{fig:validationInPractice}}
\end{SCfigure}
%\done{Figure scale is not perfect, could redo. Looks fine}




\subsection{Significance Testing}
\label{subsec:significance}

Although not our focus in this chapter, it is worth briefly exploring how we can explicitly measure whether the performance of one model is `better' than another, in terms of a formal statistical framework. So far, we have compared (regression) models in terms of their Mean Squared Errors
%\done{style, hyphenation}
though as we explored in \Cref{sec:whyMSE}, the Mean Squared Error was chosen based on an underlying assumption that model errors 
follow a Gaussian distribution.
%are randomly distributed (under a Gaussian distribution).

\index{Yelp}
\emph{Significance testing} refers to the overall process of determining whether a statistical measurement would have been likely to have occurred due to chance alone (under the assumptions of some particular model). For example, 
%if two restaurants on \emph{Yelp}
%had average ratings of 4.5 and 4.3, but the former had two reviews while the latter had 50, which would you conclude was better? 
if an established restaurant on \emph{Yelp}
has a rating of 4.3 stars based on 50 reviews, and a new restaurant has a rating of 4.5 stars based on four reviews, would you conclude that the new restaurant is better rated? Or would you conclude that the higher initial rating is likely to have occurred due to chance? Significance tests allow us to formalize these questions.

Formally, a \emph{$p$-value} measures the probability that a result as (or more) extreme than the one we actually observed could have occurred due to chance (under some statistical model). E.g.~if we assume that users' ratings follow a Gaussian distribution, with what probability would the ratings of two restaurants deviate by (at least) 0.2 stars? 
In the case of this specific measurement, this probability would depend on (a) the 
%\done{(abc) or (123)? ABC}
\emph{magnitude} of the difference between the two averages; (b) the \emph{size} of the two samples (e.g.~a difference of 0.2 stars might be significant of both restaurants had 50 ratings, but not if they had four); and (c) the \emph{variance} of the two samples (e.g.~if the two samples had highly concentrated ratings we might more quickly conclude that the difference was significant).\footnote{This specific probability would be measured via a $t$-test.}

When making comparisons between models, 
%we are typically interested in 
we will typically use a $p$-value to measure
whether one model has residuals ($y - f_\theta(x)$)
%\done{notation}
that are closer to zero than another (i.e.,~we are testing whether one model's predictions are closer to the labels than the other's). To do so we are measuring the difference in \emph{variance} between two samples.

%whether the difference between the mean squared errors of two models is likely to have occurred due to chance.

%WRONG
%Without going through the derivation here, this specific $p$-value (between two random samples that are assumed to follow a Gaussian distribution) is computed by a $t$-test, and indeed is a function of the means, variances, and sizes of the two samples.

We will compute this quantity via an \emph{$F$-test}. Other tests could also be used to compare the performance of two models, such as a likelihood ratio test, each of which has different underlying assumptions.
%\done{clunky. Fine}
Below we'll compare the performance of two models for estimating a rating, using our beer review data as in \Cref{sec:binaryFeat}:
\begin{align}
    \text{model 1:} \quad \text{rating} & = \theta_0 \\
    \text{compared to model 2:} \quad \text{rating} & = \theta_0 + \theta_1 \times (\text{ABV})
    %\done{formatting / style. Looks fine}
\end{align}
One of the assumptions of this particular test is that one of the to models has a subset of the parameters of the other. As such we are really measuring whether the additional parameters significantly improve the model's performance (i.e.,~whether adding a term based on the ABV improves the performance of a model including only $\theta_0$).

First we generate features and labels for the two models (assuming the data has already been read, shuffled, etc.):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
X1 = [[1] for d in data]
X2 = [[1, d['beer/ABV']] for d in data]
y = [d['review/overall'] for d in data]
\end{lstlisting}
\end{minipage}

Next we fit the two models (on half of the data), and compute their residuals (on the other half):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
model1 = sklearn.linear_model.LinearRegression(fit_intercept=False)
model1.fit(X0[:250], y[:250])
residuals1 = model1.predict(X1[250:]) - y[250:]
model2 = sklearn.linear_model.LinearRegression(fit_intercept=False)
model2.fit(X2[:250], y[:250])
residuals2 = model2.predict(X2[250:]) - y[250:]
\end{lstlisting}
\end{minipage}

The actual $F$ statistic depends on the sum of squared residuals, the number of parameters in each model, and the size of the sample:

%\done{lstlistings seem to run over the page? Looks okay now}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
rss1 = sum([r**2 for r in residuals1]) # sum of squared residuals
rss2 = sum([r**2 for r in residuals2])
k1,k2 = 1,2 # Number of parameters of each model
n = len(residuals1) # Number of samples
\end{lstlisting}
\end{minipage}

\index{scipy}
Finally we compute the $F$ statistic and estimate the associated $p$-value using a method from \emph{scipy}:
%\done{formatting}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
F = ((rss1 - rss2) / (k2 - k1)) / (rss2 / (n-k2))
scipy.stats.f.cdf(F,k2-k1,n-k2)
\end{lstlisting}
\end{minipage}

The obtained $p$-value is close to zero, indicating that the result (that ABV improves predictive performance) is statistically significant.\footnote{One can experiment with less-predictive features or a smaller test set to see how these influence the estimated $p$-value.}

Note that this is just one example of a significance test, that works for a particular situation (albeit a fairly common one); in different situations alternative tests may be required, see e.g.~\cite{wasserman2013all} for a more comprehensive presentation.

In spite of the importance of rigorously demonstrating the significance of claimed model improvements,
%is in general an important best-practice to follow, 
we will mostly avoid further discussing significance testing throughout the remainder of this book. Generally speaking, these types of tests are designed for small-sample contexts (e.g.~surveys or clinical trials, etc.);~on the types of large datasets we consider, even small differences between models will tend to yield extremely small (highly significant) $p$-values.
%as opposed to machine learning on large datasets)

%\done{drop entire significance section? Or clean up? It's okay}

% 


% \section{Exercises}
%\done{Typesets ugly, with problem "section" followed by problems heading. Figure out how to avoid or add some text before the actual problems}
% \label{sec:basicsExercises}
%\done{make label apply to the right section after above change. Just reference as "Exercises"}

\begin{exercises}
 \item 
 \label{ex:beer1}
%  Fit a logistic regressor that estimates gender from review length (or another feature), i.e.,
%  $$
%  %\done{should ever use $$ or always equation? Only use $$ in exercises}
%   p(\text{gender is female}) \simeq \sigma(\theta_0 + \theta_1\times (\text{review length})).
%   %σ(θ0 + θ1 × [length])
%   %\done{do I use simeq? Done}
%  $$
%\done{say a bit more about the dataset}
In this exercise we'll use the style of a beer (using the same data we've studied since e.g.~\Cref{sec:binaryFeat}) to predict its ABV (alcohol by volume). Construct a one-hot encoding of the beer style, for
those categories that appear in more than 1,000 reviews. You can build a mapping of categories to feature
indices as follows:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
categoryCounts = defaultdict(int)
for d in data:
    categoryCounts[d['beer/style']] += 1
  
categories = [c for c in categoryCounts if categoryCounts[c] > 1000]
catID = dict(zip(list(categories),range(len(categories))))
\end{lstlisting}
\end{minipage}
%\done{indentation level of all code (four spaces)}

\noindent Train a logistic regressor using this one-hot encoding to predict whether beers have an ABV greater than 
7 percent (i.e.,~\texttt{d['beer/ABV'] > 7}). 
%Train the classifier on the training set and report its performance
%in terms of the accuracy and Balanced Error Rate (BER) on the test set, using a regularization constant
%of C = 10.
Report the True Positive, True Negative, False Positive, False Negative, and Balanced Error Rates of the
classifier (see \cref{sec:evaluatingClassifiers}).
%\done{clean up, (from https://cseweb.ucsd.edu/classes/fa20/cse258-a/files/homework2.pdf)}
\item The performance of the classifier above may be unsatisfactory due to the data being highly \emph{imbalanced} (as in \cref{sec:type1type2}). Implement a \emph{balanced} version of the classifier using the \texttt{class\_weight='balanced'} option in sklearn. Report the same metrics as above for the balanced classifier.
 \item 

 %-- Some question about generating precision / recall, and ROC curves
 Generate precision and recall curves for the classifier you trained above.
 \item 
  \label{ex:pipeline}
 Implement a complete learning and regularization pipeline with your balanced model. Split your data into 50\%/25\%/25\% train/validation/test fractions. Consider values $C$ in the range $\lbrace 10^{-6}, 10^{-5}, 10^{-4}, 10^{-3}\rbrace$.
 %\done{Footnote spans two pages. fix if still a problem}
Compute the train, and validation BER for each value of $C$, and the BER for the classifier that performs best on the validation set.\footnote{$C$ plays a similar role to $\lambda$ in \protect\Cref{eq:logisticDeriveWithLamb2}, though applies to the accuracy rather than the complexity term, i.e.,~small $\lambda$ is equivalent to large $C$. See documentation in \texttt{sklearn.linear\_model.LogisticRegression}.}
 \item 
 %Show that solving a classification problem by treating it as regression on 0/1 variables isn't effective, and reason about why
 Na\"ively, to build a classifier we might simply train a \emph{regressor} by treating labels as real-valued quantities (e.g.~predicting -1/+1). %\done{Looks okay, could copy some text from class version}
 Perform a simple experiment to demonstrate that this na\"ive model does not work as well as logistic
 regression. That is, select a dataset (such as the one you used in Exercise~\ref{ex:beer1}), a few features, a label to predict, and an appropriate classifier evaluation metric, to show that the na\"ive classifier is outperformed by logistic regression.
 %\done{Not written well. Fine}
\end{exercises}


\refstepcounter{projectCounter}
\label{proj:classif}
\section*{Project \theprojectCounter: Taxicab Tip Prediction (Part 2)}
\addcontentsline{toc}{section}{Project \theprojectCounter: Taxicab Tip Prediction (Part 2)}

Below we'll revisit Project~\ref{proj:1} to (a) consider classification techniques; and (b) more rigorously evaluate our models using a learning pipeline. Using the same data from Project~\ref{proj:1}, extend your project via the following steps:
\begin{enumerate}
    %\item Consider modeling the problem using a classification objective. In 
    \item Carefully build a complete model pipeline. That is, split your data into train, validation, and test portions, and build a pipeline so that all models are trained on the training set and comparisons among models are performed on the validation set (similar to Exercise~\ref{ex:pipeline}). Consider different ways to split the data, e.g.~is it better to split the data randomly, or is it better to withhold the most recent observations for testing for the sake of selecting the model most capable of \emph{forecasting} future trends?
    \item 
    %Are there any alternatives to formulating the task as a 
    Rather than modeling the task as a regression problem, you could cast the problem as \emph{classification} by estimating whether a tip will be above or below the median; this may be less sensitive to outliers. Consider the advantages and disadvantages of various formulations as well as what evaluation metrics you might use.
    \item Some of the features we used in Project~\ref{proj:1} are potentially quite high dimensional, e.g.~if we encode timestamps using one-hot encodings for each possible day of the year, our model might be highly effective at capturing single-day trends (such as major holidays), but could also be prone to overfitting. Use your pipeline to select the best feature representation among alternatives (e.g.~different levels of granularity for your temporal features, or otherwise), and to incorporate a regularizer into your model.
    %(as in \cref{eq:logisticDeriveWithLamb2}).
\end{enumerate}

%-- Converting real-world datasets to 

%-- Show relationship between pseudoinverse and derivative-based solution



% -- Show that MSE residuals have average zero

% -- Show that Laplace MAE is MLE for Laplace distribution

% -- Exercise about MSE = mean and MAE = Median



%-- Some question about generating precision / recall, and ROC curves

%-- Compare a model that includes $\theta_0$ in the regularizer to one that doesn't %NOT INTERESTING

%\done{project}


% Exercise:
% At the beginning of \Cref{sec:classification} 

%See: https://latex.org/forum/viewtopic.php?t=16601

\part{Fundamentals of Personalized Machine Learning}
%\addcontentsline{toc}{part}{\texorpdfstring{\MakeUppercase{section}}{}}

% \part*{Fundamentals of Personalized Machine Learning}
% \addcontentsline{toc}{part}{\texorpdfstring{\MakeUppercase{PART ONE Fundamentals of Personalized Machine Learning}}{}}


\chapter{Introduction to Recommender Systems}
\label{chap:recsysIntro}

In \Cref{chap:basicsReg,chap:classPipe}, when revising regression and classification, our only means of providing \emph{personalized} predictions was to extract features associated with user characteristics (e.g.~age, location, gender).
%\done{Change this in case I break it into two chapters. Okay}
%While effective in certain scenarios, 
The success of such models largely depends on our ability to extract features that adequately explain the variation in the labels we are trying to predict. While effective in a number regression or classification scenarios, when modeling interactions in recommendation scenarios, it is less clear what features are predictive of users' actions, and less likely that those features could be collected in the first place. Consider for example:
\begin{itemize}
    \item What features would be useful to predict what movies a user would be likely to watch? `Obvious' features such as user demographics may explain only a small fraction of the variation in interactions and preferences.
    \item How would you identify the types of features that would be useful for an obscure or unusual domain? For example what features would you collect to recommend baby toys, toaster ovens, or temporary tattoos?
    %\done{good example? Good enough}
    \item Are such features likely to be available? In practice, we will often know 
    %\emph{nothing} 
    little
    about a user, other than their interaction history.
    \item How can  we
    %we use machine learning to 
    make predictions in settings where \emph{no} features are available?
\end{itemize}

%\emph{Recommender systems} are arguably the most basic and fundamental form of personalized machine learning.

\emph{Recommender systems} are a fundamental tool to try and make predictions in such
%types of 
scenarios.
At their core, recommender systems are concerned with understanding \emph{interactions} between users and items. 
%
Roughly speaking, recommender systems operate by finding common patterns and relationships among users and items,
so that recommendations for a user can be harvested from others who have similar interactions patterns.


In this chapter we explore approaches based on simple 
%set-based 
similarity 
heuristics. Our basic goal is to identify \emph{which items and which users are similar to each other}. %Trivial models 
Approaches range from simple heuristics like set overlap (\cref{sec:jaccard}), to more complex approaches based on random walks (\cref{sec:randomWalk}). As we'll discover, even simple heuristics can be surprisingly effective, and in practice drive high-profile industrial recommender systems, as we explore in our case study on \emph{Amazon} recommendations (\cref{sec:amazonCaseStudy}).
%in \Cref{sec:neighborhood}, 
%before exploring machine-learning based approaches in \Cref{sec:modelbased}.
%\done{not great}

%Ultimately, 
Critically,
the models we develop in this chapter are quite different from those 
%in previous chapters, 
we've seen so far, largely
eschewing explicit features in favor of techniques more closely related to pattern mining. 
%
%Note critically 
Note also that
%in this chapter that
for the moment
we will not use \emph{machine learning} to build recommenders: we use this chapter to explore the overall problem setting and pipeline, before exploring machine learning (or so-called `model-based') approaches in \Cref{chap:modelbasedRecsys}.





\section{Basic Setup and Problem Definition}
\label{sec:setup}

The typical modality of the data we are trying to model might consist of sequences of historical interactions between users and items, for example, we might have a collection of movie ratings such as:
\begin{equation}
    \begin{tabular}{llrr}
      (Julian, & \emph{The Godfather}, & 4, & Jan 4 2019)\\
      (Julian, & \emph{Pulp Fiction}, & 3, & Jan 6 2019)\\
      (Laura, & \emph{Seven Samurai}, & 5, & Jan 8 2019)\\
      (Laura, & \emph{The Godfather}, & 4, & Jan 11 2019)\\
      \multicolumn{4}{c}{$\vdots$}
    \end{tabular}
\end{equation}
which might further be anonymized in terms of user IDs, item IDs, and sequential timestamps:\footnote{The timestamp shown here is known as the \emph{unix time}, representing the number of seconds since January 1970 (in UTC); such a representation is often useful as it allows straightforward comparison between timestamps.}
\begin{equation}
    \begin{tabular}{llrr}
      (264, & 547, & 4, & 1546588800)\\
      (264, & 82, & 3, & 1546761600)\\
      (3473, & 231, & 5, & 1546934400)\\
      (3473, & 547, & 4, & 1547193600)\\
      \multicolumn{4}{c}{$\vdots$}
    \end{tabular}
\end{equation}
Such a format is ubiquitous across many popular recommendation datasets and tasks, including (for example) the popular \emph{Netflix Prize} dataset (\cref{sec:caseStudyNetflixTemporal}).
%\done{right ref?}
Such data may include `side information,' such as reviews, demographic information about the users, or metadata about the movies, but often it may not. In fact, in the simplest form it may not even include ratings or timestamps.

Thus in essence the data we are trying to work with simply describes \emph{interactions} among users and content. Such interactions could describe clicks, purchases, ratings, likes, (etc.). In other settings the interactions could describe social connections among users, or even `interactions' among compatible clothing items (\cref{sec:fashionCaseStudies}).

Given interaction data such as that above, we would now like to ask questions such as:
\begin{itemize}
    \item How will Laura rate Pulp Fiction?
    \item Given that Laura liked the The Godfather, what other movies will she like?
    \item What movie is Laura likely to rate next?
\end{itemize}
Answering these questions seems difficult, as we seemingly know very little about the users and items involved. However we do know, for instance, that both Laura and Julian (or users 3473 and 264) recently watched The Godfather, and gave it similar ratings; from this we could begin to reason that they may exhibit similar preferences with regard to other movies also.

Reasoning about these types of questions, and modeling these types of interactions, are the main goals of recommender systems.

\subsubsection{How is recommendation different from regression or classification?}
%\done{think about moving? This is fine}

In \Cref{chap:basicsReg,chap:classPipe}, we saw several techniques that seem like they could already be used to predict outcomes like ratings and purchases. For example, predicting a rating (e.g.~of a movie) seems like a traditional regression task, and we can imagine various user and movie features that might be associated with ratings. As such, na\"ively we might try to extract user and movie features and fit a linear model of the form
\begin{equation}
    \text{rating}(\text{user},\text{movie}) = \langle \underbrace{\phi(\text{user}, \text{movie})}_{\mathclap{\text{user and movie features}}}, \theta \rangle.
\end{equation}
User features might include attributes like the user's age, gender, location, or other demographic features that might be associated with rating patterns; movie features could capture the length, MPAA rating, budget, or presence of certain actors (etc.). Assuming user and movie features can be collected independently, and since the model is linear, this could be rewritten as
\begin{equation}
    \text{rating}(\text{user}, \text{movie}) = \langle \underbrace{\phi^{(u)}(\text{user})}_{\mathclap{\text{user features}}}, \overbrace{\theta^{(u)}}^{\mathclap{\text{user parameters}}} \rangle + \langle \underbrace{\phi^{(i)}(\text{movie})}_{\mathclap{\text{movie (item) features}}}, \overbrace{\theta^{(i)}}^{\mathclap{\text{movie parameters}}} \rangle.
    \label{eq:whyRecDifferent}
\end{equation}
When written this way, we can see that the prediction of the rating is the sum of two \emph{independent} predictions: one for the user (say $f(u)$) and one for the item (say $f(i)$). If we were to make recommendations based on these predictions, for example by recommending whichever unseen movie a user would give the highest rating to, i.e., \begin{equation}
  \argmax_{\mathclap{i \in {\text{unseen movies}}}} f(u) + f(i),
\end{equation}
our recommendation \emph{for every user} would simply be whichever movie had the highest predicted rating $f(i)$. In other words, every user would simply be recommended movies which had features associated with high ratings.

Critically, such a model could not \emph{personalize} its recommendations to individual users. Even if the model achieved a reasonable Mean Squared Error in terms of predicting ratings, it would not be an effective recommender system.

To overcome this limitation, a model must in some fashion capture \emph{interactions} between users and items, e.g.~how compatible is a user with a particular movie. 
%We discussed this idea a little in \Cref{chap:nonparametric}, 
%\done{rewrite once that chapter exists. Dropped}
%but investigate the idea in more detail here. 
%
%In summary, modeling \emph{interections} 
Explicitly modeling interactions
between users and items is the main goal of recommender systems and is the main characteristic that differentiates them from other types of machine learning (\cref{box:interactions}).

\begin{figure}
\caption{Recommender systems compared to other types of machine learning.}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
The main distinguishing feature of recommender systems compared to other types of machine learning is their goal of explicitly modeling \emph{interactions} between users and consumed items based on historical patterns. This feature allows the models to understand which items are \emph{compatible} with which users, and thus to make different recommendations to each user in a personalized way.
\end{minipage}
}
\label{box:interactions}
\end{figure}

%\section{Notation and Representation}
\section{Representations for Interaction Data}

%Basic notation is established in \Cref{tab:recsysNotation}. 
There are several ways we could represent the interaction data described above. Formally, we might simply describe the dataset as a set of tuples $(u,i,r,t)$, or $r_{u,i,t} \in \mathbb R$ indicating that a user $u$ entered the rating $r$ for item $i$ at time $t$.

But conceptually it is easier to think about these data in terms of sets or matrices. Set representations will be useful when establishing similarity between users in terms of sets of items they have consumed (or likewise similarity between items in terms of sets of users who have consumed them); matrix representations will be useful when developing models based on the concept of matrix factorization (or dimensionality reduction).

% \begin{table}
%     \centering
%     \begin{tabular}{ll}
%         \toprule
%          $u \in U$ & users $u$ in user set $U$\\
%          $i \in I$ & items $i$ in item set $I$\\
%          $I_u$ & set of items rated by user $u$\\
%          $U_i$ & set of users who have rated item $i$\\
%         \bottomrule
%     \end{tabular}
%     \caption{Common Recommender Systems Notation}
%     \label{tab:recsysNotation}
% \end{table}

\paragraph{Activities as sets}
%
For our simplest recommendation model, we can describe users in terms of the sets of items they have interacted with, e.g.~for a user $u$:
\begin{equation}
    I_u = \text{set of items consumed by $u$};
\end{equation}
likewise, we can describe items in terms of the sets of users who have interacted with them:
\begin{equation}
    U_i = \text{set of users who consumed item $i$}.
\end{equation}

\paragraph{Activities as matrices}
%
%\done{Explain that sets can't handle ratings}
%
Alternately, we can represent datasets of user/item interactions via matrices. 
%\done{rewrite with both a binary matrix and a rating matrix, then can write the set notations in terms of these}
%E.g.~a matrix encoding which i:
Interaction matrices could describe which items a user has interacted with ($C$), or could augment our previous representations to capture real-valued interaction signals such as ratings ($R$):
\begin{equation}
    R = \underbrace{\left[ \begin{tabular}{ccccc}
    5 & $\cdot$ & $\cdot$ & 2 & 3\\
    $\cdot$ & 4 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 5 & 5 & 3 & $\cdot$\\
    5 & $\cdot$ & 4 & $\cdot$ & 4\\
    1 & 1 & $\cdot$ & 4 & 5\\
    \end{tabular} \right]}_{\let\scriptstyle\textstyle
    \substack{\text{items}}}
    \quad
    \mathit{C} = \underbrace{\left[ \begin{tabular}{ccccc}
    1 & $\cdot$ & $\cdot$ & 1 & 1\\
    $\cdot$ & 1 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 1 & 1 & 1 & $\cdot$\\
    1 & $\cdot$ & 1 & $\cdot$ & 1\\
    1 & 1 & $\cdot$ & 1 & 1\\
    \end{tabular} \right]}_{\let\scriptstyle\textstyle
    \substack{\text{items}}} \left. \mathclap{\begin{tabular}{c} \\ \\ \\ \\ \\ \end{tabular}} \right\rbrace \text{users}.
    \label{eq:interactionMatrix}
\end{equation}
Each row of $R$ represents a single user, and each column represents a single item. A particular entry $R_{u,i}$ indicates the rating the user $u$ gave to item $i$. Note that the vast majority of entries in such a matrix would typically be missing (most users do not rate most items); indeed, the missing entries are exactly the quantities that we would like to predict.\footnote{It should be carefully noted that our matrix representation is largely conceptual: it is rarely feasible to enumerate a complete interaction matrix, which could have millions of rows (users) and columns (items). In practice we'll represent interaction matrices using sparse data structures mapping user/item pairs to observed values.}

Naturally, the set and matrix representations can be written in terms of each other, for example, our set representation is equivalent to:
\begin{align}
    I_u & = \lbrace i \mid R_{u,i} \neq 0 \rbrace\\ % = \lbrace i | C_{u,i} \neq 1 \rbrace
    U_i & = \lbrace u \mid R_{u,i} \neq 0 \rbrace. %= \lbrace i | C_{u,i} \neq 1 \rbrace
\end{align}

Both our set and matrix representations of interactions may seem limited---neither conveys the timestamps associated with the ratings (or any other side-information), and the set-based representation doesn't even encode users' ratings. Nevertheless they are useful for reasoning about the basic principles behind recommender systems, which will become the building blocks behind more sophisticated approaches.
%, and in fact these simple techniques are 
%among the most 
%widely deployed.
%\done{Not beautiful sentenc, lacks evidence. Is okay}

\section{Memory-based Approaches to Recommendation}
\label{sec:neighborhood}

%\done{Maybe call this section memory based rather than neighborhood based} 
Perhaps the simplest (and most ubiquitous) approaches to recommendation are based on some notion of `similarity' among items. That is, an item is recommended to a user because it is similar to one that they they have recently clicked, liked, or consumed.

`People who viewed X also viewed Y' (or `people who bought X also bought Y,' etc.)~features 
%(\cref{fig:alsoviewed}) 
are familiar examples of such similarity-based recommenders. Items are recommended to a user on the basis of how \emph{similar} they are to an item the user is currently browsing.

% \begin{SCfigure}
%     \fbox{\includegraphics[width=0.7\textwidth]{fig/alsoviewed.png}}
%     \caption{An example of memory (or `neighborhood')-based recommendation (`People who viewed X also viewed Y').
%     \label{fig:alsoviewed}}
% \end{SCfigure}

For such a recommender to be effective depends on choosing an appropriate similarity function. The similarity function 
%in \Cref{fig:alsoviewed} 
that guides such models
might be based on 
%view data (presumably clicks); 
click or purchase data (as we see in \Cref{sec:amazonCaseStudy});
but even then, by what metric should we consider patterns of clicks to be `similar'? Should we count the number of users who have clicked on both items? Or do we need some kind of normalization? Or should we consider temporal recency?
%\done{style, too informal}

Appropriately designing such similarity functions, and recommending on the basis of such similarity, is the task of so-called memory-based recommender systems.
Such systems are said to be `memory'-based since they make predictions directly from data (rather than from the parameters of a model derived from data). Most of the approaches we'll see below are alternatively titled 
\emph{neighborhood-based} recommender systems, in which items are recommended due to being in the \emph{neighborhood} of (i.e.,~similar to) other items.\footnote{Alternately various types of recommender systems are also termed \emph{collaborative filtering}, though we generally avoid the term. Such models are `collaborative' in the sense that the predictions of one user are based on those of others.} We summarize this distinction in \Cref{box:modelBased}, which we discuss further when presenting model-based approaches in \Cref{chap:modelbasedRecsys}.

\begin{figure}
\caption{Memory-based and model-based recommender systems. \label{box:modelBased}}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
The 
%neighborhood-based recommendation 
approaches we study in \Cref{sec:neighborhood} make recommendations by writing down similarity functions that take as input user and item interaction histories. There is no \emph{model} (i.e.,~no parameters) associated with each user or item; the original data is used as an input to the similarity functions. Such systems are referred to as \emph{memory-based} (or neighborhood-based) recommenders.
In contrast, the techniques we'll develop in \Cref{chap:modelbasedRecsys} learn \emph{representations} of each user and item, such that the original data generally isn't used directly when making predictions at test time. Such systems are referred to as \emph{model-based} recommenders.
%\done{Confusing, not written great. Fine}
\end{minipage}
}
\end{figure}

\subsection{Defining a Similarity Function}
\label{subsec:definingSim}

Defining a similarity-based item-to-item recommender 
%like the one in \Cref{fig:alsoviewed} 
essentially requires that we define a similarity function among items:
\begin{equation}
    \mathrm{Sim}(i,j),
\end{equation}
and then given a query item $i$, recommend a set of items $j$ that maximize the given similarity.
%, i.e.,~$\argmax_j \mathrm{Sim}(i,j)$.

Let's consider a small toy example, with four items, and sets of users (or rather user IDs) who have consumed each:
\begin{equation}
    \begin{tabular}{l}
    $U_1 = \lbrace 1, 3, 4, 8, 12, 15, 17, 24, 35, 39, 41, 43 \rbrace$\\
    $U_2 = \lbrace 2, 3, 4, 5, 9, 12, 13, 16, 19, 24, 27, 31 \rbrace$\\
    $U_3 = \lbrace 4, 5, 9, 12 \rbrace$\\
    $U_4 = \lbrace 4, 9 \rbrace$
    \end{tabular}
    \label{eq:simToy}
\end{equation}
(recall that in our notation $U_1$ represents the set of users who have bought item 1).
Na\"ively, we might assume that an item-to-item recommender 
%like the one depicted in \Cref{fig:alsoviewed} 
(e.g.~`people who bought X also bought Y')
%\done{style}
is simply counting the number of users who purchased both items in common. In our set notation this would be:
\begin{equation}
    \mathrm{Sim}(i,j) = | U_i \cap U_j |.
\end{equation}
Computing some similarities under this model we would find:
\begin{equation}
    \mathrm{Sim}(1, 2) = 4; \quad \mathrm{Sim}(2, 3) = 4; \quad \mathrm{Sim}(3, 4) = 2;  \quad \text{etc.}
\end{equation}
i.e.,~we would rate items 1 and 2, or items 2 and 3, as being \emph{more similar} than items 3 and 4.

We should examine whether these relative scores seem reasonable. Items 1 and 2 are \emph{popular} items, which most users did \emph{not} purchase in common; whereas items 3 has \emph{half} of its users in common with item 4. If we were to build recommenders on this basis, we might recommend (for example) a popular album as being highly similar to a popular pair of jeans, simply on the basis that they have many users in common. In general, such a system would tend to identify popular items (such as items 1 and 2 in \Cref{eq:simToy}) as being similar. `Niche' items with fewer associated purchases (such as items 3 and 4 in in \Cref{eq:simToy}) would rarely be recommended.

In most cases, this is not the outcome we want; such a system would make generic recommendations of popular items, that likely would not seem specific to the context of a given query item. 

This toy example is intended to demonstrate that `similarity' is not something easy to define, and that different definitions have implicit assumptions with non-obvious consequences.
Presumably, we should improve our similarity function so that it has some appropriate normalization to account for item popularity, as we will see below.

\subsection{Jaccard Similarity}
\label{sec:jaccard}

Our first attempt at correcting the above issues is to normalize similarity scores in a way that considers the popularity of each item. The \emph{Jaccard Similarity}, or `intersection over union,' does so by computing
\begin{equation}
    \mathrm{Jaccard}(i,j) = \frac{| U_i \cap U_j |}{| U_i \cup U_j |}.
    \label{eq:jaccard}
\end{equation}
This similarity function is perhaps best visualized by a Venn diagram such as that in \Cref{fig:venn}. The Jaccard similarity takes a value between 0 (when $U_i$ and $U_j$ do not overlap at all, and thus have no intersection) and 1 (when the intersection is equal to the union, i.e.,~the items were consumed by \emph{exactly} the same set of users).

%\done{Make this look better. Looks fine}
\begin{SCfigure}
%\fbox{\includegraphics[width=0.3\textwidth]{fig/venn1.pdf}}
\begin{tikzpicture}[fill=gray]
% left hand
\scope
\clip (1,0) circle (1);
\fill (0,0) circle (1);
\endscope
% right hand
\scope
\clip (0,0) circle (1);
\fill (1,0) circle (1);
\endscope
% outline
\draw (0,0) circle (1) (0,1)  node [text=black,above] {$U_i$}
      (1,0) circle (1) (1,1)  node [text=black,above] {$U_j$};
\end{tikzpicture}\hspace{1mm}
\begin{tikzpicture}[fill=gray]
% left hand
\scope
% \clip (1,0) circle (1);
\fill (0,0) circle (1);
\endscope
% right hand
\scope
% \clip (0,0) circle (1);
\fill (1,0) circle (1);
\endscope
% outline
\draw (0,0) circle (1) (0,1)  node [text=black,above] {$U_i$}
      (1,0) circle (1) (1,1)  node [text=black,above] {$U_j$};
\end{tikzpicture}
\caption[Similarity as intersection over union.]{The similarity between two items $i$ and $j$ can be computed in terms of the intersection (left) and union (right) between the sets of users $U_i$ and $U_j$ who have consumed each. \label{fig:venn}}
\end{SCfigure}

To demonstrate the Jaccard similarity in action, let's consider computing the similarity among items in terms of past purchases from \emph{Amazon.com}. 
%\done{Make sure company names and stuff are italicized consistently. Fine, improve when writing index}
We'll consider Amazon's publicly-available dataset of around 900,000 reviews from the \emph{Musical Instrument} category.\footnote{Available from \url{https://s3.amazonaws.com/amazon-reviews-pds/tsv/index.txt}}

We first build some data structures to store the sets of items consumed by each user (or the sets of users who have consumed each item), i.e.,~$I_u$ and $U_i$:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
usersPerItem = defaultdict(set)
itemsPerUser = defaultdict(set)

for d in dataset:
    user, item = d['customer_id'], d['product_id']
    usersPerItem[item].add(user)
    itemsPerUser[user].add(item)
\end{lstlisting}
\end{minipage}

We can also implement the Jaccard similarity straightforwardly:
%\done{fix indentation level for all code}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
def Jaccard(s1, s2):
    numerator = len(s1.intersection(s2))
    denominator = len(s1.union(s2))
    return numerator / denominator
\end{lstlisting}
\end{minipage}

Now, \emph{recommendation} consists of finding the items with the highest Jaccard similarity compared to some given query (i.e.,~`people who bought X also bought Y'):
%\done{comment code throughout}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
def mostSimilar(i, K): # Query item i, and number of results K to return
    similarities = []
    users = usersPerItem[i] # Users who have purchased i
    for j in usersPerItem: # Compute similarity against each item
        if j == i: continue
        sim = Jaccard(users, usersPerItem[j])
        similarities.append((sim,j))
    similarities.sort(reverse=True) # Sort to find the most similar
    return similarities[:K]
\end{lstlisting}
\end{minipage}

Finally, let's examine some recommendations, e.g.~of the \emph{`AudioQuest LP record clean brush'} (product ID \emph{B0006VMBHI}). The 5 most similar items (i.e., \texttt{mostSimilar('B0006VMBHI', 5)}) are:
\begin{equation*}
\begin{tabular}{l}
\emph{Shure SFG-2 Stylus Tracking Force Gauge} \\
\emph{Shure M97xE High-Performance Magnetic Phono Cartridge} \\
\emph{ART Pro Audio DJPRE II Phono Turntable Preamplifier} \\
\emph{Signstek [...] Long-Playing LP Turntable Stylus Force Scale Gauge Tester} \\
\emph{Audio Technica AT120E/T Standard Mount Phono Cartridge}
% \emph{Technics: 45 Adaptor for Technics 1200 (SFWE010)} \\
% \emph{GruvGlide GRUVGLIDE DJ Package} \\
% \emph{STANTON MAGNETICS Record Cleaner Kit} \\
% \emph{Shure M97xE High-Performance Magnetic Phono Cartridge} \\
% \emph{Behringer PP400 Ultra Compact Phono Preamplifier}
\end{tabular}
\end{equation*}
All of the recommended items are also related to record players (which make up only a fraction of items in the category), which seem semantically reasonable given the query.

%And in 
Using
just a few lines of code, and using a (reasonably large) real-world dataset, we have implemented our first recommender system. Our solution is simple (and our implementation is fairly inefficient\footnote{In particular it is not necessary to iterate over \emph{all} items; rather one can quickly compute a candidate set of only those items that could potentially have a non-zero Jaccard coefficient; see Exercise~\protect\ref{ex:efficientJaccard}}), but nevertheless quickly produced reasonable recommendations. 
%Perhaps not surprisingly, 
These simple types of similarity-based recommendations drive many of the most high-profile recommender systems on the Web, as we study in \Cref{sec:amazonCaseStudy}.
%\done{hastily written, weak prose}

%\done{Above code is not efficient. Wrote exercise}

\subsection{Cosine Similarity}
\label{sec:cosine}

The Jaccard similarity captures our intuition about what items ought to be similar to each other, but is only defined if interactions are represented as \emph{sets}. We would like more nuanced similarity measures for data where feedback is associated with each interaction; for example we might not regard two users as `similar' if both had watched the \emph{Harry Potter} movies, in the event that one of them liked the series and the other disliked it.

The \emph{Cosine Similarity} achieves this by representing users' (or items') interaction histories in terms of \emph{vectors} rather than 
%in terms of 
\emph{sets}. An example is shown in \Cref{fig:cosine} in which we have three items ($i_1$, $i_2$, and $i_3$) and two users ($u_1$ and $u_2$) who have each interacted with two of them. 

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/cosine.pdf}
\caption[The Cosine Similarity between two vectors.]{The Cosine Similarity is defined in terms of the angle between two vectors, here describing users $u_1$ and $u_2$. \label{fig:cosine}}
\end{SCfigure}

In our previous (set) representation we would write $I_{u_1} = \lbrace i_2, i_3 \rbrace$ and $I_{u_2} = \lbrace i_1, i_3 \rbrace$ to describe the sets of items that $u_1$ and $u_2$ have interacted with. 
% In vector representation we simply represent the users as $u_1 = (0,1,1)$ and $u_2 = (1,0,1)$;
In vector representation we could simply describe $u_1$ and $u_2$ in terms of \emph{vectors} describing which items they interacted with.
%$u_1 = (0,1,1)$ and $u_2 = (1,0,1)$; 
%Note that 
%These 
Such
vectors are equivalent to rows of our interaction matrix $R$ (\cref{eq:interactionMatrix}), i.e.,~$R_{u_1} = (0,1,1)$ and $R_{u_2} = (1,0,1)$.
%\done{Sloppy notation. Using a vector to index a matrix. Fixed}

The Cosine Similarity (in this case between two users $u_1$ and $u_2$) is now defined in terms of the \emph{angle} between the vectors $u_1$ and $u_2$. Recall that the angle between two vectors $a$ and $b$ is defined as:
\begin{equation}
    \theta = \mathrm{Cos}^{-1} \left( \frac{a \cdot b}{|a|\cdot |b|} \right); \quad \text{or} \quad \mathrm{Cos}(\theta) = \frac{a \cdot b}{|a|\cdot |b|}.
\end{equation}
The angle $\theta$ measures the extent to which the two vectors point in the same direction; in the case of interaction data, the angle will range between \ang{0} (if the two users have interacted with exactly the same 
%set of 
items) and \ang{90} (if the interaction vectors are orthogonal, i.e.,~if the users have interacted with non-overlapping sets of items). The actual cosine similarity 
%\done{consistent capitalization. looks okay}
is the cosine of this angle, e.g.~between two users $u$ and $v$:\footnote{Again it can be straightforwardly defined for \emph{item} similarity by interchanging users and items.}
\begin{equation}
    \text{Cosine Similarity}(u,v) = \frac{I_u \cdot I_v}{|I_u| \cdot |I_v|}.
    \label{eq:cosineSim}
\end{equation}
For (binary) interaction data the cosine of the angle is now between 1 (when the angle is zero, and the interactions are identical) and 0 (when the interactions are orthogonal).

It is instructive to compare \Cref{eq:jaccard,eq:cosineSim}. 
In the case of binary interaction data, both expressions take values of 1 when the interactions are identical, and 0 when the interactions are non-overlapping.
The numerator $I_u \cdot I_v$ in \Cref{eq:cosineSim} is equivalent to $|I_u \cap I_v|$, as in \Cref{eq:jaccard}. The two differ only in their denominators, both of which are essentially forms of normalization based on the size of the sets $I_u$ and $I_v$ (and both demoninators will take the same value when $I_u$ and $I_v$ are equal).

Of course the cosine similarity is more interesting once we consider numerical interactions, i.e.,~interactions associated with \emph{feedback}, rather than just binary (0/1) data. For example, consider data where each interaction is associated with a `thumbs-up' or `thumbs-down' rating. We might represent this via an interaction matrix $R$ such that $R_{u,i} \in \lbrace -1, 0, 1 \rbrace$ (where $-1/1$ indicates thumbs-down/thumbs-up, and $0$ indicates an item the user hasn't interacted with).

In this case, the Jaccard similarity would not be well-defined. But the cosine similarity can still be computed on rows (or columns) of $R$. An example is shown in \Cref{fig:cosine2}. Here, user $u_1$ has interacted with two items ($i_2$ and $i_3$), and liked both; $u_2$ has interacted with the \emph{same} items, but disliked both. The two user vectors now point in opposite directions, i.e.,~they have an angle of \ang{180} and a cosine similarity of -1.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/cosine2.pdf}
\caption[Demonstration of the Cosine Similarity.]{The Cosine Similarity for two users who rated the same items, but with opposite sentiment polarity. \label{fig:cosine2}}
\end{SCfigure}

%Our code above for the Jaccard similarity can be straightforwardly modified 
With some effort we can adapt our code above for the Jaccard similarity
to implement the cosine similarity. Here we use an auxiliary data structure (\texttt{ratingDict}) that retrieves ratings for a given user/item pair:
%(\cref{exercise:cosine}).
%\done{fix or delete exercise. Deleted}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def Cosine(i1, i2):
    # Between two items
    inter = usersPerItem[i1].intersection(usersPerItem[i2])
    numer = sum([ratingDict[(u,i1)]*ratingDict[(u,i2)] for u in inter])
    norm1 = sum([ratingDict[(u,i1)]**2 for u in usersPerItem[i1]])
    norm2 = sum([ratingDict[(u,i2)]**2 for u in usersPerItem[i2]])
    denom = math.sqrt(norm1) * math.sqrt(norm2)
    if denom == 0: return 0 # If one of the two items has no ratings
    return numer / denom
\end{lstlisting}
\end{minipage}
%\done{test code}

Doing so (for the same query item), the top recommendation remains the same (Shure SFG-2 Stylus Tracking Force Gauge). Among the next few recommendations, there are many ties (i.e.,~identical cosine similarities); upon inspection these turn out to be items with only a single (overlapping) interaction. In this case such items are preferred by the cosine similarity (and not the Jaccard) since the denominator grows quickly for items with many associated interactions (whereas the union term in \Cref{eq:jaccard} grows more slowly, assuming many of the interactions overlap).
%\done{Results look a bit funny, check. Sounds okay}

\paragraph{Which similarity metric is `better'?}
%The above argument about the difference between the Jaccard and cosine similarity is somewhat imprecise. 
In the above example the Jaccard similarity seemed to work `better' than the cosine similarity, but our argument about the difference between the two is somewhat imprecise.
Note that this argument largely applies to this specific dataset (or even the specific query item we chose). Ultimately, these similarity measures are essentially \emph{heuristics}; whether one is `better' than another depends on our own assumptions and intuition about what similarity ought to mean. This is in contrast to the machine learning approaches we saw in previous chapters in which we had a specific objective (i.e.,~a measure of success) that we were trying to optimize. We'll revisit this question when we examine model-based recommender systems in \Cref{chap:modelbasedRecsys}.

\subsection{Pearson Similarity}
\label{sec:pearsonSim}

We motivated the Jaccard similarity by considering binary interaction data (i.e.,~sets), and the cosine similarity by considering polarized interactions like `thumbs-up's and `thumbs-down's. 
%But how would 

Consider how
these similarity measurements would operate on numerical feedback scores such as star ratings (as in \cref{eq:interactionMatrix}).
%
%Consider 
Take
the users represented in \Cref{fig:pearson} (left) as an example.
%\done{Most of these 3d plots would look better with shorter $i_1$ axis. They look fine}
Here two users have rated the same items ($i_2$ and $i_3$); user $u_1$ rated them 3 and 5 stars (respectively); user $u_2$ rated them 5 and 3.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/pearson1.pdf}%
\includegraphics[scale=0.75]{fig/pearson2.pdf}
\caption[Demonstration of the Pearson Similarity.]{Pearson Similarity. Two users have rating vectors that point roughly in the same direction (left); after subtracting the average from each, they point in opposite directions (right). \label{fig:pearson}}
\end{SCfigure}

According to the Jaccard similarity (based only on interactions), we would consider the two users to be identical (Jaccard similarity of 1); according to the cosine similarity, we would regard them as being very similar, since the angle between the two vectors is small. However one could argue that these users are polar opposites of each other: if we consider 5 stars to be a \emph{positive} rating and 3 stars to be a \emph{negative} rating, then these two users indeed have opposite opinion polarity.

Our definition of the cosine similarity does not account for this interpretation, essentially because it depends on the interactions already having explicitly positive or negative polarity. 
%We might 
To
correct this we might appropriately normalize
%appropriately normalizing 
our ratings: if we subtract the average for each user (4 stars for both $u_1$ and $u_2$), we find that their ratings are each 1 star above or below their personal average. After doing so, the example becomes very similar to the one from \Cref{sec:cosine} (see \cref{fig:pearson}, right).

%\done{could mention that we'll drop |y| from summations when obvious. No}

The above is essentially the idea captured by the Pearson Similarity. %\done{again, check capitalization. Looks mostly okay} 
The \emph{Pearson Correlation Coefficient} is a classical measurement for assessing the relationship between two variables, i.e.,~whether they trend in the same direction, regardless of scale and constant differences between them. The Pearson Correlation between two vectors $x$ and $y$ is defined as
\begin{equation}
    \text{Pearson Correlation}(x,y) = \frac{\sum_{i=1}^{|x|} (x_i - \bar{x})(y_i - \bar{y})}{\sqrt{\sum_{i=1}^{|x|}  (x_i - \bar{x})^2}\sqrt{\sum_{i=1}^{|y|} (y_i - \bar{y})^2}}.
    \label{eq:pearsonCC}
\end{equation}
Compare this definition to that of the cosine similarity in \Cref{eq:cosineSim}: the only difference is that from each measurement we subtract the mean ($\bar{x}$ or $\bar{y}$) of the corresponding vector. We summarize the relationship between the Jaccard, Cosine, and Pearson similarities in \Cref{box:sumSim}.


%\done{Bullets don't look great, maybe use a {description} environment or similar. Looks okay}
\begin{figure}
\caption{Summary of similarity measures. \label{box:sumSim}}
% \fbox
% {
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
Our comparison of the Jaccard, Cosine, and Pearson similarities can be summarized as follows:
%\done{not referenced}
\begin{itemize}
    \item The \emph{Jaccard Similarity} computes the similarity between \emph{sets}. The basic idea is to find items that have been purchased in common by many users (or users who have purchased many items in common); the set union is used to normalize the quantity, so that the measure does not overly favor popular items (or highly active users).
    %large sets.
    \item The \emph{Cosine Similarity} instead represents interactions as vectors (basically, rows or columns of our interaction matrix $R$). Similarity is then computed in terms of the angle between vectors for two items (or users). This definition allows similarity to be computed for numerical interaction data, especially if a polarity (i.e.,~positive or negative) is associated with each interaction.
    \item Finally, the \emph{Pearson Similarity} was motivated by the idea that numerical feedback may need to be properly calibrated in order to associate a polarity to each score. For instance, a rating of `3.5' might be positive for one user but negative for another. The Pearson similarity calibrates this polarity simply by subtracting the average for each item (or user); after this calibration the definition is similar to that of the cosine similarity.
\end{itemize}
\end{minipage}
}
\end{figure}


When applying this concept to rating data, we should be careful not to regard unobserved ratings (i.e.,~missing values of $R_{u,i}$ in \Cref{eq:interactionMatrix}) as zeros---doing so would distort our estimate of the user mean. Thus we might define the similarity between two users $u$ and $v$ (or similarly, items) only in terms of items they have 
both 
interacted with:
\begin{equation}
    \text{Pearson Similarity}(u,v) =
    \frac
    {
      \sum_{i \in I_u \cap I_v}
        (R_{u,i} - \bar{R}_u)
        (R_{v,i} - \bar{R}_v)
    }
    {
      \sqrt{\sum_{i \in I_u \cap I_v} (R_{u,i} - \bar{R}_u)^2}
      \sqrt{\sum_{i \in I_u \cap I_v} (R_{v,i} - \bar{R}_v)^2}
      %\sqrt{\sum_{i \in I_u} (R_{u,i} - \bar{R}_u)^2}
      %\sqrt{\sum_{i \in I_v} (R_{v,i} - \bar{R}_v)^2}
    }.
    \label{eq:pearsonSim}
\end{equation}
%\done{move \bar to be above $R$ only. All \bars}
%\done{Think about putting this argument back in. Yes it's fine}
%Note that 
Our choice to define the Pearson similarity by considering only \emph{shared} items is somewhat arbitrary; we could instead have considered \emph{all} items rated by each user in the denominator. Using our definition (which appears in e.g.~\cite{sarwar2001item}), we regard users as maximally similar if they have rated shared items in the same way; if we considered all items rated by each user in the denominator, we would regard users as less similar if they had also rated some different items. Remember that these similarity functions are merely heuristics---neither option should be considered more `correct,' but rather we should choose the definition that suits our intuition (or generates the most satisfactory results) in a particular situation.
%\done{long argument to make a not very important point! Think whether to include this. Fine}

%\done{included code examples and results}



Our code for the cosine similarity above can easily be adapted to implement the Pearson similarity (noting the details above). Here we have an additional data structure (\texttt{itemAverages}) recording the mean rating for each item:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def Pearson(i1, i2):
    # Between two items
    iBar1,iBar2 = itemAverages[i1], itemAverages[i2]
    inter = usersPerItem[i1].intersection(usersPerItem[i2])
    numer = 0
    denom1 = 0
    denom2 = 0
    for u in inter:
        numer += (ratingDict[(u,i1)] - iBar1)*(ratingDict[(u,i2)] - iBar2)
    for u in inter: # Alternately could sum over usersPerItem[i1]/[i2]
        denom1 += (ratingDict[(u,i1)] - iBar1)**2
        denom2 += (ratingDict[(u,i2)] - iBar2)**2
    denom = math.sqrt(denom1) * math.sqrt(denom2)
    if denom == 0: return 0
    return numer / denom
\end{lstlisting}
\end{minipage}

Fitting the Pearson similarity given the query item from \Cref{sec:jaccard} does not produce particularly satisfactory results; 
%the similar items come from a broad range of categories that do not seem to be closely related to the query item.
using $U_i \cap U_j$ in the denominator of \Cref{eq:pearsonSim} results in many items with a similarity of 1.0 (usually just due to a single overlapping interaction); using $U_i$ and $U_j$ separately in the denominator of \Cref{eq:pearsonSim} generates more meaningful results, though they come from a broad category of items that do not seem closely related.

Possibly these results are unsatisfactory simply because ratings of (e.g.)~a record cleaning brush are not due to factors that meaningfully transfer to other items. One likely purchases a record cleaning brush for its utility, rather than because of their personal preference toward such items. If variability in ratings is primarily due to build quality, or effectiveness (for example), then the Pearson similarity might identify other items with `similar' build quality or effectiveness, but those may not be semantically similar items. In this particular example, the Jaccard similarity---which defines similarity in terms of \emph{what} was purchased---seems more appropriate than the Pearson similarity, which defines similarity in terms of preferences.

Again though, possibly this measure is simply not suitable for this dataset or this query item. Let's try again on another dataset, this time from the Amazon \emph{Video Games} category. Given the query \emph{One Piece: Pirate Warriors}, the five most similar items in terms of Pearson similarity\footnote{Again using $U_i$ and $U_j$ separately in the denominator} are:
\begin{equation*}
\begin{tabular}{l}
\emph{Full Metal Alchemist: The Broken Angel}\\
\emph{Monster Rancher 4}\\
\emph{FINAL FANTASY X X-2 HD Remaster}\\
\emph{BlazBlue: Continuum Shift EXTEND Limited Edition}\\
\emph{Killzone 3}
(etc.)
\end{tabular}
\end{equation*}
These recommendations look more reasonable. In addition to being for similar platforms (e.g.~PlayStation) most are reasonably similar in terms of genre and style (e.g.~Japanese, based on anime, etc.). Seemingly, 
%for this type of 
in this setting
%time, 
features like style and genre better explain variation in ratings, making the Pearson similarity more effective.

Finally, these similarity measures needn't be used directly for recommendations as we have done here (i.e.,~simply retrieving the most similar item given a query). In practice they might be subroutines that guide more complex algorithms. For example, to recommend items to a user we might first find similar users, and recommend items that many of those users liked, rather than simply relying on item-to-item similarity directly (see e.g.~\Cref{sec:amazonCaseStudy}).
%\done{weak prose above. Fine}

%%\done{Put this under some section heading? removed}
%Moved above
%Note that the memory-based recommendation approaches above are essentially \emph{heuristics} for determining whether items (or users) are similar to each other; each makes different assumptions about what `similarity' might mean, each of which might be appropriate given different datasets or goals.

\subsection{Using Similarity Measurements for Rating Prediction}
\label{subsec:simPredictRating}

In \Cref{chap:modelbasedRecsys}, we'll contrast the similarity-based recommendation approaches above with machine learning (or `model-based') approaches which directly seek to predict ratings as accurately as possible.

However these two goals (measuring similarity versus predicting ratings) are not at odds with each other, and indeed one can use a measure of similarity as a means of predicting ratings.

The essence of such an approach is that the rating a user will give to an item can be estimated from ratings that user has given to \emph{similar} items (again, for some appropriate definition of `similarity'). One such definition (from \cite{sarwar2001item}) predicts the rating as a weighted sum of other items the user has rated:
\begin{equation}
    r(u,i) = \frac{\sum_{j \in I_u} R_{u,j} \cdot \mathrm{Sim}(i,j)}{\sum_{j \in I_u} \mathrm{Sim}(i,j)},
    \label{eq:simPredict}
    %\done{Check summations in all of these equations. Check that subscripts of R are right (had to fix some)}
\end{equation}
where $\mathrm{Sim}(i,j)$ could be any item-to-item similarity function such as those above. Note here that $r(u,i)$ is a prediction whereas $R_{u,i}$ is a historical rating.

The intuition behind the above equation is simply that the most similar items should be the most relevant when predicting future ratings, so the user's past ratings of those items are given the highest weights. Again though this is just a heuristic for predicting ratings and could be defined differently. For example we could write the same definition in terms of user similarity:
\begin{equation}
    r(u,i) = \frac{\sum_{v \in U_i} R_{v,i} \cdot \mathrm{Sim}(u,v)}{\sum_{v \in U_i} \mathrm{Sim}(u,v)},
    \label{eq:simPredict2}
\end{equation}
or, we, could possibly improve performance slightly by weighting deviations from the average rating, rather than ratings directly:
\begin{equation}
    r(u,i) = \bar{R}_i + \frac{\sum_{j \in I_u} (R_{u,j} - \bar{R}_j) \cdot \mathrm{Sim}(i,j)}{\sum_{j \in I_u} \mathrm{Sim}(i,j)},
    \label{eq:simPredict3}
\end{equation}

Using our video game data from \Cref{sec:pearsonSim}, and following the prediction function from \Cref{eq:simPredict3} (with the Jaccard Similarity as our similarity function), the Mean Squared Error of predicted ratings compared to the true labels is 1.786,
%1.9150345376505316 - simPredict
%
%1.7861436807136897 - simPredict3
compared to 1.838
%1.838224481118372
when always predicting the mean.

Code to implement the rating prediction model of \Cref{eq:simPredict3} is included below. Here we use the Jaccard similarity, though any item-to-item similarity metric could be used in its place.
Note that for the sake of evaluating such algorithms, we must be careful to exclude the query item ($i$) from all summations:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def predictRating(user,item):
    ratings = [] # Collect ratings over which to average
    sims = [] # and similarity scores
    for d in reviewsPerUser[user]:
        j = d['product_id']
        if j == item: continue # Skip the query item
        ratings.append(d['star_rating'] - itemAverages[j])
        sims.append(Jaccard(usersPerItem[item],usersPerItem[j]))
    if (sum(sims) > 0):
        weightedRatings = [(x*y) for x,y in zip(ratings,sims)]
        return itemAverages[item] + sum(weightedRatings) / sum(sims)
    else:
        # User hasn't rated any similar items
        return ratingMean
\end{lstlisting}
\end{minipage}

%\done{In various code blocks, have line numbers follow across blocks}
%\done{Consistent indentation for code}


\section{Random Walk Methods}
\label{sec:randomWalk}
%\done{style: random-walk or random walk. Random walk}


So far we have developed recommender systems 
in which user interaction data was represented as sets 
%, and those in which interaction data is represented as matrices 
or matrices.
%based on similarity metrics over sets of interactions, and others based on the principle of matrix decomposition.
Based on these two types of representations methods based on set 
%similarity 
(\cref{sec:jaccard}) and 
vector (\cref{sec:cosine}) similarity
%matrix decomposition (\cref{subsec:matrixFactor}) 
arose naturally.
%\done{adjust references if move section}

A third possible representation of user interaction data is to treat interactions as a \emph{bipartite graph} (\cref{fig:bipartite}). Here users and items are each a set of nodes, and edges between users and items represent user interactions (where edges may be weighted by a rating or interaction frequency).

\begin{SCfigure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label=left: 1, label={[label distance=2mm, rectangle]90:users:}] {};
\foreach \i in {2,3,...,6}
  \node[fsnode,on chain] (f\i) [label=left: \i] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label=right: 1, label={[label distance=2mm, rectangle]90:items:}] {};
\foreach \i in {2,3,...,6}
  \node[ssnode,on chain] (s\i) [label=right: \i] {};
\end{scope}

\draw (s1) edge[-] (f1);
\draw (s1) edge[-] (f5);
\draw (s3) edge[-] (f4);
\draw (s4) edge[-] (f3);
\draw (s2) edge[-] (f1);
\draw (s5) edge[-] (f3);
\draw (s1) edge[-] (f2);
\draw (s5) edge[-] (f6);
\draw (s4) edge[-] (f6);
\draw (s6) edge[-] (f5);

% \draw (s1) -- (f2);
% \draw (s2) -- (f5);
% \draw (s3) -- (f1);
% \draw (s4) -- (f4);
% \draw (s5) -- (f6);
% \draw (s6) -- (f3);
% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}

    \caption[User interactions as bipartite graphs.]{User interactions can be represented as a \emph{bipartite graph} connecting users and items. Recommendations can be made by simulating random walks on the graph, such that `nearby' items are considered similar.}
    \label{fig:bipartite}
    %\done{boring caption. It's fine}
\end{SCfigure}



Based on this representation, 
%we can now develop recommender systems based on general-purpose graph
%recommendation essentially amounts to a link-prediction problem, i.e.,~determine which user/item edges are 
%\done{style: user-item user/item or neither. User/Item}
%most closely related in the graph.
\emph{Random Walk}-based methods assess the relatedness or `closeness' of nodes by 
%determining how likely a walker starting on one node is likely to visit another.
simulating a walker that traverses the graph by randomly following its edges.
%
Specifically, random walk-based methods attempt to assess the strength of a relationship between two nodes $x$ and $y$ by assessing \emph{the probability that a random walk starting on node $x$ will terminate on node $y$}. 
%\done{finish}
%\done{redundant sentence. Okay}




\paragraph{Relation to PageRank}
Note that the above closely resembles algorithms like \emph{PageRank} or \emph{HITS} \citep{brin1998anatomy,kleinberg1999hubs}. These algorithms also model random walks on graph data; there the goal is to compute a stationary distribution $\pi$,
where $\pi_x$ represents the probability that a walker will visit node $x$ at any given step. This is computed by defining the relation
\begin{equation}
    \pi^{(t)} = \pi^{(t-1)} P
    \label{eq:powerIt}
\end{equation}
and computing $\pi = \lim_{t\rightarrow \infty} \pi^{(t)}$. The above can be computed by power iteration (i.e.,~iteratively computing the relation from \Cref{eq:powerIt}), which will converge to the principal eigenvector of $P$
%\done{check principle / principal}
\citep{brin1998anatomy}.
%which corresponds to 
\emph{PageRank} includes an addition detail,
a \emph{damping factor} $d$, which simulates the `click-through probability' that a random walker will terminate (and randomly restart) their walk at any step. Without this term, the stationary probability $\pi_i$ would become dominated by `sink' pages (i.e.,~with no outgoing links). See \cite{brin1998anatomy} for a full description.

When using the above approach 
%for 
in the context \emph{PageRank} or \emph{HITS}, 
%is to
we generally
treat $\pi_x$ as an overall measure of the `quality' or `authoritativeness' of a page. In a recommendation setting, we might instead be interested in transition probabilities, i.e.,~the probability that a random walker will visit item $j$ having started from item $i$. 
%This expression can be written as



To do so, we 
%must assign a 
start with a
\emph{transition probability} $p(j|i)$ between nodes; this represents the probability that a walker currently at node $i$ will visit node $j$ during the next step (we give a few examples of such transition probabilities below).
These probabilities can be aggregated into a transition matrix $P$ (where $P_{i,j} = p(j|i)$).
% \cite{li2009grocery}, page 1216
These \emph{first-order} probabilities represent the probability that a random walker at node $i$ would visit $j$ in the next step;
%would visit item $j$ having started from $i$ \emph{in the next step}.
%We are interested in computing the probability that a walker simulated for several steps will eventually visit $j$
to compute the probability that the walker will eventually visit $j$ on \emph{some} step, we can take powers of the transition matrix $P$:
\begin{equation}
    P^* = \sum_{n=1}^{\infty} \frac{(dP)^n}{|(dP)^n|}.
    \label{eq:walkSim}
\end{equation}
Here $d$ is again a damping factor. The damping factor prevents the probability from becoming saturated by the walker eventually transitioning to popular items \citep{yildirim2008random}



%Grocery shopping recommendations based on basket-sensitive random walk

%\done{work above paper in there}
Representative papers include
\cite{li2009grocery}, which most closely follows the setting (and notation) above. They
use this type of paradigm for the setting of grocery recommendations.
They start by defining transition probabilities between users and items (on a bipartite graph similar to that in \Cref{fig:bipartite}):
\begin{equation}
    p(i|u) = \frac{f(u,i)}{(\sum_{j}f(u,j))^{\alpha_1}}; \quad 
    p(u|i) = \frac{f(u,i)}{(\sum_{v}f(v,i))^{\alpha_2}}.
\end{equation}
Here $f(u,i)$ measures the historical purchase frequency between the user $u$ and the item $i$; $\alpha_1$ and $\alpha_2$ penalize users or products associated with many transactions.
The transition probability between items ($p(j|i)$) can then be found by summing over all users:
\begin{equation}
    p(j|i) = \sum_{u=1}^{|U|} p(j|u)p(u|i).
\end{equation}
\cite{li2009grocery} compare similarity functions like that of \Cref{eq:walkSim} with traditional item-to-item similarity functions like those in \Cref{sec:neighborhood}.

% Too elaborate
% \cite{liu2008eigenrank} adopt a similar approach, where (first-order) item-to-item similarities are defined as
% \begin{equation}
%     \Psi(i,j) = \frac{\sum{v \in U_i \cap U_j}\text{Sim}(u,v)(R_{v,i} - R_{v,j})}{\sum{v \in U_i \cap U_j}\text{Sim}(u,v)}; \quad p(j|i) = \frac{\Psi(j,i)}{\sum_{k \in I}\Psi(k,i)}
% \end{equation}

Others adopt similar approaches, albeit with different ways of defining transition probabilities between items; 
%in 
\cite{liu2008eigenrank} define transition probabilities between items 
%are defined 
in terms of the ratings those items have received. 

% Other techniques that don't just use a bipartite graph:

Note that the idea behind random walk based methods is quite general; above we describe only the simplest setting where (bipartite) graphs are defined in terms of user and item similarities. 
%More complex 
Other
approaches based on this paradigm establish more complex graph structures to uncover different types of relationships among users, items, or other features. For example \emph{authors} from \emph{organizations} may publish \emph{papers} in \emph{venues} (essentially a `four-partite' graph) \citep{dong2017metapath2vec}; or richer item relationships could be defined in terms of which items were co-purchased in the same basket (in addition to user-to-item relationships) \citep{wan2018representing}. 
Ultimately graph-based representations give us a straightforward means to incorporate several types of relationships via a common framework. We study a specific instance that models interactions within sessions in \Cref{subsec:shortTerm}.

% Research Paper Recommender Systems: A Random Walk Based Approach

% TrustWalker: A Random Walk Model for Combining
% Trust-based and Item-based Recommendation



% metaPath2Vec

%\done{move section?}

%\done{finish}







\section{Case Study: Amazon.com Recommendations}
\label{sec:amazonCaseStudy}

In a 2003 paper \citep{linden2003amazon}, researchers described the techniques underlying \emph{Amazon}'s recommendation technology. The paper described systems
%similar to that in \Cref{fig:alsoviewed}, 
that recommend related items,
e.g.~`Customers who bought items in your shopping card also bought.'

The first recommendation method the paper describes is based on the Cosine Similarity (\cref{sec:cosine}). Interestingly, cosine similarity is defined between \emph{users}, rather than between items as we did in \Cref{sec:jaccard}; the goal is then to recommend items that have previously been purchased by similar customers. The paper discusses the issues of scaling this type of similarity computation to the large number of \emph{Amazon} users, and discusses an alternative strategy to \emph{cluster} users based on a user-to-user similarity metric; similar customers to a given user can then be found by determining the user's cluster membership (which is cast as a classification problem).

Although \cite{linden2003amazon} go into little detail about the specifics of what is implemented by Amazon, their work does stress the key point that real-world, large-scale recommenders need not be based on complex models. Rather, primary considerations include building models that are simple but scale well.

%Cite the old amazon recommendation paper, and `Two Decades of Recommender Systems at Amazon.com'

Following \cite{linden2003amazon}, a follow-up paper was published describing more modern recommendation techniques on Amazon \citep{smith2017two}.
%\done{italics in this section. Looks okay}
The paper starts by describing minor modifications to the algorithms described above. For instance, they describe an item-to-item based approach which is more reminiscent of that in \Cref{sec:jaccard}, and describe how most of the computation for this type of problem can be done offline. \cite{smith2017two} also stresses the importance of choosing a good similarity heuristic, and discuss some strategies for doing so. Finally, \cite{smith2017two} discuss the importance of considering temporal factors when designing recommenders, which is our main focus in \Cref{chap:temporalSequential}.


%\section{Model-based Approaches to Recommendation}
%\label{sec:modelbased}







% \section{Exercises}

\begin{exercises}

% \item Modify the Pearson similarity implementation from \Cref{sec_?} to implement the cosine similarity. Compare the results of the two implementations, as well as the Jaccard similarity, on?
%\done{hastily written. deleted}


%\done{Which dataset?}


\item 
\emph{These exercises could be completed using any dataset with users, items, and ratings} (including the same dataset used in Project~\ref{proj:1111}, below).
The Jaccard similarity-based recommender 
we implemented 
in 
\Cref{sec:jaccard} proved an effective recommender, though our implementation was inefficient. The main source of inefficiency was due to iteration over \emph{all} items. A more efficient implementation might first build smaller a \emph{candidate set} of items, by noting that only those items with at least one user in common with the query could potentially have non-zero Jaccard similarity. This candidate set can be built by taking all users who have purchased the query $i$, and taking the union over \emph{other} items they have purchased (other than $i$), i.e.,~$\bigcup_{u \in U_i} I_u \setminus \lbrace i \rbrace$. After modifying our implementation from \Cref{sec:jaccard} to use this candidate set, confirm that it produces identical recommendations, and compare its running time to the na\"ive implementation.
\label{ex:efficientJaccard}

\item Although we discuss evaluation in detail in \Cref{chap:modelbasedRecsys}, in this exercise we'll build a simple quality measure for similarity-based recommenders. Specifically, an item-to-item recommender might be considered `useful' if it tends to rate items $i$ and $j$ that are both purchased by $u$ as being \emph{more similar} than two items \emph{not} purchased by the same user. For each user, randomly sample two of their interactions $i$ and $j$, and a third interaction $k \in I \setminus I_u$ \emph{not} purchased by $u$. Measure how often the system rates $\text{Sim}(i,j) \geq \text{Sim}(i,k)$. Compute this measure for the Jaccard, Cosine, and Pearson similarities (or other variants) to measure which is best suited to a particular dataset.\footnote{Also consider the most effective way to handle ties; ties could either be counted as a failure of the algorithm, or could be counted separately from successes or failures.}
%\done{reread for clarity}
\label{ex:compareJCP}

\item 
\label{ex:itemRecommend2}
%Compare the Jaccard, Cosine, and Pearson similarity metrics (\cref{sec:}) by using them to build a simple recommender. 
The code we developed in \Cref{sec:jaccard} (and in Exercise~\ref{ex:efficientJaccard}) is so far just an item-to-item recommender, and does not produce recommendations based on the user's \emph{history}. However it could be adapted to do so in several ways, e.g.:
\begin{itemize}
    \item Recommend an item $i$ based on the \emph{average} similarity compared to all items $j$ from the user's history;
    \item Rather than averaging over \emph{all} items from the user's history, average over only the last $K$, or otherwise weight the average by recency;
    \item Find the most similar users $v$, and recommend their most recent interaction;
    \item etc.
\end{itemize}
Explore alternatives such as those above to determine which is best at predicting users' future interactions based on their history.
Methods may be compared using a similar approach to Exercise~\ref{ex:compareJCP}:
for each user, 
%consider their history $I_u$, excluding a single interaction
withhold a single interaction 
$i \in I_u$ from their history;\footnote{The withheld interaction could be chosen randomly, or for the sake of forecasting future interactions, you could withhold the most recent.} determine which method is best at predicting (i.e.,~assigning a higher score or ranking to) $i$ based on the history $I_u \setminus \lbrace i \rbrace$, compared to a randomly chosen non-interaction $k \in I \setminus I_u$.
%, i.e.,~by assessing which is best at assigning higher scores to a user's next consumption compared to a random 
%Compare a few similarity metrics

%\done{could expand to project. No}



\item Implement rating prediction models 
%based on 
following
the formulas in \Cref{eq:simPredict,eq:simPredict2,eq:simPredict3}. Compare the three in terms of their Mean Squared Error (using either the entire dataset or a random sample).
\label{ex:ratingFromSim}
%\done{Which dataset?}

% -- Show that the Pearson correlation is related to the $R^2$ coefficient. (nOT INTERESTING)

%-- Some comparison of the various recommender systems error metrics, possibly following the rating prediction experiment above.
%\Done{added}

\end{exercises}

\refstepcounter{projectCounter}
\label{proj:1111}
\section*{Project \theprojectCounter: A Recommender System for Books (Part 1)}
\addcontentsline{toc}{section}{Project \theprojectCounter: A Recommender System for Books (Part 1)}
%\done{project, fill in *both* titles above}


In this project we'll build recommender systems to make recommendations related to book reviews from \emph{Goodreads} (which we studied a little in \Cref{chap:basicsReg}). Here, we'll build simple similarity-based recommenders, before continuing this project with more complex recommendation approaches in \Cref{chap:modelbasedRecsys} (Project~\ref{proj:2}). We'll also use this project to set up an evaluation pipeline for this type of task (though we discuss evaluation strategies for recommender systems in more detail in \Cref{chap:modelbasedRecsys}).

While this project could be completed using any dataset 
%(including users, items, and ratings), 
that includes users, items, and ratings, 
we suggest using a small subset (e.g.~reviews from the \emph{Poetry} or \emph{Comic Book} categories) for the sake of quickly benchmarking alternative methods.

\begin{enumerate}
\item First, implement simple item-to-item recommendation strategies, as in Exercise~\ref{ex:itemRecommend2} (you may follow a similar evaluation strategy as in that exercise).
\item 
Although the approaches explored in this chapter were largely \emph{not} based on machine learning, there is no reason why we couldn't 
%Second, 
%let's consider using the same data for an implicit 
%\done{haven't defined implicit feedback yet, doesn't make sense}
%feedback task, namely to predict whether a user will 
train a \emph{classifier} to predict (or rank) books that a user is likely to read.
%\emph{read} 
%read
%a book or not. %Before solving this using complex approaches specifically designed for implicit feedback, 
%it's worth considering how much progress can be made using simpler approaches based on \emph{classification}. 
We'll explore more complex methods for this setting in \Cref{chap:modelbasedRecsys}, but for the moment let's see how far we can get by trying to extract some simple features to describe user/item interactions.
Start by building a training set consisting of all pairs $(u,i)$ of books $i$ that user $u$ has read; next build an (equally-sized) set of \emph{negative} pairs $(u,j)$ of books the user \emph{hasn't} read (e.g.~by sampling randomly). Now, we want to build a feature vector $\phi(u,i)$ that can be used to predict whether a user $u$ has read book $i$ or not. 
%As we discussed in \Cref{sec_?}, 
To build a useful recommender system, we must include features that describe \emph{interactions} between the user and the book. Examples of features you might use could include:
    \begin{itemize}
        \item The popularity of the book (e.g.~the number of times item $i$ appears in the training set, or its average rating);
        % \item The predicted rating score for the pair $(u,i)$ that you found 
        % using your latent factor model.
        %\done{removed, haven't defined latent factors yet}
        %in Part~\ref{project2part2};
        %\done{referencing the part is ugly. Gone}
        \item The Jaccard similarity (or any other similarity measure) between $i$ and the \emph{most similar} book the user $u$ has read (i.e.,~$\max_{j \in I_u} \mathrm{Sim}(i,j)$);
        \item Likewise, the similarity between the \emph{user} $u$ and the most similar user who has read $i$;
        \item Any other similarity measures, user, or item features.
    \end{itemize}
    %\done{could move the above exercise to previous chapter}
Your classifier can be evaluated using standard accuracy or ranking metrics (though we further discuss evaluation in this setting in \Cref{sec:evaluateRec}). Compare this classification-based approach to methods like those from Exercise~\ref{ex:itemRecommend2}.\footnote{If implemented properly, the classification approach should perform better since other similarity measures are used as \emph{features} by the classifier; in essence our classifier is implementing a simple form of \emph{ensembling}.}
\item Finally, consider using the data to predict ratings, as in Exercise~\ref{ex:ratingFromSim}. We'll compare these predictions to 
%more complex 
model-based approaches in Project~\ref{proj:2}.
%Start by implementing simple neighborhood-based approaches as in \Cref{chap:recsysIntro},
\end{enumerate}



\chapter{Model-based Approaches to Recommendation}
\label{chap:modelbasedRecsys}

So far when developing recommender systems in \Cref{chap:recsysIntro} we have avoided any discussion of \emph{machine learning}. Although the types of `memory-based' recommender systems (see \cref{box:modelBased}) we've developed so far can be used to make predictions (either by estimating the next item or predicting a rating as in \Cref{subsec:simPredictRating}), they were in some sense not \emph{optimized} to do so. That is, we used \emph{heuristics} to rank items and predict ratings. This is in contrast to the approaches we developed in \Cref{chap:basicsReg,chap:classPipe} where we were concerned with \emph{objectives} to be optimized (involving an accuracy or error term), in terms of several model parameters.

%\done{check all crefrange, replace adjacent chapters by cref}

In this chapter, we develop \emph{model-based} approaches to recommendation, which adapt the regression and classification approaches from \Cref{chap:basicsReg,chap:classPipe} to problems of estimating interactions between users and items.
%
That is, we are concerned with fitting models that take a user $u$ and item $i$ as input in order to estimate an interaction label $y$ (such as a purchase, click, or rating):
\begin{equation}
    f(u,i) \rightarrow y.
    \label{eq:fui}
\end{equation}
%\done{move to next chapter}
%\done{notation, looks fine}
Superficially, solving such a prediction task seems no different than the regression or classification scenarios 
we've already developed:
%we explored in %\Cref{chap:basicsReg,chap:classPipe}: 
na\"ively we might imagine collecting some appropriate user or item features and applying the techniques we've already developed.
%\done{wordy, whole paragraph}
However as we study below,
%(\cref{sec:setup}),
%
%several differences between this 
certain characteristics of this scenario render traditional regression and classification approaches ineffective, and demand that we explore 
%design
new approaches specifically designed to capture the dynamics of interaction data. Specifically:
%\done{combine with above paragraph}
\begin{itemize}
    \item 
    Most of the techniques we'll develop in this chapter discard features altogether, and make predictions purely on the basis of historical interactions. This owes partly to the difficulty of collecting useful features, and also to the complex semantics that underlie people's preferences and behavior.
    \item
    As such, rather than having parameters associated with features as we did in \Cref{chap:basicsReg,chap:classPipe}, the models we'll develop here have parameters associated with individual \emph{users} (or items). This shall be our introduction to the idea of \emph{model-based} personalization, as we discussed in \Cref{sec:contextVsPerson}.
    \item
    To model users (and items) we'll introduce the concept of \emph{latent spaces} in \Cref{subsec:matrixFactor}, %Essentially we'll develop models 
    %The models we develop are based on the principle of dimensionality reduction, 
    whereby we automatically discover hidden dimensions that explain the variation in people's opinions---without necessarily knowing exactly what the dimensions correspond to.
    %\done{not great}
\end{itemize}


% Our study of model-based recommendation systems
% Relation to dimensionality reduction
%\done{removed}


Our discussion of recommender systems will form the basis of many of the models we develop throughout the remainder of this book. Although in this chapter we'll build predictive models based purely on interaction histories, later we'll show how similar models can be extended by incorporating features (\cref{chap:contentStructure}) and temporal information (\cref{chap:temporalSequential}). Later, as we further develop personalized models of text (\cref{chap:text}) and images (\cref{chap:visual}) this same notion of modeling users via latent spaces will appear repeatedly.






In contrast to `memory-based' recommendation approaches, \emph{model-based} approaches seek to learn parameterized representations of users and items, so that recommendations can be made in terms of the learned parameters. Model-based approaches are typically cast in terms of supervised learning, so that the goal is to predict ratings, purchases, clicks (etc.)~as accurately as possible. We summarize the differences between these two classes of approach in \Cref{fig:memoryVsModel}.

\begin{figure} 
\caption{Memory-based versus model-based approaches.
\label{fig:memoryVsModel} 
%\done{Reference figure}
}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
There are a variety of reasons why one might choose a model-based or memory-based approach. We summarize a few of the advantages and disadvantages as follows:
\begin{description}
  \item[Training and Inference Complexity] Model-based approaches often require (expensive) offline training; on the other hand, once trained, recommendations can potentially be retrieved quickly, e.g.~by retrieving a nearest neighbor or a maximum inner product in parameter space (\cref{sec:retrieval}). In contrast, memory-based approaches, while requiring no training, may depend on computationally-intensive heuristics.
  \item[Interpretability] Often, simple recommendations 
  %such as those in \Cref{fig:alsoviewed} 
  may be preferable simply because they are easy to explain to a user. In contrast, machine learning-based recommendations may make users uncomfortable due to their `black box' nature (we explore notions of explainability and interpretability a little further in \Cref{subsec:textExplain}).
  %(see \cref{sec:interpretability}).
  \item[Accuracy] Model-based systems are appealing because they directly optimize a desired error measure. On the other hand, error measures that are tractable may not be those that relate meaningfully to user satisfaction, and may distract from qualitative improvements.
\end{description}
\end{minipage}}
\end{figure}

%SLIM
%FISM
%Some sort of deep learning based model


\subsubsection{The Netflix Prize}
%\done{Put this subsection somewhere, or make it a case study. Is fine here}

In 2006, \emph{Netflix} released a dataset of 100,000,000 movie ratings (across 17,770 users and around 480,000 users). Their dataset took exactly the form described in \Cref{sec:setup}, i.e.,~it consisted purely of (user, item, rating, timestamp) tuples. Associated with the dataset was a competition \citep{bennett2007netflix} to reduce the RMSE (on a test set of withheld ratings) by 10\% compared to Netflix's existing solution. The first team to do so would win a \$1,000,000 prize. 

The history of the competition is also interesting. Early leaders joined forces to develop ensemble approaches, and the winning teams were nearly tied in a nail-biting finish. The competition also led to a broader discussion around the value of such high-profile competitions, as well as the question of whether narrowly reducing a Mean Squared Error actually improves recommendations. It also led to a lawsuit against Netflix following de-anonymization of the competition data \citep{narayanan2006break}.\footnote{This too is an interesting story, as the competition data, with anonymized user and item IDs, at first glance appears to be sufficiently anonymized.} Finally there is the question of whether the complex models that achieved the best competition performance---which have many complex, interacting, and carefully-tuned components, and are expensive to train---can really be \emph{deployed}.
%\done{Could discuss further. Also include e.g. whether monolithic models can really be implemented. No}

%\done{mention FAISS in retrieval section}

%But other than the 
%history 
Other than the specifics of the prize itself,
%Regardless of the specifics of 
%surrounding 
%the Netflix Prize, 
the dataset and high-profile competition spawned a great deal of research on recommender systems in general, 
%and in particular the specific problem of 
especially the specific setting of
rating prediction.
%\done{rest of this sentence}
%and in particular, 
In particular, winning approaches were
%and 
model-based
solutions based on matrix factorization, as we begin to develop below.

% \begin{figure}
% \caption{Lessons from the Netflix Prize}
% %\done{this. Drop}
% Add text on ``Lessons from the Netflix Prize Challenge'', maybe in box.
% \end{figure}



%\done{need to upgrade to section, adjust levels nearby in this chapter}
\section{Matrix Factorization}
\label{subsec:matrixFactor}

The basic assumption made by model-based recommenders is that there is some underlying low-dimensional \emph{structure} among the interactions we are trying to predict. Put differently, model-based recommender systems are essentially a form of dimensionality reduction.
%\done{This and below could be in the start of the section, or the subsection heading could be below}

In simple terms, we assume that users' opinions, or the properties of the items they consume, can be efficiently summarized. Do you tend to like action movies (and is this an action movie)? Do you tend to enjoy movies with a high budget, certain actors, or a long runtime? To the extent that purchases, clicks, or ratings can be explained by such factors, the goal of model-based recommendation is to discover them.

Considering data of the form in \Cref{sec:setup}, this seems a difficult process: we have no knowledge of the necessary \emph{features} that would be needed to discover these important factors (i.e.,~we don't know which movies are action movies, which movies have a long runtime, etc.). But surprisingly one can still uncover these underlying dimensions without them. 
As a motivating example,
consider the interaction (e.g.~click) data depicted below:
\begin{equation}
    R = 
    \begin{tabular}{cc}
    $\left[ \begin{tabular}{ccccc}
    1 & $\cdot$ & 1 & $\cdot$ & $\cdot$\\
    1 & 1 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 1 & 1 & $\cdot$ & $\cdot$\\
    $\cdot$ & $\cdot$ & $\cdot$ & 1 & 1\\
    $\cdot$ & $\cdot$ & $\cdot$ & 1 & $\cdot$\\
    \end{tabular} \right]$
    &
    \hspace{-6mm}
        \begin{tabular}{r}
        $u_1$\\
        $u_2$\\
        $u_3$\\
        $u_4$\\
        $u_5$
        \end{tabular}\\
        
        \begin{tabular}{ccccc}
        \hphantom{1}$i_1$ &
        \clap{$i_2$}\hphantom{1} &
        \clap{$i_3$}\hphantom{1} &
        \clap{$i_4$}\hphantom{1} &
        \clap{$i_5$}\hphantom{1}
        \end{tabular}
    \end{tabular}
        \label{eq:interactionMatrix2}
\end{equation}
The matrix appears to decompose roughly into two `blocks': if we wrote
\begin{equation}
    \begin{tabular}{l}
      $\gamma_{i_1} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_2} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_3} = \lbrack 1,0 \rbrack$\\
      $\gamma_{i_4} = \lbrack 0,1 \rbrack$\\
      $\gamma_{i_5} = \lbrack 0,1 \rbrack$
    \end{tabular}; \quad
    \begin{tabular}{l}
      $\gamma_{u_1} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_2} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_3} = \lbrack 1,0 \rbrack$\\
      $\gamma_{u_4} = \lbrack 0,1 \rbrack$\\
      $\gamma_{u_5} = \lbrack 0,1 \rbrack$
    \end{tabular},
    \label{eq:interactionFactors}
\end{equation}
then we could (approximately) summarize the matrix $R$ in \Cref{eq:interactionMatrix2} by writing
%\begin{equation}
    $R_{u,i} = \gamma_u \cdot \gamma_i$.
%    \label{eq:matfac1}
%\end{equation}
% \done{Introduce the idea of a `user manifold' somewhere. Done elsewhere}
The two blocks in $R$ might conceivably correspond to some feature in the data, e.g.~male and female users who buy men's and women's clothing. If so, the values in \Cref{eq:interactionFactors} would correspond to genders for users and items. %though 
Note critically though that we discovered these factors 
%we discovered these factors automatically 
simply because they \emph{summarized the structure of the matrix}, rather than needing to rely on observed features.
%\done{Kind of hate the above example. Keep for now}
%\done{Could drop above example}
%\candrop{this example}

\emph{Matrix Factorization} follows this same idea, again by looking for underlying structure that explains observed interactions.

Essentially, our goal is to describe a matrix in terms of lower-dimensional factors, i.e.,
\begin{equation}
    \underbrace{\left[ \begin{tabular}{ccc}
    \ & \ & \ \\
    \ & R & \ \\
    \ & \ & \ 
    \end{tabular} \right]}_{|U| \times |I|} = 
    \underbrace{\left[ \begin{tabular}{c}
    \ \ \ \\
    $\gamma_U$\\
    \  
    \end{tabular}
    \right]}_{|U| \times K} \times
    \underbrace{\left[
    \begin{tabular}{ccc}
    \parbox{1mm}{\vspace{5mm}} & $\gamma_I^T$ & \parbox{1mm}{\vspace{5mm}}\\
    \end{tabular} \right]}_{K \times |I|}.
\end{equation}
%\done{Kind of hate the notation above. Could just use U and I for number of users and items?}
%\done{Probably should use |U| and |I| for the number of users and items}
That is, we are assuming that the matrix $R$, of dimension $|U| \times |I|$ (the number of users times the number of items), can be approximated by a `tall' matrix $\gamma_U$ and a `wide' matrix $\gamma_I$. 
%$\gamma_U$ and $\gamma_I$ essentially summarize the users and items (respectively)
Now, a single entry $R_{u,i}$ can be estimated by taking the corresponding row of $\gamma_U$ and column of $\gamma_I$:
\begin{equation}
    R_{u,i} = \gamma_u \cdot \gamma_i,
    \label{eq:matfac2nobias}
\end{equation}
as in our motivating example above.
%\Cref{eq:matfac1}. 
%\done{Kinda ugly that this is *exactly* the same as the previous equation. Removed above}
$\gamma_u \in \mathbb R^K$ is now a latent vector that describes a user, and $\gamma_i \in \mathbb R^K$ describes an item.

Examples of such vectors are depicted in \Cref{fig:matfac1}. Intuitively, $\gamma_u$ might be thought of as describing the `preferences' of the user $u$, whereas $\gamma_i$ describes the `properties' of item $i$. Then, the user $u$ will like (e.g.~give a high rating to or interact with) the item $i$ if their preferences are \emph{compatible} with the item's properties (i.e.,~they have a high inner product). The latent dimensions, $\gamma_{\cdot,1}$, $\gamma_{\cdot,2}$ (etc.)~now describe those latent factors that best explain variability in $R$. For example, if such a model were trained on the Netflix dataset they might measure the extent to which a movie is a comedy or a romance, or the quality of its special effects. Again though, these factors are \emph{latent} and are discovered purely so as to maximally explain the observed interactions; models based on the principle of matrix factorization are commonly referred to as \emph{latent factor models}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/matfac1.pdf}
\caption[Representation of a user and item in a latent factor model.]{Representation of a user $u$ and item $i$ in a latent factor model. \label{fig:matfac1}}
\end{SCfigure}


%\done{Mention SVD and cite Koren paper}

\subsubsection{Relationship to the Singular Value Decomposition}

Briefly, we note that
the factorization described in \Cref{eq:matfac2nobias} is closely related to the \emph{Singular Value Decomposition} (SVD). Under the Singular Value Decomposition, a matrix $M$ is decomposed as
\begin{equation}
    M = U\Sigma V^T,
\end{equation}
where $U$ and $V$ are left and right singular values of $M$ (eigenvectors of $MM^T$ and $M^TM$), and $\Sigma$ is a diagonal matrix of eigenvectors of $MM^T$ (or $M^TM$). Critically, \emph{the best possible rank $K$ approximation of $M$ (in terms of the MSE) is found by taking the top $K$ eigenvectors/eigenvalues in $U$, $\Sigma$ and $V$} (Eckart-Young theorem).
%Eckart–Young theorem
While this appears to give us a recipe for choosing the best possible $\gamma_u$ and $\gamma_i$ in \Cref{eq:matfac2nobias}, 
%the SVD is not defined for partially-observed matrices (like $R$).
%However, 
the Singular Value Decomposition is defined only for \emph{fully observed} matrices, rather than partially observed interactions as in $R$.
%even if $R$ were fully observed, it would be impractical to compute the SVD for a matrix of size $|U|\times |I|$ for any reasonably large dataset.
Even if this could be addressed (e.g.~via a data imputation strategy on the missing values), it would not be practical to compute the SVD on a matrix that could potentially have millions of rows and columns.
As such, in practice we won't compute eigenvectors and eigenvalues, and will instead resort to gradient-based approaches, as described below. Nevertheless, the relationship to the SVD gives us a hint as to the type of factors $\gamma_u$ and $\gamma_i$ are likely to correspond to. 
%See \cite{koren2008factorization}


\subsection{Fitting the Latent Factor Model}
\label{subsec:fitLFM}

So far we have described the intuition behind modeling interactions in terms of latent user and item factors, but have not yet described how to fit a model based on this principle. That is, we would like to choose $\gamma_U$ and $\gamma_I$ so as to fit the interaction data most closely, e.g.~by minimizing some loss such as the Mean Squared Error, following the setting of the Netflix Prize above:
%\done{Don't think I mentioned that the Netflix Prize used the MSE yet? Maybe not necessary. I did}
\begin{equation}
    \argmin_{\gamma} \frac{1}{|R|} \sum_{(u,i) \in R} (f(u,i) - R_{u,i})^2,
    \label{eq:lfModelBasic}
\end{equation}
where $f(u,i)$ is our prediction function $f(u,i) = \gamma_u \cdot \gamma_i$.

% Note that this would be straightforward if the matrix $R$ were fully observed (i.e.,~every user rated every item). In this case, we could rely on the Singular Value Decomposition (SVD). Briefly, this decomposition allows us to factorize a real $M\times N$ matrix according to
% \begin{equation}
    % R = U\Sigma V^T,
% \end{equation}
% where $U$, $V$, and $\Sigma$ eigenvectors and eigenvalues of $R^TR$ and $RR^T$. Critically, if we keep just the $K$ eigenvectors corresponding to the largest eigenvalues, this corresponds to the \emph{best possible rank $K$ factorization in terms of the MSE} \citep{golub1971singular}.

% However, the Singular Value Decomposition is defined only for \emph{fully observed} matrices, rather than partially observed interactions as in $R$. Even if this could be addressed (e.g.~via a data imputation strategy on the missing values), it would not be practical to compute the SVD on a matrix that could potentially have millions of rows and columns.

%Instead, 
As mentioned when discussing the SVD above,
we seek a solution based on gradient descent. When minimizing the Mean Squared Error, the solution is similar to the one we saw in \Cref{sec:gradient}. Note that as usual we should be careful to split interactions $(u,i) \in R$ into training, validation, and test sets, and to include a regularizer to avoid overfitting, as we describe below.

\subsubsection{User and item biases}

Before describing the gradient-descent-based solution to fitting models like those in \Cref{eq:lfModelBasic}, 
%in detail, 
we first suggest some steps to augment the model that will improve 
%the solution.
prediction accuracy.

Although a simple solution of the form $r(u,i) = \gamma_u \cdot \gamma_i$ seems to capture the types of interactions we want, it is difficult to regularize. Consider adding a simple $\ell_2$ regularizer such as
\begin{equation}
    \Omega(\gamma) = \lambda \left( \sum_{u = 1}^{|U|} \sum_{k = 1}^K \gamma_{u,k}^2 +
                                    \sum_{i = 1}^{|I|} \sum_{k = 1}^K \gamma_{i,k}^2
                            \right)
    \label{eq:latentreg1}
\end{equation}
%\done{Never use the notation Omega elsewhere. Maybe use other places or drop here.
%\done{Write |I| / |U| here but is N_i / N_u above. Like |I| better but I forget why I did N_i in the first place}
This regularizer (for large $\lambda$) will encourage the parameters to be close to zero; as such the predictions $\gamma_u \cdot \gamma_i$ will also be pushed toward zero, and the system will systematically underpredict ratings.

There are several ways this could be avoided. Trivially, we might simply subtract the mean ($\bar{R}$) from all ratings before training so that they are centered around zero. Alternately, recall as in \Cref{sec:regularization}  that we were careful to exclude the intercept term $\theta_0$ from our regularizer.
%\done{That section doesn't exist yet}
Although our current model lacks such an intercept term, we can straightforwardly add one:
\begin{equation}
    r(u,i) = \alpha + \gamma_u \cdot \gamma_i.
\end{equation}
Note that we would still regularize as in \Cref{eq:latentreg1}.

Although the offset term $\alpha$ corrects the problem of systematically underpredicting ratings, it retains a similar issue at the level of individual users or items. Again, the regularizer pushes $\gamma_u$ and $\gamma_i$ toward zero, and therefore pushes predictions toward $\alpha$. But individual users or items may tend to systematically give much higher (or lower) ratings than $\alpha$, meaning that our regularizer again encourages us to systematically under (or over) predict.

Again we correct for this by adding additional bias terms, this time at the level of individual users or items:
\begin{equation}
    r(u,i) = \alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i.
    \label{eq:MFscore}
\end{equation}
$\beta_u$ now encodes the extent to which user $u$'s ratings trend higher or lower than $\alpha$, and $\beta_i$ encodes the extent to which item $i$ tends to receive higher or lower ratings than $\alpha$.\footnote{Note that we are careful not to refer to $\alpha$ as an `average' rating, and indeed in general once we fit the model $\alpha \not= \bar{R}$, just as the offset $\theta_0$ is not the average $\bar{y}$ in a linear regression model.}

Adding these bias terms introduces an additional $|U| + |I|$ parameters to the model. Whether these terms should be included in the regularizer $\Omega$ is arguable: on the one hand they are similar to offset terms, which we would normally not regularize (for the same reason we do not regularize $\alpha$); on the other hand, failing to regularize them may lead to overfitting. In practice, the terms may simply be included in the regularizer:
\begin{equation}
    \Omega(\beta, \gamma) = \lambda\left(
    \sum_{u = 1}^{|U|} \beta_u^2 + \sum_{k = 1}^K \gamma_{u,k}^2 + 
    \sum_{i = 1}^{|I|} \beta_i^2 + \sum_{k = 1}^K \gamma_{i,k}^2
                      \right).
    \label{eq:latentreg2}
\end{equation}
Alternately one could regularize $\beta$ and $\gamma$ with different strengths (since we generally expect $\beta$ to have larger values):
\begin{equation}
    \Omega(\beta) + \Omega(\gamma) = \lambda_1 \left(
    \sum_{u = 1}^{|U|} \beta_u^2 + \sum_{i = 1}^{|I|} \beta_i^2 \right) + 
    \lambda_2 \left(
    \sum_{u = 1}^{|U|} \sum_{k = 1}^K \gamma_{u,k}^2 + 
    \sum_{i = 1}^{|I|} \sum_{k = 1}^K \gamma_{i,k}^2
                      \right),
    \label{eq:latentreg3}
\end{equation}
though doing so is difficult as it results in multiple regularization constants to tune.

\subsubsection{Gradient update equations}

%Finally, 
Under this new model of \Cref{eq:MFscore},
the objective we wish to minimize (on a training set of interactions $\mathcal T$) is
\begin{align}
\text{obj}(\alpha; \beta; \gamma | \mathcal{T}) & = \frac{1}{|\mathcal T|} \sum_{(u,i) \in \mathcal{T}} (r(u,i) - R_{u,i})^2 + \lambda \Omega(\beta, \gamma)\\
%\done{check above. Looks fine}
    & = \frac{1}{|\mathcal{T}|} \sum_{(u,i) \in \mathcal{T}} (\alpha + \beta_i + \beta_u + \gamma_i + \gamma_u - R_{u,i})^2 + \lambda \Omega(\beta, \gamma).
    \label{eq:MFObjective}
\end{align}
%\done{Consistent notation for R_{(u,i)} in \mathcal{T} vs just (u,i) \in R or whatever.
Assuming the regularizer takes the form given in \Cref{eq:latentreg2}, the partial derivatives (for $\alpha$, $\beta_u$, and $\gamma_{u,k}$) are given by:
\begin{align}
\frac{\partial \text{obj}}{\partial \alpha} & = \frac{1}{|\mathcal{T}|}\sum_{(u,i) \in \mathcal{T}} 2(r(u,i) - R_{u,i})^2 \label{eq:MFgradientFirst}\\
\frac{\partial \text{obj}}{\partial \beta_u} & = \frac{1}{|\mathcal{T}|}\sum_{i \in I_u} 2(r(u,i) - R_{u,i})^2 + 2\beta_u\\
\frac{\partial \text{obj}}{\partial \gamma_{u,k}} & = \frac{1}{|\mathcal{T}|}\sum_{i \in I_u} 2\gamma_{i,k}(r(u,i) - R_{u,i})^2 + 2\gamma_{u,k}.
\label{eq:MFgradientLast}
\end{align}
Note the change of summation in the last two terms: the derivative for user $u$ is based only on items $I_u$ that they consumed (in the training set). Derivatives for $\beta_i$ and $\gamma_{u,k}$ can be computed similarly.
%and are left as an exercise (\cref{ex_?}).

\paragraph{Other considerations for gradient descent}
When we first introduced gradient descent in \Cref{sec:gradient}, we noted some potential issues in terms of local minima, learning rates etc.
It is worth revisiting some of those issues in light of the more complex model we are fitting here:
\begin{itemize}
    \item The problem in \Cref{eq:MFObjective} is certainly non-convex and has many local minima.\footnote{The proof can roughly be sketched as follows. The objective is smooth, and given any global optimum $\gamma_U$, $\gamma_I$, any permutation applied to both (i.e.,~$\gamma_U\pi$ and $\gamma_I\pi$) will result in an \emph{equivalent} local optimum.} Surprisingly though, this problem is not prone to `spurious' local optima, and if carefully implemented should converge to a global optimum \citep{ge2016matrix}.
    \item Nevertheless the problem is sensitive to initialization. For example if multiple columns of $\gamma_U$ and $\gamma_I$ are initialized to the same value, they will have identical gradients and will remain in `lock step' during successive iterations. This can normally be avoided simply by random initialization.
    \item Rather than computing the full gradient (as in \crefrange{eq:MFgradientFirst}{eq:MFgradientLast}), alternate approaches such as stochastic gradient descent, or alternating least squares, may converge faster or require less memory.\footnote{Alternating Least Squares notes that the optimization problem in \protect\Cref{eq:MFObjective} has a closed form if either $\gamma_U$ or $\gamma_I$ is fixed; 
    %\done{capital U and I? Ever used elsewhere? Yes, sometimes}
    optimization proceeds by alternately fixing one term and optimizing the other.} See e.g.~\cite{bottou2010large,yu2012scalable}.
\end{itemize}
% The gradient descent strategy presented above
% Faster gradient descent (coordinate descent): \cite{yu2012scalable}

%\done{Add something about how we've moved away from features completely}

\subsection{What Happened to User or Item Features?}

It is interesting to briefly consider that between our regression models in \Cref{chap:basicsReg} and the model-based recommender systems we developed above, we have gone from models that \emph{completely depend} on features to models that \emph{completely avoid} them. 

This can come as a surprise when first exploring recommender systems: obviously, features such a movie's budget or its genre ought to be predictive of users' preferences toward it. However,
%\begin{itemize}
    %\item 
    to the extent that a feature 
    %(such as a high budget, or a large amount of action)
    is predictive, it will already be captured by $\gamma_u$ or $\gamma_i$. These parameters will capture whatever dimensions \emph{maximally explain variance} in interactions, without any need to explicitly measure that feature.
%\end{itemize}

As such, one might argue that if we observe enough interactions, 
%we can learn latent preferences and properties via 
$\gamma_u$ and $\gamma_i$ will 
%tell us whatever patterns 
capture whatever user and item characteristics are useful. We'll revisit this argument in later chapters and explore various exceptions, e.g.~what can we do if we \emph{don't} have sufficient interaction data (e.g.~for new users or items, \Cref{sec:coldStart}), or what should we do if user preferences or item properties aren't stationary over time (\cref{chap:temporalSequential}).

\section{Implicit Feedback and Ranking Models}
\label{sec:implicit}

So far, our discussion of (model-based) recommender systems has focused on predicting real-valued outcomes such as ratings, using objectives based on the Mean Squared Error. That is, we have described model-based recommendation in terms of \emph{regression} approaches.

Just as we developed separate approaches for neighborhood-based recommendation when considering click, purchase, or rating data (\cref{sec:neighborhood}), here we consider how our regression-based approaches should be adapted to handle binary outcomes (such as clicks and purchases).

Na\"ively, we might imagine that we could adapt our regression-based approaches to handle binary outcomes much in the same way we that developed logistic regression in \Cref{chap:classPipe}. That is, we could pass the model output (\cref{eq:MFscore}) through a sigmoid function, such that positive interactions are associated with high probabilities, and negative interactions are associated with low probabilities.

However, when dealing with click or purchase data, we should consider that items which haven't been clicked or purchased are not necessarily \emph{negative} interactions---in fact items that \emph{haven't} been clicked or purchased are exactly the ones we intend to recommend.

Several techniques have been proposed to handle recommendation in this context. Often this setting is referred to as \emph{one-class} recommendation, as only the `positive' class (clicks, purchases, listens, etc.)~is observed. The setting is also referred to as
%Also called 
\emph{implicit feedback recommendation}, given that the signals (whether or not to buy something) only implicitly measure whether we like or dislike an item.

\subsection{Instance Re-weighting Schemes}
\label{subsec:instanceReweight}

One category of methods for dealing with implicit feedback data attempts to reweight instances as having various `confidences' of being positive or negative.

\cite{hu2008collaborative} consider cases where positive instances are associated with `confidence' measures $r_{u,i}$, which could measure e.g.~the number of times a user listened to a song or watched a particular program.
%in \cite{hu2008collaborative} the 
%confidence values encode how many times a user has watched a particular program.
Negative instances still have $r_{u,i} = 0$, such that the model essentially 
%captures the fact 
assumes
that negative instances are necessarily associated with low confidence, whereas confidence may vary substantially among positive instances.

Ultimately, the goal is still to predict a binary outcome $p_{u,i}$, and the model is trained to predict
\begin{equation}
    p_{u,i} = \left\lbrace \begin{tabular}{cl}
    1 & if $r_{u,i} > 0$\\
    0 & otherwise
    \end{tabular}\right..
\end{equation}
The form of the predictor is similar to that of \Cref{eq:MFObjective}, i.e.,~latent user and item factors are use to predict $p_{u,i}$ using a (regularized) Mean Squared Error. The main difference is that the MSE is weighted according to the confidence of each observation:\footnote{For brevity we will sometimes omit the normalization $\frac{1}{|\mathcal T|}$ from our training objective. In practice this term is optional as it simply scales the objective by a constant.}
\begin{equation}
    \argmin_\gamma \sum_{(u,i) \in \mathcal T} c_{u,i}(p_{u,i} - \gamma_u \cdot \gamma_i)^2 + \lambda \Omega(\gamma),
    \label{eq:weightedMF}
\end{equation}
where $c_{u,i}$ is a weighting function associated with each observation, which ultimately is a monotone transform of $r_{u,i}$, e.g.
\begin{equation}
    c_{u,i} = 1 + \alpha r_{u,i}; \quad \text{or} \quad c_{u,i} = 1 + \alpha \log(1 + r_{u,i} / \epsilon),
\end{equation}
where $\alpha$ and $\epsilon$ are tunable hyperparameters. Note that the transform $c_{u,i}$ ensures that negative instances have small but non-zero weight, whereas positive instances receive increasingly higher weight according to their associated confidence.

%Paper 1: Collaborative Filtering for Implicit Feedback Datasets

\cite{pan2008one} approach the problem in a similar way, also fitting a function of the form in \Cref{eq:weightedMF}, though their weighting scheme is applied to \emph{negative} instances. Several schemes are proposed, two of which are as follows:
\begin{equation}
  c_{u,i} = \alpha \times | I_u |; \quad \text{or} \quad c_{u,i} = \alpha ( m - | U_i | ).
\end{equation}
The first (which they call `user oriented' weighting) suggests that a negative instance should be weighted higher if the corresponding \emph{user} has interacted with many items; the second assumes that a negative instance should be weighted higher if the corresponding \emph{item} has few associated interactions. 
%\cite{pan2008one} also explores sampling schemes,



% Although the scheme above is ultimately a simple heuristic for reweighting the model we developed in \Cref{eq:MFObjective}, experiments in \cite{hu2008collaborative} show that this scheme outperforms models that try to predict $r_{u,i}$ or $p_{u,i}$ directly.

Although the schemes above are ultimately simple heuristics for reweighting the model we developed in \Cref{eq:MFObjective}, experiments in \cite{hu2008collaborative}
and
\cite{pan2008one}
show that these scheme outperforms models that try to predict $p_{u,i}$ (or $r_{u,i}$) directly.


%\subsection{Contrastive Learning for Personalized Ranking}
%\done{generic section title, could be better. Is fine}
\subsection{Bayesian Personalized Ranking}
\label{subsec:BPR}

While the above reweighting schemes demonstrate the importance of treating `negative' and `positive' feedback carefully in implicit-feedback settings, 
%they are ultimately simple heuristics whose success depends upon whether they can be adapted or tuned to a specific application or dataset.
they ultimately optimize \emph{regression} objectives, and therefore still seek to assign `negative' scores to unseen instances.

A potential objection to such an approach is that the unseen instances are exactly the ones we \emph{want} to recommend, and thus we should not encourage a model to assign them a negative score. A weaker assumption might state that while unseen instances should have \emph{lower} scores than positive instances, they need not have negative scores. That is, items which we know a user likes are `more positive' than unseen items, but unseen items could still 
%be positive.
have positive scores.

%Paper 2: BPR
%The above papers treated positive and negative instances differently, but still relied on regression objectives. As such, non-observed instances are still 
\cite{rendle2012bpr} built models based on the above principle by borrowing ideas from ranking. Recall from \Cref{sec:errorRanking} that the above principle is similar to our goal when adapting classifiers for ranking: while positive (or relevant) items should appear near the top of a ranked list, we are not concerned with their actual (positive or negative) scores.

Likewise, the principle behind their method, Bayesian Personalized Ranking (BPR), is that we should generate ranked lists of items such that positive items appear first. This is achieved by training a predictor $x_{u,i,j}$ that assigns a score based on which of the two items ($i$ or $j$) is preferred (i.e.,~ranked higher) by $u$:%
\begin{align}
    x_{u,i,j} > 0 & \rightarrow \text{$u$ prefers $i$}\\
    x_{u,i,j} \leq 0 & \rightarrow \text{$u$ prefers $j$}.
\end{align}
% \begin{equation}
%     \underbrace{\texttt{+++++++++}}_{\text{positives}}\underbrace{\texttt{?????????????}}_{\text{unseen items}}
% \end{equation}
Now, if we know that $i$ is a positive and $j$ is a negative (or unseen) item for user $u$, then a good model should tend to output positive values of $x_{u,i,j}$. This type of prediction strategy (which compares two samples, rather than assigning a score to a single sample) is known as a \emph{pairwise} model (\cref{box:pairwise}).
%we can measure whether the model correctly assigns a positive score by $\delta(x_{u,i,j} > 0)$, which we will later use to define our objective function.
%\done{poor organization}


\begin{figure}
\caption{Pointwise versus pairwise recommendation.
\label{box:pairwise}}
%\fbox{
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
The type of predictor we develop in \Cref{eq:bprOpt0}---which compares two samples $i$ and $j$, rather than assigning a score to a single sample---is known as a \emph{pairwise} predictor.

\emph{Pointwise predictors} estimate a score or label associated with a particular sample $i$. All of the regression and classification models from \Cref{chap:basicsReg,chap:classPipe} are examples of pointwise predictors, as are our latent factor models from \Cref{subsec:matrixFactor}.

\emph{Pairwise predictors} compare two samples $i$ and $j$.
%in order to determine which has a higher score. 
Such predictors are often preferable when training ranking functions (since they act as proxies for objectives like the AUC, as in \Cref{eq:bprOpt1}). They are often used in implicit feedback settings, where neither sample has a `negative' label, but we can still assume that positive instances should rank higher. We'll also use such predictors in cases where outcomes are associated with pairs of samples, such as when generating compatible outfits in \Cref{chap:visual}.
\end{minipage}
}
\end{figure}
%\done{fbox all boxes}
%\done{maybe make boxes light grey background?}
%\done{work more on formatting, spacing, color}


$x_{u,i,j}$ could be any predictor, though the most straightforward option is to define it in terms of \emph{difference} between 
%pairwise predictors, 
predictions,
e.g.:
\begin{equation}
    x_{u,i,j} = \underbrace{x_{u,i}\vphantom{x_{u,j}}}_{\mathllap{\text{$u$'s preference toward $i$\vphantom{$j$}}}\hspace{-5mm}} - \underbrace{x_{u,j}}_{\hspace{-5mm}\mathrlap{\text{$u$'s preference toward $j$}}}.
    \label{eq:bprOptDiff}
\end{equation}
%\done{define pointwise versus pairwise. Adjust exercise to refer}
%For instance, 
Compatibility $x_{u,i}$ could be defined via a latent factor model, similar to that of \Cref{eq:MFscore}:\footnote{The implementation of BPR in \protect\cite{rendle2012bpr} does not include bias terms $\beta_i$ or $\beta_j$, though they can straightforwardly be included in \Cref{eq:bprOpt0}.}
%\done{should have bias term or not?}
\begin{equation}
    x_{u,i,j} = x_{u,i} - x_{u,j} = \gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j.
    \label{eq:bprOpt0}
\end{equation}
%\done{Replace all \clap with \mathclap or silimar (same for rlap and llap), which preserves font size}
Again, note that our goal is not that $\gamma_u \cdot \gamma_i$ should be positive for the positive item, nor that $\gamma_u \cdot \gamma_j$ should be negative for the unseen item, only that the \emph{difference} is positive, i.e.,~that the positive item has a higher compatibility score.



We can now define our objective in terms of whether the model correctly outputs positive values $x_{u,i,j}$ given a positive item $i$ and an unseen item $j$.
%\done{Kind of want to use x_{uij} rather than x_{u,i,j} (etc.) everywhere. Doesn't matter}
Ideally, we would like to count how often the model is able to correctly rank positive items higher than unseen items. For a specific user $u$ we have:
\begin{equation}
    \mathrm{AUC}(u) = \frac{1}{|I_u||I \setminus I_u|} \underbrace{\sum_{i \in I_u \vphantom{j \in I \setminus I_u}}}_{\mathllap{\text{positive items for user $u$}}\hspace{-5mm}} \underbrace{\sum_{j \in I \setminus I_u}}_{\hspace{-5mm}\mathrlap{\text{unseen items for user $u$}}} \delta(x_{u,i,j} > 0).
    \label{eq:bprOpt1}
\end{equation}
The name `AUC' stands for `Area Under the ROC Curve;' this measure is equivalent to computing the area under ROC curve as we introduced in \Cref{sec:errorRanking}.
%which originates from a different way of deriving the same quantity.\footnote{Specifically, it is the Area Under the Receiver-Operating Characteristic Curve, which is a curve that relates the false positive and true positive rates of a classifier.}
%\done{Cover ROC curve along with Precision Recall curves, and drop this footnote and previous sentence.}
For an entire dataset we average the above across all users:
\begin{equation}
    \mathrm{AUC} = \frac{1}{| U |} \mathrm{AUC}(u)
\end{equation}
%\done{AUC should probably have an argument, like AUC(\gamma|something). Doesn't matter}
Note that this quantity takes a value between 0 and 1, where an AUC of 1 means that the model always ranks positive items higher than unseen items; an AUC of 0.5 means that the model is no better than random.

Optimizing the above presents two issues. First, it is not feasible to consider all $(u,i,j)$ triples; to address this one can randomly sample a fixed number of unseen items $j$ per positive item $i$.\footnote{Furthermore, the sampled items could change during each iteration of training.}

Second, the objective in \Cref{eq:bprOpt1} is a step function, whose derivative is zero almost everywhere. This is much the same issue we encountered when developing logistic regression in \Cref{sec:logisticRegression}; as such we can take the same approach by replacing the step function $\delta(x_{u,i,j})$ with a differentiable surrogate such as the sigmoid function (see \cref{fig:sigmoid}). Using the sigmoid function allows us to interpret $\sigma(x_{u,i,j})$ as a probability:
\begin{equation}
    p(\text{$u$ prefers $i$ over $j$}) = \sigma(x_{u,i,j}).
\end{equation}
From this point, optimization proceeds in much the same way as we developed logistic regression: we use $\sigma(x_{u,i,j})$ to define a (log-)probability of a model given a training set, and subtract a regularizer:
\begin{align}
  \mathrm{obj}^{\text{(BPR)}} & = \ell (\gamma ; \mathcal T) - \lambda \Omega(\gamma)\\
                  & = \sum_{(u,i,j) \in \mathcal T} \log \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j) - \lambda \Omega(\gamma).
                  \label{eq:BPRobjective}
\end{align}
Assuming an $\ell_2$ regularizer $\Omega(\gamma) = \| \gamma \|_2^2$, we can compute the derivative, for example with respect to $\gamma_{u,k}$:
\begin{align}
    \frac{\partial \text{obj}}{\partial \gamma_{u,k}} & = \frac{\partial}{\partial \gamma_{u,k}} \sum_{(u,i,j) \in \mathcal T} \log \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j) - \lambda \| \gamma \|_2^2\\
    & = \frac{\partial}{\partial \gamma_{u,k}} \sum_{(u,i,j) \in \mathcal T} -\log(1 + e^{\gamma_u \cdot \gamma_j - \gamma_u \cdot \gamma_i}) - \lambda \| \gamma \|_2^2\\
    & = \sum_{(i,j) \in I_u} (\gamma_{i,k} - \gamma_{j,k}) (1 - \sigma(\gamma_u \cdot \gamma_i - \gamma_u \cdot \gamma_j)) - 2\lambda\gamma_{u,k}
\end{align}
%\done{Use \ln and \log consistently. Always use \log}
%\done{\mathcal l or \ell ? don't think \mathcal l exists}
%\done{\gamma | T , or \gamma ; T, or something else? Looks alright}
%\done{Do I ever call the regularizer an \ell_2 regularizer? Maybe just drop. Yes}
(we abuse notation slightly so that $(i,j) \in I_u$ includes both positive and unseen items sampled from a user's history).
Derivatives of other terms can be computed similarly. In practice, it is often preferable to use libraries that compute such derivatives automatically, as we explore in \Cref{subsec:LFMTensorflow}.

\section{`User-free' Model-based Approaches}
\label{sec:userFree1}

%\done{intro. Moved from another chapter}
%Recommend items on the basis of other items
%When establishing a distinction between memory
At the beginning of this chapter, we drew a distinction between memory-based versus model-based recommenders. Roughly, \emph{memory-based} approaches 
%recommend 
make recommendations
using algorithms that operate on \emph{histories} associated with users and items; \emph{model-based} approaches generally distill these histories into low-dimensional user and item representations.

In practice, this distinction is not always so clear. Below we study two models that learn item representations but eschew user representations. At inference time, predictions are made in terms of parameters associated with items in the user's history, though the history itself is never distilled into a single representation.

Such models may be preferable for a few reasons. First, avoiding user terms and directly making use of their interaction history can make models easier to deploy: the model does not have to be updated as new user interactions are observed. Second, this approach can be preferable when user interactions are \emph{sparse}, meaning that we can fit complex item representations but cannot reliably fit parameters like $\gamma_u$. Third, when we explore sequential settings in \Cref{chap:temporalSequential}, there may be important information in the user's history (such as the order in which items are consumed) which is not preserved by user representations.

We briefly explore a few such methods below, which we summarize in \Cref{tab:userFree1}.

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Summary of user-free recommendation models.]{Summary of user-free recommendation models. References: \protect\cite{ning2011slim,kabbur2013fism,barkan2016item2vec}.}
%\done{could be two tables? Is}
\label{tab:userFree1}
%\done{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{|XXX|}
% Method & Reference & Description\\
% %\done{column ordering across tables}
% Sparse Linear Methods (SLIM) & \cite{ning2011slim} & Each user is associated with a linear model weighting their interactions over past items; sparsity-inducing regularizers are used to deal with the large number of model parameters.\\
% Factored Item Similarity Models (FISM) & \cite{kabbur2013fism} & Replaces the user term in a latent factor model with a second term that represents the user by averaging over item representations from their history. \\ 
% %\done{mouthful}
% item2vec & \cite{barkan2016item2vec} & \\
% \end{tabularx}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.29\textwidth}X}
\toprule
\textbf{Ref.} & \textbf{Method} & \textbf{Description}\\
\midrule
%\done{column ordering across tables}
%\cite{ning2011slim}
NK11 &
Sparse Linear Methods (SLIM)  & Each user is associated with a linear model weighting their interactions over past items; sparsity-inducing regularizers are used to deal with the large number of model parameters (\cref{subsec:slim}).\\ 
% \cite{kabbur2013fism}
K13 &
Factored Item Similarity Models (FISM) &  Replaces the user term in a latent factor model with a second term that represents the user by averaging over item representations from their history (\cref{subsec:fism}). \\ 
%\done{mouthful}
% \cite{barkan2016item2vec}
BK16 &
item2vec & Items should have compatible representations to those that appear nearby in user sessions (we discuss this method in \Cref{chap:text}).\\
%\done{put this in here, need to move method or table. Or move to sequential recommendation table}
%\done{Or maybe with sequential models? Or maybe just drop this table. Put here for now}
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}


\subsection{Sparse Linear Methods (SLIM)}
\label{subsec:slim}
%\done{Why is SLIM here? It isn't a temporal model. Probably move to previous chapter. Put in same place as FISM}

%While not specifically a sequential model, 
A direct way to avoid including an explicit user term (i.e.,~$\gamma_u$) is to describe all of a user's interactions in terms of a binary feature vector enumerating which items they have interacted with (i.e.,~a vector of length $|I|$). To predict the score associated with an item $i$, we can then train a linear model (again with $|I|$ parameters), much as we did in \Cref{chap:basicsReg}:
\begin{equation}
f(u,i) = R_{u} \cdot W_i
\label{eq:slimEq0}
\end{equation}
Here $R_u$ is a (sparse) vector describing all of a user's interactions, 
i.e.,~equivalent to a row of the interaction matrix $R$ from \Cref{eq:interactionMatrix}. 

Fitting such a model na\"ively is not straightforward, given the high dimensional feature and parameter vectors involved. 
\cite{ning2011slim} attempt to fit this type of model by exploiting the specific sparsity structure of the vector $R_{u}$, noting that \Cref{eq:slimEq0} can be rewritten in terms of just the items $I_u$ that the user has interacted with:
\begin{equation}
 f(u,i) = \sum_{j \in I_u} R_{u,i} W_{i,j}.
    % f(u,i) & = & R_{u} \cdot W_i\\
    %       & = & \sum_{j \in I_u} R_{u,i} W_{i,j}
           %\done{Check 2nd bit. Looks okay}
           \label{eq:slimEq}
\end{equation}
%Here $R_u$ is a (sparse) vector describing all of a user's interactions, 
%i.e.,~equivalent to a row of the interaction matrix $R$ from \Cref{eq:RRR?}. 
Here $W$ is an $|I|\times |I|$ parameter matrix which essentially measures item-to-item compatibilities (or similarities).

Conceptually, 
\Cref{eq:slimEq} 
%\done{Figure out why equation above points to the wrong place. Looks fine now}
is similar to the simple heuristic we developed in \Cref{subsec:simPredictRating},  in which we predicted a rating using a weighted average of previous ratings, where the weighting function was determined by an item-to-item similarity measure (such as the cosine similarity). Essentially, SLIM follows the same reasoning, but replaces the heuristic item-to-item similarities from \Cref{subsec:simPredictRating} with a learned matrix $W$.
%\done{Maybe motivate the model with this argument rather than putting it here. No}

The main challenge in fitting $W$ is that it has dimension $|I| \times |I|$.
%\done{notation}
Were $W$ a dense matrix (i.e.,~every item interacts with every item) training and inference would be expensive; this is circumvented by using a regularization approach which ensures that $W$ is sparse.\footnote{Though in practice experiments are conducted with moderate item vocabularies (e.g.~$|I| \simeq 50000$),
%\done{notation}
and various computational tricks are used to allow for parallelization and efficient inference.}
Sparsity is achieved via a regularization strategy which includes both an $\ell_2$ and $\ell_1$
%\done{notation}
regularizer:
%, the latter of which induces sparsity as we introduced in \Cref{sec_?}:
\begin{align}
\begin{split}
    % \argmin_W \| R - R W^T \|_2^2 + \lambda \Omega^{(2)}(W) + \lambda' \Omega^{(1)}(W)\\
    \argmin_W \| R - R W^T \|_2^2 + \lambda \Omega_2(W) + \lambda' \Omega_1(W)\\
    \text{s.t.} \quad W_{i,j} \geq 0;\quad
    W_{i,i} = 0.
    \label{eq:slimEq2}
\end{split}
\end{align}
%\done{Do I ever use the \| \| notation? Yes add to notation page}
%\done{Don't love the Omega (1) (2) notation. Is okay}
%\done{Sometimes Omega^(1), sometimes \Omega_1. Fix}
Note that $\| R - RW^T \|_2^2$ is merely a matrix shorthand for the predictions made for all interactions $(u,i)$ following \Cref{eq:slimEq}. The first constraint in \Cref{eq:slimEq2} ensures that all terms in the weighting function are positive; the second constraint ($W_{i,i} = 0$) ensures that each item $i$'s rating is predicted only based on interactions with \emph{other} items $j$. $\Omega_1$ is an $\ell_1$ regularizer (i.e.,~$\Omega_1(W) = \sum_{i,j} | W_{i,j} |$); 
%\done{Above looks wrong, what is the | doing there? Fixed}
as we discussed in \Cref{sec:regularization}, $\ell_1$ regularization leads to sparsity of the matrix $W$.
%\done{Probably just write out the regularizer to avoid notation. Written out}

\cite{ning2011slim} discuss various merits of the above approach. Notably the rapid inference time (i.e.,~the rate at which recommendations can be made) compared to standard recommendation approaches, and also the \emph{long-tail} performance of the approach. For the latter, compared to (e.g.)~latent factor approaches---whose representations tend to favor whichever types of items predominate in the data, but fail to capture the dynamics of rarer items---SLIM maintains (relatively) good performance even for tail items.
%\done{clunky sentence. Looks okay}

\subsection{Factored Item Similarity Models (FISM)}
\label{subsec:fism}
%\done{FISM and related models (and exercises) could be moved to "structured" chapter. Moved here instead}

Factored
%\done{I think should be factored, not factorized, check elsewhere}
Item Similarity Models \citep{kabbur2013fism} attempt 
%to directly apply the above
%\done{Not really "above" anymore}
%reasoning, by replacing 
to replace
the user term $\gamma_u$ in a latent factor model (\cref{eq:MFscore}) with a term that aggregates item representations from a user's history. Specifically, the user term in \Cref{eq:MFscore} is replaced with an average over \emph{item} terms for all items consumed by that user:
\begin{equation}
    f(u,i) = \alpha + \beta_u + \beta_i + \frac{1}{|I_u \setminus \lbrace i \rbrace |} \sum_{j \in I_u \setminus \lbrace i \rbrace} \gamma'_j \cdot \gamma_i
    %\done{I assume this term should be adjusted to exclude i from j \in I_u?}
    \label{eq:fismEq}
    %\done{does fism have a user bias? Sounds funny. Yes}
\end{equation}
(recall that $I_u$ is the set of items consumed by user $u$, and we exclude the query item $i$ during model training). Note that the item term $\gamma_i$ is separate from the term used to average user actions $\gamma'_j$, i.e.,~the model learns \emph{two} sets of latent factors per item.\footnote{Strictly speaking, FISM is not `user-free' in the sense that it includes a bias term $\beta_u$; however including this term requires only a single parameter per user, or could be excluded.}

Spiritually, the average $\frac{1}{|I_u|} \sum_j \gamma'_j$ fulfils the same role as $\gamma_u$, by summarizing the dimensions that are compatible with a particular user.

\cite{kabbur2013fism} consider variants of \Cref{eq:fismEq} for both rating prediction and ranking problems (i.e.,~to optimize the MSE as in \Cref{subsec:fitLFM} or the AUC as in \Cref{subsec:BPR}).

\cite{kabbur2013fism} argue that the above approach is particularly useful in \emph{sparse} datasets (presumably, datasets where \emph{users} have few associated interactions, while items have several). That is, a traditional latent factor model as in \Cref{eq:MFscore} would struggle to meaningfully fit $\gamma_u$ for a user who has only a few interactions; whereas if item histories are denser, a reasonable estimate of user preferences can be estimated by averaging over item terms. Indeed, experiments in \cite{kabbur2013fism} show that the settings in which FISM are effective are closely related to dataset sparsity.


%FISM 


\subsection{Item2Vec}

Finally, we briefly mention \emph{item2vec} \citep{barkan2016item2vec}, though will not discuss this method in detail until we discuss text-based methods in \Cref{chap:text}.

\emph{Item2vec} is an adaptation of \emph{word2vec}, a natural language model that learns semantic representations that describe relations among words (\cref{sec:word2vec}). 
Just as word2vec discovers `synonyms' 
%that occur in the same context in a sequence of words,
based on relationships among words in a sequence,
%Essentially, 
item2vec learns item representations $\gamma_i$ that 
are capable of predicting which items occur in the same sequential context.
%in a sequence (in much the same way that word2vec identifies
`Sequential context' is derived by ordering users' interactions (i.e.,~based on interaction timestamps); such models are also related to the sequential (or `session-based') models we develop in \Cref{chap:temporalSequential}.

%\done{mention briefly and reference later chapter.}
%\done{mention relationship between these models and sequential models}


\section{Evaluating Recommender Systems}
\label{sec:evaluateRec}

%see e.g. http://ceur-ws.org/Vol-910/rue2012.pdf

So far, when developing models to predict ratings as in \Cref{subsec:matrixFactor}, we have done so by optimizing objectives based on a \emph{sum of squared errors} (or equivalently, a Mean Squared Error). Recall that in \Cref{sec:whyMSE}, we discussed the motivation behind the Mean Squared Error, as well as some potential pitfalls when using it.

In the case of recommender systems, we must be aware of the same pitfalls, but also some different ones. Critically, since the system is likely used to provide ranked lists of items to the user, actual prediction of ratings may not be critical so long as desirable items appear near the top of the ranking.  
%
% Paper: Precision-Oriented Evaluation of Recommender Systems:
% An Algorithmic Comparison
%
%misclassified decoys (see refs in http://proceedings.mlr.press/v81/ekstrand18b/ekstrand18b.pdf)
%
Consider, for example, some potential problems with the Mean Squared Error in a recommendation context:
\begin{itemize}
  \item Using the MSE, mispredicting a 5-star rating as 4-stars incurs a smaller penalty than mispredicting 3-stars as 1-star. Arguably, the latter should have a smaller penalty, as it concerns an item which should never have been recommended anyway.
  \item Similarly, mispredicting ratings of 3 and 3.5 as 4 and 4.5 (respectively) would incur a larger penalty than mispredicting them as 3.5 and 3. However the former preserves the \emph{ordering} of the two items, whereas the latter does not.
  \item As we saw in \Cref{sec:whyMSE}, the MSE corresponds to an implicit assumption that errors are normally distributed; critically this assumes that outliers are extremely rare and should be penalized accordingly. In practice, outliers may be common, or alternately errors could be bimodal (or otherwise violate our model assumptions).
  \item \cite{bellogin2011precision} noted the issue of `popularity bias,' whereby strong performance on popular items can mask performance issues for less-popular ones.\footnote{Though this is not an issue with the MSE specifically, but rather a general problem of evaluation in imbalanced datasets.}
\end{itemize}

Ultimately, such problems raise the question of whether reductions in Mean Squared Error actually correspond to increased utility of a recommender system. Interestingly, it is reported in \cite{koren2009collaborative} that a carefully implemented model with temporally-evolving bias terms (which we discuss in \Cref{sec:caseStudyNetflixTemporal}) outperformed Netflix's previous solution (\emph{Cinematch}) in terms of the RMSE. Critically, a system without any interaction terms (e.g.~$\gamma_u$ or $\gamma_i$) can do little more than recommend popular items over time; as such their experiment suggests that the system with a better MSE is not necessarily the better recommender.

Some of the above issues suggest the use of alternative regression metrics, such as the Mean Absolute Error, which (for example) is less sensitive to outliers, as we argued in \Cref{subsec:errorsArent}. 
%Others require more subtle solutions; for example
Others suggest that perhaps a recommender system should be evaluated less like a regression problem and more like a ranking problem. That is, so long as items matching a user's interests have the \emph{highest} predicted scores, the precise accuracy of our predictions is unimportant.

Arguably, such problems with the Mean Squared Error are driving research toward settings that rely on implicit feedback (clicks, purchases, etc.)~rather than ratings; alternately, these settings may be preferable simply because such data is more available than explicit feedback (few users rate items, but \emph{every} user clicks on them). 
%such as ratings.
More crudely, optimizing clicks or purchases may simply correspond more closely to business goals compared to identifying highly-rated items.

%\done{precision/recall " @ K" vs "@K"}

We already saw in \Cref{subsec:BPR} one technique to train recommender systems to optimize a ranking loss based on implicit feedback, namely the AUC. Conceptually, the AUC reflects our ability to guess which of two items is `relevant:' an AUC of 1 means that we always correctly select the correct item, whereas an AUC of 0.5 means our guesses are no better than random.

However, the AUC is but one choice of ranking loss, and was primarily chosen for its convenience when formulating the optimization problem in \Cref{eq:BPRobjective}. As in \Cref{sec:errorRanking}, when considering cases where recommendations are surfaced to a user via an interface, we may be particularly interested in how the recommender system performs among the top $K$ ranked items.

Below we present a few alternative evaluation functions to measure recommendation performance, most of which are focused on achieving high accuracy among the top-ranked items.

%\subsection{Recall @ $\boldsymbol{K}$ and Precision @ $\boldsymbol{K}$}
\subsection[Precision and Recall @ $K$]{Precision and Recall @ $\boldsymbol{K}$}
%\done{why rec and prec rather than prec and rec? No reason, fixed}

When we evaluated classifiers in \Cref{sec:evaluatingClassifiers}, we motivated the precision and recall@$K$ as useful metrics in the context of evaluating user interfaces, where we have a fixed budget ($K$) of results that can be returned. Likewise, when recommending items, we might consider whether relevant items (e.g.~those that a user eventually interacts with) are given a high ranking.

For convenience, when evaluating recommenders in this setting, it is useful to define a variable $\mathit{rank}_u(i)$ that specifies in what position an item $i$ was ranked for a particular user $u$. That is, given a compatibility function $f(u,i)$, and a set of $N$ items that can potentially be recommended (potentially excluding e.g.~interactions that already appeared in the training set), then $\mathit{rank}_u(i) \in \lbrace 1 \ldots N \rbrace$ such that
\begin{align}
    \mathit{rank}_u(i) < \mathit{rank}_u(j) & \Leftrightarrow  f(u,i) > f(u,j)\\
    \mathit{rank}_u(i) = \mathit{rank}_u(j) & \Leftrightarrow  i = j.
\end{align}
Now, given a test set of observed interactions $I_u$ we 
%could 
define the precision@$K$
%\done{formatting. Looks fine}
(for a particular user $u$) as
\begin{equation}
  \text{precision}@K(u) = \frac{|\lbrace i \in I_u \mid \mathit{rank}_u(i) \leq K \rbrace|}{K}.
\end{equation}
As in \Cref{eq:precRecDefn1}, the numerator is the number of relevant items that were retrieved, while the denominator is the number of retrieved items. Now to compute the precision@$K$ we simply average over all users:
\begin{equation}
    \text{precision}@K = \frac{1}{|U|} \sum_{u \in U}\text{precision}@K(u).
\end{equation}
%\done{formatting, rank_{u,i} or rank(u,i), also prec or precision or Precision or \text{Precision}}
Likewise the recall@$K$ is defined similarly:
\begin{equation}
    \text{recall}@K = \frac{1}{|U|} \sum_{u \in U} \frac{|\lbrace i \in I_u \mid \mathit{rank}_u(i) \leq K \rbrace|}{|I_u|}
\end{equation}

%\paragraph{Hit Rate}

%\Done{Are there any papers that use recall or do they all call it hit rate? Addressed in footnote above}

%One potential challenge with the precision and recall as defined above is that we are averaging across users who may have very different numbers of relevant items $I_u$ in the test set; optimizing an algorithm 
%\done{Don't know what I had in mind here}

%\done{max marginal relevance refs could accidentally point here. Nothing points here}

\subsection{Mean Reciprocal Rank}
\label{subsec:MReciprocalR}
%\done{All paragraphs need . in current format. No they don't}
%
%\done{Above is quite sloppy. Is fine}
%
%Mean Average Precision
%
The \emph{Mean Reciprocal Rank} (MRR) is another metric to assess whether a recommender system (or any classifier) ranks positive items highly; unlike the precision and recall@$K$
this expression does not depend on a particular size of the returned set of items, but rather rewards methods for ranking relevant items near the top of the list.
%\done{messy description. Okay}

Traditionally, in search settings, the Mean Reciprocal Rank is defined in terms of the \emph{first} relevant item among a ranked list of retrieved results, though in recommendation settings the metric is typically used by building a test set that consists of only a single relevant item per user, $i_u$. Then, the Mean Reciprocal Rank is defined in terms of the inverse (reciprocal) of the rank of the relevant item:
\begin{equation}
    \mathit{MRR} = \frac{1}{|U|} \sum_{u \in U} \frac{1}{\mathit{rank}_u(i_u)}.
\end{equation}
%This metric will typically rank between 1 (
A score of 1 means 
%if 
the relevant item is always ranked in the first position; 
%and 0.5 (if the relevant item appears near the middle, or at a random position in the ranked list).
a value of $1/n$ would mean items are on average ranked in the $n^\text{th}$ position.
%\done{Looks fine}

%\done{Looks wrong (not normalized correctly? If there were 2000 items the MRR would be 1/1000 on average.}

\subsection{Cumulative Gain and NDCG}
\label{subsec:cumGain}
%\done{Not great paragraph title. It's fine}
%\done{maybe change to subsubsections though. Looks okay as subsection}
The \emph{Cumulative Gain} (and its variants) aim to measure ranking performance in a setting that resembles a user browsing a page of search results: relevant results should be among the top $K$ results, and ideally should be close to the top of the ranked list.
%
The \emph{Cumulative Gain} (here for a particular user $u$) simply counts the number of relevant items among the top $K$ results:
%\done{Renders bad}
%\done{should have a sum over users or something, as in the DCG expression. Removed both}
\begin{equation}
    \text{Cumulative Gain}@K = \sum_{\hspace{5mm}\mathclap{i \in \lbrace i \mid \mathit{rank}_u(i_u) \leq K \rbrace}\hspace{5mm}} y_{u,i},
\end{equation}
where $y_{u,i}$ is either a binary label (e.g.~whether an item was purchased) or a relevance score (such as a rating). That is, the Cumulative Gain will be high if there are many relevant items (or highly rated items) among the top $K$ results.

Ideally, relevant results should appear closer to the top of the list; the \emph{Discounted} Cumulative Gain (DCG) accomplishes this by discounting the reward for items in lower ranks: 
%(so that an ideal ranker would rank all items in decreasing order of relevance):
\begin{equation}
    \text{DCG}@K = 
    %\sum_u 
    \sum_{i \in \lbrace i \mid \mathit{rank}_u(i_u) \leq K \rbrace} \frac{y_{u,i}}{\log_2(\mathit{rank}_u(i) + 1)}.
    \label{eq:DCG}
\end{equation}
This expression is often normalized by comparison against an idealized ranking function to obtain the \emph{Normalized} Discounted Cumulative Gain (NDCG):%
\begin{equation}
    \text{NDCG}@K = \frac{\mathit{DCG}@K}{\mathit{IDCG}@K}
\end{equation}
where $\mathit{IDCG}@K$ is the `ideal' discounted cumulative gain, i.e.,~the discounted cumulative gain that would have been achieved via an optimal ranking function $\mathit{rank}^{\text{opt}}_u(i)$ (i.e.,~where labels $y_{u,i}$ are sorted in decreasing order of relevance). This normalization, and the specific choice of logarithmic scaling in \Cref{eq:DCG} are theoretically justified in
\cite{wang2013theoretical}.



%\paragraph{Novelty, Diversity, Serendipity, etc.}
\subsection{Evaluation Metrics Beyond Model Accuracy}
%\done{Should I mention all of those things? I don't think they're covered elsewhere. All covered in last chapter}
%
%Serendipity: \cite{herlocker2004evaluating} 
%(serendipity, according to http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.62.9683&rep=rep1&type=pdf)
%
%Coverage: An algorithmic framework for performing collaborative filtering (maybe)?
%according to same ref above
%
Finally, it should be noted that there are several other qualities we might desire from a recommender system beyond its immediate utility to users (i.e.,~its ability to predict the next action). For example, we might be interested in exposing users to diverse viewpoints, ensuring that recommendations aren't biased against certain groups, that users don't get pushed toward `extreme' content, etc. We'll revisit these issues in \Cref{chap:consequence},
%(\cref{sec:filterBubble}), 
when we consider the broader consequences and ethics of personalized machine learning. Several are backed by user studies aimed at determining the qualitatively desirable features of recommender systems.
%\done{check above paragraph}


% \subsection{User Satisfaction}

% \subsection{Online Evaluation of Recommender Systems}

%\subsection{Online Evaluation}

% See some refs in http://proceedings.mlr.press/v81/ekstrand18b/ekstrand18b.pdf

%\section{Other Aggregation Functions and 
\section{Deep Learning for Recommendation}
\label{sec:deepRec}
%\section{Aggregation Functions for Recommender Systems}
%\label{sec:aggregation}
%\done{funny section titles here, just replace with "deep learning" or something. Done}

Increasingly, state-of-the-art recommendation models are based on deep learning approaches. In principle, the appeal of deep learning-based recommenders is that they can capture complex, non-linear relationships among users and items, beyond what is possible with the simple aggregation functions such as those in \Cref{eq:MFscore}. Later, deep learning-based approaches will allow us to 
uncover complex sequential patterns (\cref{chap:temporalSequential}),
or incorporate complex features from text (\cref{chap:text}) and images (\cref{chap:visual}).
%incorporate complex features from text (\cref{chap:text}) and images (\cref{chap:visual}), and to uncover complex sequential patterns (\cref{chap:temporalSequential}). 
For the moment, we explore a few of the main approaches to model interaction data in `traditional' settings, though revisit deep learning-based models repeatedly in future chapters.

%\done{Intro paragraph}

\subsection{Why the Inner Product?}
\label{sec:whyInner}
%\done{drop? Keep}

To motivate the potential of deep learning for recommendation, it is worth briefly revisiting our specific choice of the objective in \Cref{subsec:matrixFactor}, in which we computed compatibility between users and items via an inner product, i.e.,
\begin{equation}
    \text{Compatibility}(u,i) = \gamma_u \cdot \gamma_i.
    \label{eq:innerSim1}
\end{equation}
This seemed a reasonable enough choice, and was motivated by a connection to matrix factorization and the Singular Value Decomposition.
%
%So far, most of the model-based recommenders
%Briefly, it is worth mentioning that the type of recommendation objective in \Cref{subsec:matrixFactor} in which we measure compatibility via an inner product is only one potential choice of compatibility relationship.
However it should be carefully noted that this is only one choice of compatibility function, and is by no means sacred. For instance, consider measuring compatibility between representations via a (squared) distance function:
\begin{equation}
    \text{Compatibility}(u,i) = \| \gamma_u - \gamma_i \|_2^2.
    \label{eq:squaredSim1}
\end{equation}


\Cref{fig:InnerNNSim} shows recommendations that might be generated under these two compatibility conditions. Conceptually, these recommendations have quite different semantics: roughly, inner product-based compatibility (\cref{eq:innerSim1}) implies that a user who likes action movies should be recommended movies with \emph{a lot} of action, whereas a distance-based compatibility suggests that users who like action movies should be recommended other movies with a \emph{similar amount} of action.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/IPSim.pdf}
    \includegraphics[scale=0.75]{fig/NNSim.pdf}
    \caption[Highly compatible items in latent space.]{User vectors $\gamma_u$ in latent space, and candidate items to be recommended. Highly compatible items appear in the highlighted region under inner product (left) and Euclidean (right) compatibility models. \label{fig:InnerNNSim}}
    %\done{not referenced}
\end{figure} 

Neither compatibility function is `better,' and either could be preferable under certain conditions. Ideally, deep learning-based recommenders could give models the flexibility to determine the right compatibility functions for a particular scenario.


%\done{Move elsewhere and reference (probably previous chapter even)}

%\done{could cut the text, *very* long intro. Is fine}

% While measuring compatibility between a user $u$ and item $i$ in terms of $\gamma_u \cdot \gamma_i$ arises naturally when deriving solutions in terms of matrix factorization

% \emph{Neural Collaborative Filtering} \cite{he2017neural} argued that deep learning approaches can be used to learn 
%\done{confirm this is actually what the paper says. Removed}

%\done{Define "collaborative filtering" on its first use?}

% Could discuss deep learning papers: e.g. the two YouTube papers

%\done{Rename section probably}

% \section{Deep Learning for Recommendation}


% Some from
% \cite{zhang2019deep}

\cite{zhang2019deep} discuss various potential benefits and limitations of deep learning-based recommender systems. Arguably, the main benefit of deep learning-based is the ability to uncover complex, non-linear relationships between user and item representations. For example, %previously 
we'll later study various settings
(e.g.~\Cref{sec:PRME})
%\done{was it previously?}
%we discussed the relative benefits of the inner-product versus a Euclidean metric when comparing user and item representations. 
where the Euclidean distance may be preferable to the inner product when comparing  representations.
%\footnote{
%We'll see specific examples where distance-based compatibilities are preferable in \Cref{sec_?}, and explore consequences of these choices in \Cref{chap:consequence}.}
%\done{Do I ever have that example where I explain how inner product versus Euclidean metrics differ? Could do while referencing figure}
In principle, deep learning approaches could learn more flexible aggregation functions aggregation functions, reducing the need for manual engineering.

\cite{zhang2019deep} 
%also point 
suggest
other appealing properties of deep learning-based recommenders, including the effectiveness of deep learning when dealing with structured data such as sequences, images, or text, and the ubiquity of high-level libraries that facilitate straightforward implementation. 
%\done{Depending on organization, may have already mentioned libraries. Don't think so, looks fine}
We revisit these topics throughout the book.

%Discuss that recent paper by the BPR author
Conversely, there 
%has been 
is
some question as 
%to the extent to which deep learning-based recommender systems are capable of delivering their promised value, 
to whether deep learning-based recommender systems over-promise in terms of their perceived value,
or whether `traditional' recommender systems might still deliver better results if carefully tuned. We discuss these questions in \Cref{sec:howEffective}.
%\cite{rendle2020neural}

%\begin{description}
\subsection{Multilayer Perceptron-based Recommendation}
\label{subsec:MLPRec}
%\done{Could move to "basics" chapter, but maybe not. No}


%\done{Write short captions for all figures}

%\done{should figure captions, and short captions end in a full stop? Yes for now}


Multilayer perceptrons (MLPs) are a staple of artificial neural networks, offering a straightforward way to learn non-linear transformations and interactions among features.

Roughly speaking, a `layer' of a multilayer perceptron transforms a vector of input variables to a (possibly lower dimensional) vector of output variables; 
%via a linear function of the input 
typically the output variables are related to the input variables via a linear transformation followed by a non-linear activation, e.g.:
\begin{equation}
    f(x) = \sigma(Mx).
\end{equation}
%where $\sigma$ is applied elementwise.
Here $x$ is a vector of input variables, $f(x)$ is a vector of output variables, and $M$ is a learned matrix, such that each term in $Mx$ is a weighted combination of the original features in $x$. The sigmoid function is applied elementwise, in this case transforming the output variables to lie in the range $(0,1)$.

While the above is just one \emph{layer} of a multilayer perceptron, several such layers can be `stacked' in order for the network to learn complex non-linear functions.
%\done{(nothing else to do?)}
%
Eventually, the final layer predicts some desired output, e.g.~a regression or classification objective. For example, the final layer might simply take a weighted combination of features from the previous layer:
\begin{equation}
    f(x) = \sigma(\theta \cdot x),
\end{equation}
i.e.,~similar to the output of a logistic regressor (for a classification task).

We depict a multilayer perceptron in \Cref{fig:multilayer-perceptron}. Note that a trivial linear model of the form $y = X\theta$
%\done{notation}
would be depicted by a similar figure in which the inputs were connected directly to the output.

%https://github.com/davidstutz/latex-resources/blob/master/tikz-multilayer-perceptron/multilayer-perceptron.tex
\tikzset{>=To}

\begin{figure}[t]
	\centering
	\begin{tikzpicture}[shorten >=1pt]
		\tikzstyle{unit}=[draw,shape=circle,minimum size=0.8cm]
% 		\tikzstyle{hidden}=[draw,shape=circle,fill=black!25,minimum size=1.15cm]
		\tikzstyle{hidden}=[draw,shape=circle,minimum size=0.8cm]

		\node[unit](x0) at (0,3.5){$x_0$};
		\node[unit](x1) at (0,2){$x_1$};
		\node at (0,1.125){\vdots};
		\node[unit](xd) at (0,0){$x_F$};

		\node[hidden](h10) at (2,4){$h_0^{(1)}$};
		\node[hidden](h11) at (2,2.5){$h_1^{(1)}$};
		\node at (2,1.125){\vdots};
		\node[hidden](h1m) at (2,-0.5){$h_{m}^{(1)}$};

		\node(h22) at (3.5,0){};
		\node(h21) at (3.5,2){};
		\node(h20) at (3.5,3.75){};
		
		\node(d3) at (4,0){$\ldots$};
		\node(d2) at (4,2){$\ldots$};
		\node(d1) at (4,3.75){$\ldots$};

		\node(hL12) at (4.5,0){};
		\node(hL11) at (4.5,2){};
		\node(hL10) at (4.5,3.75){};
		
		\node[hidden](hL0) at (6,4){$h_0^{(L)}$};
		\node[hidden](hL1) at (6,2.5){$h_1^{(L)}$};
		\node at (6,1.125){\vdots};
		\node[hidden](hLm) at (6,-0.5){$h_{m}^{(L)}$};

% 		\node[unit](y1) at (12,3.5){$y_1^{(L+1)}$};
		\node[unit](y2) at (8,1.75){$y$};
% 		\node at (12,1){\vdots};	
% 		\node[unit](yc) at (12,0){$y_C^{(L+1)}$};

        \draw[->] (x0) -- (h10);
		\draw[->] (x0) -- (h11);
		\draw[->] (x0) -- (h1m);

        \draw[->] (x1) -- (h10);
		\draw[->] (x1) -- (h11);
		\draw[->] (x1) -- (h1m);

        \draw[->] (xd) -- (h10);
		\draw[->] (xd) -- (h11);
		\draw[->] (xd) -- (h1m);

% 		\draw[->] (hL0) -- (y1);
% 		\draw[->] (hL0) -- (yc);
		\draw[->] (hL0) -- (y2);

% 		\draw[->] (hL1) -- (y1);
% 		\draw[->] (hL1) -- (yc);
		\draw[->] (hL1) -- (y2);

% 		\draw[->] (hLm) -- (y1);
		\draw[->] (hLm) -- (y2);
% 		\draw[->] (hLm) -- (yc);

        \draw[-,path fading=east] (h10) -- (h20);
		\draw[-,path fading=east] (h10) -- (h21);
		\draw[-,path fading=east] (h10) -- (h22);
		
		\draw[-,path fading=east] (h11) -- (h20);
		\draw[-,path fading=east] (h11) -- (h21);
		\draw[-,path fading=east] (h11) -- (h22);
		
		\draw[-,path fading=east] (h1m) -- (h20);
		\draw[-,path fading=east] (h1m) -- (h21);
		\draw[-,path fading=east] (h1m) -- (h22);
		
		\draw[->,path fading=west] (hL10) -- (hL0);
		\draw[->,path fading=west] (hL11) -- (hL0);
		\draw[->,path fading=west] (hL12) -- (hL0);
		
		\draw[->,path fading=west] (hL10) -- (hL1);
		\draw[->,path fading=west] (hL11) -- (hL1);
		\draw[->,path fading=west] (hL12) -- (hL1);
		
		\draw[->,path fading=west] (hL10) -- (hLm);
		\draw[->,path fading=west] (hL11) -- (hLm);
		\draw[->,path fading=west] (hL12) -- (hLm);
		
		\draw [yshift=0pt] (x0) node [yshift=+0.7cm]{input layer};
		\draw [yshift=0pt] (h10) node [yshift=+0.7cm]{hidden layer};
		\draw [yshift=0pt] (hL0) node [yshift=+0.7cm]{hidden layer};
		\draw [yshift=0pt] (y2) node [yshift=+0.7cm]{output};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (2.5,4.5) -- (3.75,4.5) node [black,midway,yshift=+0.6cm]{$1^{\text{st}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (8.5,4.5) -- (9.75,4.5) node [black,midway,yshift=+0.6cm]{$L^{\text{th}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (11.5,4) -- (12.75,4) node [black,midway,yshift=+0.6cm]{output layer};
	\end{tikzpicture}
	\caption[Representation of a multilayer perceptron with $L$ layers.]{Representation of a multilayer perceptron with $L$ layers. Here for simplicity each layer has the same number ($m$) of units, and we have only a single output $y$ (so that the inputs and outputs are similar to the regression and classification problems we studied in \protect\Cref{chap:basicsReg,chap:classPipe}).}
	\label{fig:multilayer-perceptron}
\end{figure}

Ultimately, multilayer perceptrons handle similar modalities of data and problems to those we saw in \Cref{chap:basicsReg,chap:classPipe}, i.e.,~feature vectors as inputs and regression or classification targets as outputs; the main difference compared to 
%the models from \Cref{chap:basics} 
our earlier models
is simply their ability to learn complex non-linear transformations and relations among features.
%\done{multilayer or multi-layer. Multilayer}

% \paragraph{Neural Collaborative Filtering}
\subsubsection{Neural Collaborative Filtering}
    %\done{I think}
    %Approaches include neural collaborative filtering
\cite{he2017neural} attempted to apply the benefits of multilayer perceptrons to latent factor recommender systems. The essential idea is fairly straightforward: rather than combining user and item latent factors via an inner product (as in \cref{eq:MFscore}), $\gamma_u$ and $\gamma_i$ are combined via a multilayer perceptron to predict the model output (note that both the latent factors and the MLP parameters are learned simultaneously). As we discussed in \Cref{sec:whyInner}
%\done{check that this exists and comes before this discussion}
the inner product function is only one possible choice when combining user and item preferences and other choices (such as a Euclidean distance) may be more appropriate in other settings; conceptually, the promise of a solution based on a multilayer perceptron is that one can be agnostic to these choices with the expectation that the model will learn the correct aggregation function automatically. While \cite{he2017neural} showed this method to be effective in some settings, there has recently been some question as to the value of this type of technique: while MLPs can in principle learn quite general functions, in practice specific functions (like the inner product) are not easily recoverable by such models, meaning that simpler models may still often outperform these more complex approaches. We discuss this issue further in \Cref{sec:howEffective}.
%\done{include figure for architecture? No}
%\done{also include deepFM figure, see. No} 
%https://d2l.ai/chapter_recommender-systems/deepfm.html



    
\subsection{Autoencoder-based Recommendation}

%\done{add. (But didn't say much)}:
%Auto-Encoders for Collaborative Filtering (MVAE) \cite{mvae}:} State-of-the-art recommendation algorithm operating on implicit/sequential feedback data that builds upon the Variational Auto-Encoder (VAE) \cite{vae} 
%Sequential Variational Auto-Encoders for Collaborative Filtering (SVAE)

Roughly speaking, the role of an \emph{autoencoder} is to learn a low-dimensional representation of some input data that preserves the ability to reconstruct the original (high-dimensional) data from the low-dimensional representation.
%\done{cite some examples of autoencoders. Nah}
%\cite{zhang2019deep} suggest 
%to 
%\done{This sentence. Below}
%common ways in which this principle can be applied in the context of recommendation: first, 
%AutoRec: \cite{sedhain2015autorec}
The basic principle of an autoencoder is depicted in \Cref{fig:autoencoder}. Here an input vector $x$ is projected into a lower-dimensional space via a function $g(x)$ (following an approach similar to that followed by a multilayer perceptron above, which may include several layers). The low-dimensional representation is then mapped back into the original space via $f(g(x))$; the goal is that $f(g(x))$ should match the original data $x$ as closely as possible. In this way $g(x)$ acts as a `bottleneck,' forcing the model to learn a compressed representation that succinctly captures the meaningful information in $x$. Several variants of autoencoders exist, for instance \emph{denoising autoencoders} partially corrupt the input in order to learn representations that are robust to noise; \emph{sparse autoencoders} attempt to learn compressed representations that are sparse, etc.

\begin{SCfigure}
	\centering
	\begin{tikzpicture}[shorten >=1pt]
		\tikzstyle{unit}=[draw,shape=circle,minimum size=0.8cm]
% 		\tikzstyle{hidden}=[draw,shape=circle,fill=black!25,minimum size=1.15cm]
		\tikzstyle{hidden}=[draw,shape=circle,minimum size=0.8cm]

		\node[hidden](h10) at (2,4){$x_0$};
		\node[hidden](h11) at (2,2.5){$x_1$};
		\node at (2,1.125){\vdots};
		%\done{Fix position of vdots in other figure too}
		\node[hidden](h1m) at (2,-0.5){$x_n$};

% 		\node(h22) at (3.5,0){};
% 		\node(h21) at (3.5,2){};
% 		\node(h20) at (3.5,3.75){};
		
		\node[hidden](d3) at (4,0){$g_k$};
		\node[hidden](d2) at (4,2){$g_1$};
		\node[hidden](d1) at (4,3.75){$g_0$};
		%\done{Why don't the arrows go all the way to the thing? Copy code from other figure to fix}

% 		\node(hL12) at (4.5,0){};
% 		\node(hL11) at (4.5,2){};
% 		\node(hL10) at (4.5,3.75){};
		
		\node[hidden](hL0) at (6,4){$f_0$};
		\node[hidden](hL1) at (6,2.5){$f_1$};
		\node at (6,1.125){\vdots};
		\node[hidden](hLm) at (6,-0.5){$f_n$};

% 		\node[unit](y1) at (12,3.5){$y_1^{(L+1)}$};
% 		\node[unit](y2) at (8,1.75){$y$};
% 		\node at (12,1){\vdots};	
% 		\node[unit](yc) at (12,0){$y_C^{(L+1)}$};

% 		\draw[->] (hL0) -- (y1);
% 		\draw[->] (hL0) -- (yc);
% 		\draw[->] (hL0) -- (y2);

% 		\draw[->] (hL1) -- (y1);
% 		\draw[->] (hL1) -- (yc);
% 		\draw[->] (hL1) -- (y2);

% 		\draw[->] (hLm) -- (y1);
% 		\draw[->] (hLm) -- (y2);
% 		\draw[->] (hLm) -- (yc);

        \draw[->] (h10) -- (d1);
		\draw[->] (h10) -- (d2);
		\draw[->] (h10) -- (d3);
		
		\draw[->] (h11) -- (d1);
		\draw[->] (h11) -- (d2);
		\draw[->] (h11) -- (d3);
		
		\draw[->] (h1m) -- (d1);
		\draw[->] (h1m) -- (d2);
		\draw[->] (h1m) -- (d3);
		
		\draw[->] (d1) -- (hL0);
		\draw[->] (d2) -- (hL0);
		\draw[->] (d3) -- (hL0);
		
		\draw[->] (d1) -- (hL1);
		\draw[->] (d2) -- (hL1);
		\draw[->] (d3) -- (hL1);
		
		\draw[->] (d1) -- (hLm);
		\draw[->] (d2) -- (hLm);
		\draw[->] (d3) -- (hLm);
		
		\draw [yshift=0pt] (h10) node [yshift=+0.7cm]{input};
		\draw [yshift=0pt] (d1) node [yshift=+0.7cm]{encoded};
		\draw [yshift=0pt] (hL0) node [yshift=+0.7cm]{decoded};
% 		\draw [yshift=0pt] (y2) node [yshift=+0.7cm]{output};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (2.5,4.5) -- (3.75,4.5) node [black,midway,yshift=+0.6cm]{$1^{\text{st}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (8.5,4.5) -- (9.75,4.5) node [black,midway,yshift=+0.6cm]{$L^{\text{th}}$ hidden layer};
% 		\draw [decorate,decoration={brace,amplitude=10pt},xshift=-4pt,yshift=0pt] (11.5,4) -- (12.75,4) node [black,midway,yshift=+0.6cm]{output layer};
	\end{tikzpicture}
	\caption[Autoencoder representation.]{Autoencoder. $g_i$ and $f_i$ are shorthand for $g(x)_i$ and $f(g(x))_i$ respectively. Either of the encoding or decoding operations could include multiple layers.}
	\label{fig:autoencoder}
\end{SCfigure}

\cite{zhang2019deep,zhang2020survey} survey various ways that autoencoders can be used in the specific context of recommendation, including methods that use autoencoders as a component in complex recommendation frameworks, autoencoders that model sequential dynamics (as in \cref{chap:temporalSequential}, see e.g.~\cite{sachdeva2019sequential}), etc. Below we explore a single approach that is representative of the general setup. 



% \paragraph{AutoRec}
\subsubsection{AutoRec}

\cite{sedhain2015autorec} adapt the principle of autoencoders to recommendation problems. In their setting the data to be encoded is a vector of ratings for an item $i$, or equivalently a column of an interaction matrix $R_{\cdot,i}$. Since $R_{\cdot,i}$ is only partially observed (rather than a dense vector as in a traditional autoencoder), the compressed representation is only responsible for
(and gradient updates are only applied to)
the observed entries $R_{u,i}$. At inference time the compressed representation can then be used to estimate the entries for unobserved pairs. 
\cite{sedhain2015autorec} term this version of the model \emph{AutoRec-I}, since an autoencoder is used to learn compressed item representations; alternately, \emph{AutoRec-U} consists of the same approach applied to user vectors $R_{u,\cdot}$.

%\done{Add autorec-I to user-free table. It's in DL table}

Note that AutoRec-I lacks any user parameters (and likewise AutoRec-U lacks item parameters). As such it is a form of \emph{user-free} personalization (like those from \Cref{sec:userFree1}), that personalizes predictions using a model that aggregates data from the entire user history rather than `memorizing' user preferences via an explicit parameter. 
We further explore such notions of user-free personalization when exploring methods based on sequences 
%and text 
in \Cref{sec:rnnRec}.
%\done{make sure ordering is correct}

%\done{Could mention other autoencoder methods, not critical. No}

\subsection{Convolutional and Recurrent Networks}
\label{subsec:CNNRNN}

Finally, \cite{zhang2019deep} survey various recommender systems based on recurrent networks and convolutional neural networks. Recurrent networks are typically chosen as a way of exploring \emph{sequential} dynamics in user activities; we explore this type of approach in \Cref{chap:temporalSequential}, including deep learning based approaches in \Cref{sec:rnnRec}. Convolutional neural networks are often used as a means of incorporating representations of rich content (such as images) into recommender systems; we explore this type of approach in \Cref{chap:visual}.
% \end{description}

\cite{zhang2019deep} 
%\done{Not every paragraph should start with citation. It's fine}
also discuss potential limitations of deep learning-based approaches, including the challenges involved in interpreting the predictions of deep learning systems, and the difficulty of tuning hyperparameters in systems with many complex, interacting parts. They also highlight the potential lack of interpretability of deep learning-based models (though this is to some extent a challenge with any model based on latent representations), as well as the `data hungriness' of deep learning approaches. The latter issue arises whenever fitting models with a large number of parameters, and indeed is also a problem when fitting traditional latent factor models as in \Cref{subsec:fitLFM}. On the one hand, this may narrow the conditions under which deep learning-based approaches are effective, e.g.~they may underperform in cold-start situations (\cref{sec:coldStart}) where few interactions are available per user or per item. On the other hand, deep learning approaches may extend the modalities of data that can be incorporated into recommendation approaches (including in cold-start settings), for instance by leveraging text or image data; we explore such approaches in \Cref{chap:text} and \Cref{chap:visual}.

%\subsection{Neural Collaborative Filtering and Deep Factorization Machines}
%\done{Needs to move to next chapter after FMs are discussed}

%\subsection{Autoencoder Based Recommendation

We summarize the methods from this section in \Cref{chap:contentStructure} (\cref{tab:dlRecExamps}), after presenting additional deep learning-based recommenders that make use of content and structure.


\subsection{How Effective are Deep Learning-Based Recommenders?}
\label{sec:howEffective}

In spite of the proliferation of deep learning-based %recommendation approaches, 
approaches to recommendation,
their benefit over simpler, more traditional forms of recommendation is perhaps questionable.
\cite{dacrema2019we} conducted a thorough evaluation of several of the predominant deep learning-based recommender systems (including several of the approaches discussed above), and found that deep learning approaches were often outperformed by simpler methods, so long as those methods were carefully tuned. Most of the models for recommendation we've seen so far involve many tunable factors (e.g.~number of factors, regularization schemes, and details of the specific training approaches), as well as choices in terms of dataset selection, pre-processing, etc.~that can favor certain models over others. 
%
Although the evaluation in \cite{dacrema2019we} was limited to a few specific (but popular) approaches, it raised broader issues of evaluation and benchmarking in recommender systems.
%\done{could connect better to something actually in the above paper. Is fine}
Some general points raised include the difficulty in reproducing reported results (while releasing research code is common practice, releasing exact hyperparameter settings or tuning strategies is not), and the proliferation of datasets, metrics, and evaluation protocols that make fair comparison difficult.

\cite{rendle2020neural} explored the same issue, focusing on the comparison of inner product-based recommendation versus solutions based on multilayer perceptrons.
%\done{MLPs are defined somewhere above? Yes}
They reiterate the main point from \cite{dacrema2019we} that simpler methods remain competitive so long as they're carefully tuned. They also argue that in spite of the hope that multilayer perceptrons can learn complex, non-linear relationships, that in practice even simple functions (like the inner product) are difficult for such models to reproduce.

Finally, both \cite{dacrema2019we} and \cite{rendle2020neural} 
discuss issues of computational complexity, and whether the marginal benefits of deep learning-based approaches justify the substantial added complexity.
\cite{rendle2020neural} 
argue that simpler models may be preferable in production environments, especially when considering the efficiency of item retrieval (as we discuss below).
%\done{check whether it's still below. Yes}
%\done{above paragraphs are fine but basically just the abstract from that paper. Fine}

Note that the above criticisms are not an indictment of deep learning-based approaches in general, but only with respect to their ability to model specific types of interaction data (essentially the same settings discussed in this chapter). In later chapters we'll explore the use of deep learning-based approaches in a variety of other settings, in order to model sequence, text, and image data, with goals ranging from cold-start performance to interpretability.

\section{Retrieval}
\label{sec:retrieval}
%\done{Probably collapse these to a paragraph somewhere}

%\done{balance parens}

Briefly, it is worth discussing one of the fundamental considerations when deploying a recommender system, namely, how 
%can we 
to
efficiently retrieve items. Na\"ively, having defined a compatibility function $f(u,i)$
%\done{is this the right notation?}
between a user and an item (as in \cref{eq:fui}), our goal might be to rank (unseen) items according to their compatibility, i.e.,
\begin{equation}
    \mathit{rec}(u) = \argmax_{i \in I \setminus I_u} f(u,i).
    %\done{check notation}
    \label{eq:recsysRetrieve}
\end{equation}
Presumably, recommendations must be made rapidly, for use in interactive settings. Given a large vocabulary of items, this procedure is likely to be prohibitively expensive if we were to attempt to enumerate scores for all items $i\in I$;
%\done{notation}
as such it is worth thinking about what types of relevance functions $f(u,i)$ admit efficient solutions to \Cref{eq:recsysRetrieve}.\footnote{The setting of \protect\Cref{eq:recsysRetrieve} is admittedly oversimplified; computing such a ranking is likely one piece is a more complex pipeline.}

\paragraph{Euclidean distance}
Perhaps the most straightforward function for efficient retrieval is a Euclidean distance function, 
%as in \Cref{sec:aggregation?}, 
i.e.,
\begin{equation}
    f(u,i) = \| \gamma_u - \gamma_i \|.
\end{equation}
In this case, retrieval can be done efficiently (i.e.,~$O(\log(| I |)$ on average) using traditional data structures such as a KD-tree. A KD-tree \citep{bentley1975multidimensional} is a data structure that represents $K$-dimensional points (in this case $\gamma_i$ for each item) in such a way as to allow efficient retrieval given a query ($\gamma_u$); such data structures predate recommender systems and have classical applications in nearest-neighbor retrieval for classification (for example).

%X-box inner product
%https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/XboxInnerProduct.pdf

\paragraph{Inner product and cosine similarity}
\cite{bachrach2014speeding} showed that the same types of data structure can be adapted for other types of relevance function. Fundamentally, they showed that both inner product and cosine distance-based relevance functions can be related to nearest neighbor search (as above) via appropriate transformations:
\begin{align}
    \argmax_i \| \gamma_u - \gamma_i \| & \quad \text{nearest neighbor (NN)}\\
    \argmax_i \gamma_u \cdot \gamma_i & \quad \text{maximum inner product (MIP)}\\
    \argmax_i \frac{\gamma_u \cdot \gamma_i}{\| \gamma_u \| \| \gamma_i \|} & \quad \text{maximum cosine similarity (MCS)}.
\end{align}
%\done{Doesn't render properly}
%\done{dunno if need abbreviations. Fine}
Doing so allows the same data structures that facilitate nearest-neighbor
%(Euclidean) 
search to be used for recommenders based on inner products (as in \cref{subsec:matrixFactor}) or cosine similarity (as in \cref{sec:cosine}).

\paragraph{Approximate search and Jaccard similarity}
%Other data structures can also be used, for example
In practice, 
%approximation schemes may also be used for efficient retrieval, 
efficient retrieval may be accomplished via approximation schemes,
such as techniques based on locality-sensitive hashing (whereby `similar' items are hashed to the same bucket). Versions of locality-sensitive hashing 
%have been proposed 
can be used 
to retrieve similar items based on similarity functions including Euclidean distance \citep{indyk1998approximate}, 
Jaccard \citep{broder1997resemblance}, and
Cosine similarity \citep{charikar2002similarity}. 
\cite{bachrach2014speeding} compare exact techniques for retrieval (as described above) to these types of hashing-based approximations, as well as other exact techniques for recommendation \citep{koenigstein2012efficient}. Search techniques like those above are implemented in libraries such as \emph{FAISS}.\footnote{\url{https://github.com/facebookresearch/faiss}}

%Maximum inner product search

%Efficient nearest neighbor

%\subsection{Recommendation based on Euclidean Distance, Translation, Etc.}
%Covered in seequential recsys section



% \section{Recommending for Diversity}
%\done{Could put in ethics chapter or whatever}

\section{Online Updates}
\label{sec:onlineUpdates}

Our presentation of recommender systems has so far assumed  we have access to historical interaction data from which we can train a model to make predictions. That is, we've assumed we can train the model \emph{offline}. In practice, when deploying such a system, we may continually collect new interactions (as well as new items and new users). Given the complexity of model training, it is naturally impractical to retrain the model `from scratch' for every new interaction, item, or user. Although our focus in this book is not on model deployment, below we outline some of the general strategies for dealing with new interactions in an \emph{online} setting.

\paragraph{Regressing on $\boldsymbol{\gamma}_{\boldsymbol{u}}$ or $\boldsymbol{\gamma}_{\boldsymbol{i}}$}
Most straightforwardly, we can update \emph{some} parameters of a model without retraining the entire model. In particular, note that the model of \Cref{eq:matfac2nobias} (and its variants) is what is termed \emph{bilinear} in $\gamma_u$ and $\gamma_i$: if either of $\gamma_u$ or $\gamma_i$ is fixed (i.e.,~treated as a constant), fitting the remaining part of the model becomes equivalent to linear regression (and can be solved as in \Cref{eq:pseudoInv} or \Cref{eq:fitWithLambda}). Doing so allows us to take a fitted model and include new user or item terms based on a few observations (or likewise, to update $\gamma_u$ or $\gamma_i$ for existing users and items without updating the entire model). This specific approach only applies to a limited class of models for which individual parameters can be updated in closed form, though alternately one can use gradient-based approaches to update only a selection of model parameters.  

\paragraph{Cold-start and user-free models}
Other models are specifically designed to deal with new users and items. In \Cref{sec:coldStart} we explore models designed for cold-start scenarios (i.e.,~users and items with few or no associated interactions). Such models generally make use of features or side-information to compensate for a lack of historical observations.
A second class of models avoids modeling users altogether, and directly makes use of an interaction history at inference time (meaning such models can naturally adapt to user cold-start settings), including some of the approaches we explored in \Cref{sec:userFree1}.  
%We explore such models in \Cref{sec_?}.

\paragraph{Strategies for online training}
Finally, we mention schemes that are designed specifically to handle model updates in online settings.
%Online-Updating Regularized Kernel Matrix Factorization
%Models for Large-Scale Recommender Systems
%Largely, 
Such approaches generally follow the outline we described above, i.e.,~efficiently updating a subset of model parameters in the presence of an otherwise fully-trained model. See e.g.~\cite{rendle2008online}, which outlines efficient gradient descent-based update schemes under this setting.

\section{Recommender Systems in Python with \emph{Surprise} and \emph{Implicit}}

Although the types of recommender systems we've seen so far can (with some effort) be implemented `from scratch' either by computing the gradient expressions as in \Crefrange{eq:MFgradientFirst}{eq:MFgradientLast} or by using high-level optimization libraries like Tensorflow (we'll explore a Tensorflow implementation in \Cref{subsec:LFMTensorflow}),
%\done{capitalization}
the recommendation techniques we've covered so far are reasonably well-supported by popular Python
%\done{italics? No}
libraries.

Here we examine two specific libraries, \emph{Surprise} and \emph{Implicit} for latent factor recommendation (as in \cref{subsec:matrixFactor}) and Bayesian Personalized Ranking (as in \cref{subsec:BPR}. However these examples serve more to introduce the overall recommendation pipeline, rather than to dive deeply into the specifics of these libraries.

\subsection{Latent Factor Models (\emph{Surprise})}

%\done{latent factor vs. latent-factor}

\index{Surprise}
\emph{Surprise} \citep{Hug2020}
%\done{Modified citation, doesn't allow url}
is a library that implements various 
%collaborative filtering
recommendation
%\done{Do I ever use this word? No}
%methods and 
algorithms based on \emph{explicit feedback} (e.g.~ratings). Below we show how to use Surprise's
%\done{Formatting}
implementation of a latent factor model as in \Cref{eq:MFscore} (`SVD' as in \cite{koren2009matrix}).

First we import the model (`SVD'), and utilities to read and split the dataset:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from surprise import SVD, Reader, Dataset
from surprise.model_selection import train_test_split
\end{lstlisting}
\end{minipage}

\index{csv}
While the library has various routines to read 
%in the dataset, 
data,
the most straightforward is to read 
%the data 
from a \emph{csv}/\emph{tsv} file. Here we've processed the \emph{Goodreads} `fantasy' data
%\done{Do I talk about this dataset above? Yes}
to extract just the `user\_id', `book\_id', and `rating' fields, though this example could be applied to any similar dataset. After reading in the data, we split it into train and test fractions, with 25\% of the data withheld for testing:

%\done{lstlisting has backward quote character here}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
reader = Reader(line_format='user item rating', sep='\t')
data = Dataset.load_from_file('goodreads_fantasy.tsv', reader=reader)
dataTrain, dataTest = train_test_split(data, test_size=.25)
\end{lstlisting}
\end{minipage}

Next we fit the model, and collect its predictions on the test set:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
model = SVD()
model.fit(dataTrain)
predictions = model.test(dataTest)
\end{lstlisting}
\end{minipage}

From `predictions' we can then 
extract and compare the model's prediction (\texttt{p.est}) and the original value (\texttt{p.r\_ui}), in this case to compute the Mean Squared Error:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
sse = 0
for p in predictions:
    sse += (p.r_ui - p.est)**2

MSE = sse / len(predictions)
\end{lstlisting}
\end{minipage}

%print(sse / len(predictions))

%predictions[0].est

\subsection{Bayesian Personalized Ranking (\emph{Implicit})}

\index{Bayesian Personalized Ranking}
\emph{Implicit}\footnote{\url{https://github.com/benfred/implicit}} is a library for recommender systems that operate on implicit feedback datasets.
%, as in \Cref{sec:implicit}. 
Here we show how to use the library for Bayesian Personalized Ranking, 
%\done{capitalization of BPR? Always caps}
as in \Cref{subsec:BPR}.

First, we read in the data. This time, the required data format is a \emph{sparse} matrix describing all user/item interactions. Despite this matrix having hundreds of thousands of rows and columns, only observed interactions are stored:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from implicit import bpr

Xiu = scipy.sparse.lil_matrix((nItems, nUsers)) # Initialized after extracting the number of users and items
for d in data:
    itemsToUsers[itemIDs[d['book_id']],userIDs[d['user_id']]] = 1 # Only storing positive feedback instances
    
Xui = scipy.sparse.csr_matrix(Xiu.T)
\end{lstlisting}
\end{minipage}
%\done{Have I shown how to read the data? Could put here. Not a good place}
%\done{Sometimes call it 'data' and sometimes 'dataset', should be consistent. Nah}

%\done{Fix code: Xui is never referenced -- Yes it is}

Next, we initialize and fit the BPR model:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
model = bpr.BayesianPersonalizedRanking(factors = 5)
model.fit(Xiu)
\end{lstlisting}
\end{minipage}
%\done{Make the various examples have the same number of factors. Doesn't matter}

Having fit the model, we can retrieve the user and item factors $\gamma_u$ and $\gamma_i$, as well as recommendations (high $\gamma_u \cdot \gamma_i$) and similar items (high similarity to $\gamma_i$):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
itemFactors = model.item_factors
userFactors = model.user_factors

recommended = model.recommend(0, Xui) # Recommendations for the first user
related = model.similar_items(0) # Highly similar to the first item (cosine similarity)
\end{lstlisting}
\end{minipage}
%#

\subsection{Implementing a Latent Factor Model in Tensorflow}
\label{subsec:LFMTensorflow}
%\done{style}

Following our introduction to Tensorflow
%\done{style}
in \Cref{subsec:regressionTensorflow},
%\done{still has nonlinear components in that section? Looks fine}
it is now fairly straightforward to implement more complex models such as those developed in this chapter. 
Here we fit a latent factor model following \Cref{subsec:fitLFM}.
%\done{slightly messy paragraph. Fine}

We start by initializing our model, which takes as parameters the model dimensionality $K$ and the regularization strength $\lambda$.
%We start by defining 
Here we define
our variables to be fit ($\alpha$, $\beta_u$, $\beta_i$, $\gamma_u$, $\gamma_i$). In practice, appropriate initialization of such variables is important; here alpha is initialized to the mean rating $\mu$ while all other parameters are initialized following a normal distribution:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
class LatentFactorModel(tf.keras.Model):
    def __init__(self, mu, K, lamb):
        super(LatentFactorModel, self).__init__()
        self.alpha = tf.Variable(mu)
        self.betaU = tf.Variable(tf.random.normal([len(userIDs)],stddev=0.001))
        self.betaI = tf.Variable(tf.random.normal([len(itemIDs)],stddev=0.001))
        self.gammaU = tf.Variable(tf.random.normal([len(userIDs),K],stddev=0.001))
        self.gammaI = tf.Variable(tf.random.normal([len(itemIDs),K],stddev=0.001))
        self.lamb = lamb
\end{lstlisting}
\end{minipage}
%\done{ordering of init arguments in TF examples. Fine, lamb always last}
%\done{Need to update my earlier TF examples to be consistent with this one? Done}

Next we define our function (a method in the same class) that makes a prediction for a given user/item pair, i.e.,~$f(u,i) = \alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i$ as in \Cref{eq:MFscore}:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
    def predict(self, u, i):
        p = self.alpha + self.betaU[u] + self.betaI[i] +\
            tf.tensordot(self.gammaU[u], self.gammaI[i], 1)
        return p
\end{lstlisting}
\end{minipage}

Similarly we define our regularizer as in \Cref{eq:latentreg1} (which could easily be adapted to include different coefficients for different terms, for example):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
    def reg(self):
        return self.lamb * (tf.reduce_sum(self.betaU**2) +\
                            tf.reduce_sum(self.betaI**2) +\
                            tf.reduce_sum(self.gammaU**2) +\
                            tf.reduce_sum(self.gammaI**2))
\end{lstlisting}
\end{minipage}
%\done{columns aren't quite vertically aligned}

%\done{fixed parens above, make sure they're fixed in notebook too}

Finally we define the function to compute the squared error for a single sample, which will be called when computing gradients:
    
\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
    def call(self, u, i, r):
        return (self.predict(u,i) - r)**2
\end{lstlisting}
\end{minipage}
%
%model = LatentFactorModel(mu, 5, 0.00001)

%\done{Shrink code, fix word wrapping. Hard to make much better}
%\done{have code line numbers continue (everywhere)}


\subsection{Bayesian Personalized Ranking in Tensorflow}
\label{subsec:BPRTensorflow}
%\done{Change this and the above to be paragraphs rather than sections. Fine I think}

Bayesian Personalized Ranking (as in \cref{subsec:BPR}) can be implemented similarly. Again we initialize our model variables (this time only $\beta_i$, $\gamma_u$, and $\gamma_i$) are included:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
class BPR(tf.keras.Model):
    def __init__(self, K, lamb):
        super(BPR, self).__init__()
        self.betaI = tf.Variable(tf.random.normal([len(itemIDs)],stddev=0.001))
        self.gammaU = tf.Variable(tf.random.normal([len(userIDs),K],stddev=0.001))
        self.gammaI = tf.Variable(tf.random.normal([len(itemIDs),K],stddev=0.001))
        self.lamb = lamb
\end{lstlisting}
\end{minipage}
%\done{shrink. It's fine. Could make code a bit smaller}

Our prediction function estimates the unnormalized score
$x_{u,i} = \beta_i + \gamma_u \cdot \gamma_i$:
%\done{is this defined somewhere and is it called $x_{u,i}$ there? Yes}
        
\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
    def predict(self, u, i):
        p = self.betaI[i] + tf.tensordot(self.gammaU[u], self.gammaI[i], 1)
        return p
\end{lstlisting}
\end{minipage}

% \noindent
% \begin{minipage}{\linewidth}
% \begin{lstlisting}[escapechar=\%]
%     def reg(self):
%         return self.lamb * tf.reduce_sum(self.betaI**2) +\
%                           tf.reduce_sum(self.gammaU**2) +\
%                           tf.reduce_sum(self.gammaI**2)
% \end{lstlisting}
% \end{minipage}

Finally we define our loss for a single sample, this time including a user $u$, and items $i$ and $j$ that they did and did not interact with:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
    def call(self, u, i, j):
        return -tf.math.log(tf.math.sigmoid(self.predict(u,i) - self.predict(u,j)))
\end{lstlisting}
\end{minipage}

%model = BPR(5, 0.00001)

\subsection{Efficient Batch-Based Optimization}
\label{subsec:batchBased}

Although the above implementations are straightforward, they are not particularly efficient 
%and are memory 
%\done{not sure about memory, think about it}
%hungry 
if we attempt to compute the complete MSE (\cref{eq:MFObjective})
%\done{style. Fine}
or likelihood (\cref{eq:BPRobjective}) across the entire dataset. Instead, we compute gradients and update parameters in \emph{batches} consisting of a random sample of our data. 

First we generate our sample; for a BPR-like model this consists of three lists, corresponding to user, positive item, negative item triples $(u,i,j)$:
%\done{ugly text. Fine}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
sampleU, sampleI, sampleJ = [], [], []
for _ in range(Nsamples):
    u,i,_ = random.choice(interactions) # positive sample
    j = random.choice(items) # negative sample
    while j in itemsPerUser[u]:
        j = random.choice(items)
    sampleU.append(userIDs[u])
    sampleI.append(itemIDs[i])
    sampleJ.append(itemIDs[j])
\end{lstlisting}
\end{minipage}

Next we must redefine our score function to operate over a sample rather than a single data point. Note that rather than merely iterating over all points, estimates for all samples in our batch are computed using efficient vector operations:

%To do so we first redefine our score (and predict)
%\done{\texttt. Dunno what this was}
%functions to operate on a sample

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
    def score(self, sampleU, sampleI):
        u = tf.convert_to_tensor(sampleU, dtype=tf.int32)
        i = tf.convert_to_tensor(sampleI, dtype=tf.int32)
        beta_i = tf.nn.embedding_lookup(self.betaI, i)
        gamma_u = tf.nn.embedding_lookup(self.gammaU, u)
        gamma_i = tf.nn.embedding_lookup(self.gammaI, i)
        x_ui = beta_i + tf.reduce_sum(tf.multiply(gamma_u, gamma_i), 1)
        return x_ui
\end{lstlisting}
\end{minipage}

The `call' function is similarly modified:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
    def call(self, sampleU, sampleI, sampleJ):
        x_ui = self.score(sampleU, sampleI)
        x_uj = self.score(sampleU, sampleJ)
        return -tf.reduce_mean(tf.math.log(tf.math.sigmoid(x_ui - x_uj)))
\end{lstlisting}
\end{minipage}
%\done{could probably drop this part}
%\done{Replace with the updated code. Looks like the latest}

%\candrop{This part}

For the complete implementation (including various `boilerplate' components), see the online supplement (\cref{sec:onlineResources}). 

\section{Beyond a `Black-Box' View of Recommendation}

%\done{not sure if below argument should go here. Seems fine}
Finally, we should mention that our view of recommendation through the lens of machine learning represents only part of the study of recommender systems. For the most part, we have taken a `black box' view in which we regard the `recommender system' as merely a model that predicts user/item interactions (clicks, purchases, ratings, etc.)~as accurately as possible.

Although high-fidelity prediction is clearly necessary to build a successful recommender, it is only part of the picture. For example, we have not considered broader questions of what makes a recommender system `usable' or would ultimately drive user satisfaction or engagement. For example, if a user watches \emph{Harry Potter} should they be recommended its sequel, or another movie from the same genre? The former might maximize some na\"ive metric like click probability, whereas the latter is more likely to generate a suggestion that the user isn't already aware of. But either could be a legitimate goal of building a recommender system: helping a user quickly navigate a user interface by predicting their next interaction is just as important as recommending for novelty or discovery.

Such questions go beyond the black-box supervised learning view of recommendation: they are less questions about how to accurately predict the next action, and more about what we should do with that prediction. At the very least such questions require more nuanced evaluation metrics, if not user studies. While this book largely avoids discussion of user interface design, in \Cref{chap:consequence} we'll revisit the consequences of how recommender systems are applied, and look at strategies to improve personalized recommendation beyond simply optimizing prediction accuracy.


\section{History and Emerging Directions}
%\done{Probably drop from all sections. Not sure}

%Early approaches: memory-based to model-based, trend away from rating prediction to implicit feedback models

So far we have attempted to construct something of a narrative behind the development of recommender systems: we started with simple `memory-based' solutions 
%(\cref{sec:neighborhood}), 
(\cref{chap:recsysIntro}),
followed by `model-based' approaches such as latent factor models (\cref{chap:modelbasedRecsys}); later we argued about the benefits of leveraging implicit feedback (clicks, purchases, etc.)~rather than relying on ratings (\cref{subsec:BPR}); finally, we began to discuss emerging trends in neural network-based recommendation (\cref{sec:deepRec}).
%
While this narrative reflects current thinking on the topic, the 
%However the 
actual history of recommender systems is substantially more complicated;
for example one survey paper
\citep{burke2002hybrid}
%https://link.springer.com/content/pdf/10.1023/A:1021240730564.pdf
points out that even neural-network based recommender systems have been proposed since the early nineties
\citep{jennings1993user}.



To a large extent, research on recommender systems has been driven by the release and adoption of large-scale benchmark datasets. High-profile competitions such as the \emph{Netflix Prize} 
%(including a dataset of 100 million ratings) 
\citep{bennett2007netflix} have driven widespread interest in recommendation problems: the specific nature of the data (purely based on interactions with no side-information); the choice of metrics used (the Mean Squared Error);
%\done{style}
and the specific dynamics of the data itself (e.g.~the critical role of temporal dynamics), all show their influence in the models we've explored throughout this chapter. Likewise other datasets and competitions, including industrial datasets (e.g.~\emph{Yelp}, \emph{Criteo}) and 
academic projects
(e.g.~\emph{MovieLens} \citep{harper2015movielens})
have inspired models based on alternate settings and evaluation metrics. 

A constant theme in such research is the extent to which new models must be designed to adapt to the specific dynamics of new datasets. 
%which we revisit when exploring models based on temporal dynamics, text, and visual data in later chapters.
%\done{not great. Fine}
As we'll explore in upcoming chapters, research on recommender systems has sought to incorporate rich signals in the form of text, temporal and social signals, or images. Such factors serve not only to improve the predictive accuracy of recommendation models, but can also help to make recommendation models more interpretable, and to deal with modalities of data not supported by traditional recommendation approaches.
%Given the variety of textual, temporal, and visual signals, 
%Content rich approaches (extend above para)
We revisit such \emph{content aware} approaches throughout the remainder of this book, as we begin to develop techniques that make use of social (\cref{chap:contentStructure}), temporal (\cref{chap:temporalSequential}), textual (\cref{chap:text}), and visual (\cref{chap:visual}) signals.

Methodologically, recent research on recommender systems has been dominated by deep learning-based approaches, as we discussed a little in \Cref{sec:deepRec}.
%Development of DL-based approaches
%
Besides models based on multilayer perceptrons, convolutional neural networks, or autoencoders, a major trend has been to incorporate ideas from natural language processing. Roughly speaking, models of natural language are concerned with modeling the semantics of sequences of discrete tokens (i.e.,~words or characters), and thus translate naturally to recommendation problems involving sequences of interactions over a discrete set of items. Recommender systems based on natural language models (e.g.~\emph{Self-Attention},  \emph{Transformer}, \emph{BERT}, etc.)~arguably represent the current state-of-the-art \citep{kang2018self,sun2019bert4rec}. We'll explore this relationship in \Cref{chap:temporalSequential,chap:text} when developing general-purpose models of sequences and text.
%Development of models from natural language and recent directions

%Images, sets, text
The study of complex recommender systems that make use of data from images, text, and other forms of structured side-information will dominate our discussion for the next several chapters. 
Partly these complex forms of side-information allow us to build increasingly more accurate recommender systems that leverage complex signals (\cref{chap:contentStructure}). They also facilitate novel types of recommendation such as generating sets of compatible items (\cref{chap:visual}), or recommender systems with natural-language interfaces (\cref{chap:text}).
%As we'll argue, the 
We'll also argue that \emph{personalization} in such domains goes well beyond recommendation, playing a role in several settings where differences among individuals explains significant variance in data.
%\done{pretty redundant (compare to two paragraphs above). Is okay}

%ethics

%A more detailed history of recommender systems is provided in several survey papers
Several survey papers present more detailed histories of recommender systems.
%Summary of existing surveys
\cite{konstan1998recommender} discuss early research from the \emph{GroupLens} project, several of whose papers and datasets 
%we've already discussed; 
we discuss throughout this book.
%\done{already? changed}
Their survey gives an interesting early perspective on recommender systems, with their focus mainly centering around memory-based methods (as in \cref{sec:neighborhood}), but also discussing broader topics such as user interfaces and benchmarking beyond accuracy.
%
%
%https://www.aaai.org/Papers/Workshops/1998/WS-98-08/WS98-08-016.pdf
%
\cite{burke2002hybrid} discuss \emph{hybrid} recommender systems---systems which combine multiple types of recommendation, feature representation, or knowledge extraction approaches. Their survey focuses on the setting of restaurant recommendation, though broadly serves as an excellent introduction to a breadth of recommendation techniques, their trade-offs, and how they can be effectively combined.
More recent surveys include
\cite{bobadilla2013recommender},
%http://romisatriawahono.net/lecture/rm/survey/information%20retrieval/Bobadilla%20-%20Recommender%20Systems%20-%202013.pdf
which provides a high-level survey of many of the same techniques we've presented so far, as well as those we'll explore in later chapters such as socially-aware and content-based approaches; other surveys focus on specific collections of techniques such as deep-learning based recommendation
\citep{zhang2019deep}.

%\done{Not the most useful section}

%Development of large benchmark datasets, popular challenges. Movielens, Netflix, Criteo, Yelp, etc., a few more exotic choices

%and have often highli
%including recommender systems that make use of text, social networks, or images.

%Development of content-based approaches, images, text, additional datasets, etc.



% \section{Exercises}

\begin{exercises}

% \item Modify the Pearson similarity implementation from \Cref{sec_?} to implement the cosine similarity. Compare the results of the two implementations, as well as the Jaccard similarity, on?
% %\done{hastily written. Gone}
% %\done{Deleted}


% \item The Jaccard similarity-based recommender we implemented in \Cref{sec:jaccard} proved an effective recommender, though our implementation was inefficient. The main source of inefficiency was due to iteration over \emph{all} items. A more efficient implementation might first build smaller a \emph{candidate set} of items, by noting that only those items with at least one user in common with this the query could potentially have non-zero Jaccard similarity. This candidate set can be built by taking all users who have purchased the query $i$, and taking the union over \emph{other} items they have purchased (other than $i$), i.e.,~$\bigcup_{u \in U_i} I_u \setminus \lbrace i \rbrace$. After modifying our implementation from \Cref{sec:jaccard} to use this candidate set, confirm that it produces identical recommendations, and compare its running time to the na\"ive implementation.


% \item Implement rating prediction models based on the formulas in \Cref{eq:simPredict,eq:simPredict2,eq:simPredict3}. Compare the three in terms of their Mean Squared Error (using either the entire dataset or a random sample).
%\done{Which dataset? Deleted}

% -- Show that the Pearson correlation is related to the $R^2$ coefficient. (nOT INTERESTING)

%-- Some comparison of the various recommender systems error metrics, possibly following the rating prediction experiment above.
%\done{added}

\item 
%\done{Use emph like this at beginning of other exercises}
\emph{All of the exercises in this section can be completed on any dataset involving users, items, and ratings.}
Before implementing the latent factor recommender system described in \Cref{eq:MFscore}, it is instructive to implement simpler variants in order to understand the model-fitting procedure. Implement a \emph{bias-only} model, i.e.,~one that makes predictions according to $r(u,i) = \alpha + \beta_u + \beta_i$. This can be achieved either by computing derivatives for this simplified model (as we did in \Cref{subsec:fitLFM}), or more simply by discarding the latent factor terms from the Tensorflow
%\done{style}
code from \Cref{subsec:LFMTensorflow}. Implement this model and compare its performance (in terms of the MSE) to one which always predicts the average rating. Find the items with the largest values of $\beta_i$, and compare them to the items with the highest average ratings.\footnote{Consider why these two lists might not be the same: for example, a mediocre item which tends to be rated by `generous' users (high $\beta_u$) could have a high average rating but a low value of $\beta_i$.}
\label{ex:RS1}

% \item In practice the model in Exercise~\ref{ex:RS1}
% does little more than measure the overall quality of items (through $\beta_i$) and rating biases of users ($\beta_u$), i.e.,~its predictions are not personalized. Note however that the items with the largest values of $\beta_i$ may not be those items with the highest average ratings.\footnote{Consider why this should be the case. For example, a mediocre item which tends to be rated by `generous' users (high $\beta_u$) could have a high average rating but a low value of $\beta_i$.} Using the model you trained in Exercise~\ref{ex:RS1}, 



\item Implement a complete latent factor model, either by computing derivatives for all terms ($\frac{\partial \text{obj}}{\partial \gamma_{i,k}}$, $\frac{\partial \text{obj}}{\partial \gamma_{u,k}}$, etc.)~in the objective from \Cref{eq:MFObjective}, or by following the Tensorflow implementation. For your model to outperform the bias-only model from Exercise~\ref{ex:RS1}, you'll need to carefully experiment with the number of latent dimensions $K$, initialization strategies, and regularization.\footnote{When debugging gradient-descent models, it can be instructive to isolate individual terms (i.e.,~updating only a single parameter or a subset of parameters at a time) to determine that each update results in an improvement of the objective; it can also be useful to start with only a \emph{single} latent factor (i.e.,~$K=1$) before experimenting with higher-dimensional models.}
% $\frac{\partial \text{obj}}{\partial \beta_i}$ and $\frac{\partial \text{obj}}{\partial \gamma_{i,k}}$ 
%\done{exercise too similar to one above}
\label{ex:RSLFM}

\item 
%(Hard) 
%In \Cref{sec:latentfactor} 
In Exercise~\ref{ex:RSLFM}
we predicted star ratings using a model that optimized the Mean Squared Error. However the ratings we are predicting in many datasets are \emph{integer} valued, e.g.~$r_{u,i} \in \lbrace 1,2,3,4,5 \rbrace$. In light of this, it might be tempting to round the predictions of our model to the nearest integer. 
Surprisingly, this type of rounding is generally not effective, and results in a \emph{higher} MSE compared to non-rounded values. Explain why this might be the case (e.g.~by constructing a simple counterexample), and consider whether other rounding strategies might be more effective (e.g.~rounding ratings above 5 or below 1).
%In this exercise, we'll show that this type of rounding is not effective, i.e.,~that the rounded predictions will tend to have higher MSEs. For this exercise, suppose that when your model outputs a prediction number $a$, the correct answer is $\lfloor a \rfloor$ with
%probability $\lceil a \rceil - a$, and $\lceil a \rceil$ with probability $a - \lfloor a \rfloor$ (e.g.~if it outputs 4.2 then the correct answer is 4 80\% of the time and 5 20\% of the time).
%\done{Rewrite above, copied from class}


%\item Implicit-feedback such as BPR (\cref{sec?}) are known as \emph{pairwise} models because they consider a contrastive error between positive and negative samples. Before implementing BPR in Exercise~\ref{ex:RSLast}, implement a \emph{pointwise} model which uses a `standard' latent factor model (as in Exercise~\ref{ex:RSLast}), and simply labels positive interactions 
%\candrop{this}

\item Implement Bayesian Personalized Ranking (starting from the code in \Cref{subsec:BPRTensorflow} or otherwise), and compare this method to simpler approaches based on item-to-item or user-to-user compatibility such as those we studied in \Cref{chap:recsysIntro} (e.g.~recommend items with high Jaccard similarity compared to those the user has recently consumed). In doing so, consider several of the evaluation metrics from \Cref{sec:evaluateRec}, including the AUC, Recall, Mean Reciprocal Rank, etc.
\label{ex:RSLast}
%\done{what dataset?}
\end{exercises}

\refstepcounter{projectCounter}
\label{proj:2}
\section*{Project \theprojectCounter: A Recommender System for Books (Part 2)}
\addcontentsline{toc}{section}{Project \theprojectCounter: A Recommender System for Books (Part 2)}
%\done{project title. Done}

%In this project we'll build recommender systems to make predictions related to book reviews from \emph{Goodreads} (much as we used in \Cref{chap:basicsReg}).
%MOVED TO PREVIOUS CHAPTER
%\emph{Goodreads} data. Write introduction. Which dataset, how to split, etc.
%\done{above}

Here we'll extend our work from Project~\ref{proj:1111} to incorporate and compare model-based recommendation techniques. 


%\done{are projects normally enum or item? Most have an enum}
\begin{enumerate}
    \item 
    Start by comparing model-based approaches to the similarity-based recommenders you developed in Project~\ref{proj:1111}. Start by comparing rating prediction approaches (e.g.~models like those of \Crefrange{eq:simPredict}{eq:simPredict3}) to latent-factor modeling approaches (as in \cref{subsec:matrixFactor}).
    It can be useful to develop your model in several stages: e.g.~starting with
    %followed by a model-based approaches as in \Cref{chap:modelbasedRecsys}. Consider at least the following baselines: (a) the approach of \Cref{eq:simPredict3} (or its variants); (b) the same approach, interchanging users and items; (c) 
    (a)
    a model
    %-based approach using 
    including
    only an offset $f(u,i) = \alpha$; (b) using only an offset and biases $f(u,i) = \alpha + \beta_u + \beta_i$; (c) using latent factors $f(u,i) = \alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i$.
    %There are several recommendation tasks that you might consider on such a dataset. For example, you could predict what book a user would read next,
    
    \item Next, compare \emph{implicit feedback} models, such as the Bayesian Personalized Ranking model from \Cref{subsec:BPR}. Much as we measured the performance of BPR in terms of evaluation metrics like the AUC (\cref{eq:bprOpt1}), simple memory-based ranking schemes such as those we developed in Project~\ref{proj:1111} can also be evaluated based on how effectively they distinguish interactions (positive samples) from non-interactions.
    
    \item Try to thoroughly tune and regularize the latent factor model you developed above. Some factors you might consider include (a) the number of latent factors $K$; (b) the regularization approach, for example you might be able to improve performance by using separate regularizers for the bias terms and latent factors, i.e.,~$\lambda_1 \Omega(\beta)$ and $\lambda_2 \Omega(\gamma)$; (c) other factors, such as learning rates, initialization schemes, etc.
    \label{project2part2}
    \item Experiment with fast retrieval techniques (or libraries) such as those we examined in \Cref{sec:retrieval}.
\end{enumerate}

%\done{ugly orphan on last page of chapter. Looks alright now}



\chapter{Content and Structure in Recommender Systems}
\label{chap:contentStructure}
%\done{Could fold into above chapter, depending on scale. Much too long.}

So far, the systems we've built for personalized recommendation have been based purely on interaction data. 
We argued in 
%\Cref{sec_?} 
%When developing recommender systems in 
\Cref{chap:recsysIntro,chap:modelbasedRecsys}
as to why interactions are often sufficient to capture all of the critical signals that we need,
simply by finding patterns among users and items that maximally explain variance.
%However, our argument only applies to 
This argument holds in theory under certain conditions, though is quite limited. For one, collecting sufficient interaction data to fit parameter-hungry latent factor models is not feasible when we consider the long-tail of new users and rarely-consumed items. Even when we can harvest sufficient interaction data, several recommendation settings simply don't conform to the canonical setting of predicting an interaction given a user item pair.

In practice, several situations deviate from the classical setting we've described so far, and require more complex models that leverage side-information or problem structure to improve performance. Leveraging content and structure can be useful in a variety of situations, for example:
%\done{doesn't connect well. And slow intro. Is fine}
%
\begin{itemize}
    \item Only a limited amount of interaction data may be available. Our argument that interaction data is sufficient to capture subtle preference signals applies only in the limit, i.e.,~when a large number of interactions are available for each user (or item). When few interactions are available (or none, in \emph{cold-start} settings), one must instead rely on user or item \emph{features} to estimate initial preference models.
    \item Beyond improving performance, incorporating features into recommender systems may be desirable for the sake of model interpretability. For example, we may wish to understand how a user will react to a change in price; doing so effectively may require that price features are appropriately `baked in' to the model (we study this specific case in \Cref{sec:priceRecSys}).
    \item User preferences or item properties may not be \emph{stationary}. Even the simple fact that Christmas movies are unlikely to be watched in July cannot be captured simply by adding more latent dimensions. Although not the topic of this chapter, we'll revisit models of such temporal and sequential dynamics in \Cref{chap:temporalSequential}.
    \item Many settings simply do not follow the setup we developed in \Cref{chap:recsysIntro,chap:modelbasedRecsys}. For example, many recommendation scenarios have a social component (dating, bartering, etc.),~or other constraints that must be accounted for in addition to user-to-item compatibility.
\end{itemize}
%\done{not the best examples above. They're fine}

In this chapter we'll develop models that help us to adapt to the situations above. First, we'll explore general-purpose strategies to incorporate \emph{content} (or simply \emph{features}) into recommender systems, starting with factorization machines in \Cref{sec:factorization}. For the most part, our goal is to study strategies to incorporate simple numerical and categorical features; we develop strategies for the specific cases of temporal, textual, and visual features in \Crefrange{chap:temporalSequential}{chap:visual}.
We are especially interested in how features can be incorporated for the sake of solving so-called \emph{cold-start} problems (\cref{sec:coldStart}), whereby we have little (or no)
%\done{check if I've already said the above recently}
data associated with new users or items, as must infer an initial model of their preferences.

%In this chapter 
Beyond incorporating features into recommender systems,
we'll also explore various modalities of recommendation that 
%don't 
%exactly 
%follow 
deviate from
the basic setup in \Cref{chap:modelbasedRecsys}. We'll explore examples including online dating (\cref{subsec:onlineDating}), bartering (\cref{subsec:bartering}), social and group recommendation (\cref{sec:groupSocialRec}),  among others. 
%\done{update list. Looks okay (though not complete)}
In exploring such settings, our goal is not only to explore a few specific applications of interest, but more importantly to understand the overall process of designing and adapting personalized machine learning techniques for situations that exhibit additional structure, or otherwise don't perfectly align with traditional settings.


\section{The Factorization Machine}
\label{sec:factorization}

Factorization Machines \citep{rendle2010factorization} are a general-purpose approach that seeks to incorporate 
%general 
features into models that capture pairwise interactions (such as interactions between users and items).

In essence, the factorization machine extends the approach behind the latent factor model (\cref{subsec:matrixFactor}). The latent factor model embeds users and items into low dimensional space via $\gamma_u$ and $\gamma_i$, and then models the interaction between them via an inner product; the factorization machine extends this approach to incorporate arbitrary pairwise interactions between users, items, and other features.

The input to the model is a feature matrix $X$ and a target $y$. In the simplest case, $X$ might simply encode the identity of the user and item via a one-hot encoding, though can be extended to incorporate any additional properties associated with the interaction:
\begin{equation}
\left[
    \begin{tabular}{c}
    1000000 $\ldots$ 000100000 $\ldots$ 0001000 $\ldots$ 15.95\\
    0001000 $\ldots$ 000000010 $\ldots$ 0001000 $\ldots$ 12.25\\
    0100000 $\ldots$ 000100000 $\ldots$ 0000010 $\ldots$ 15.00\\
    0000100 $\ldots$ 010000000 $\ldots$ 0010000 $\ldots$ 17.50\\
    1000000 $\ldots$ 000000010 $\ldots$ 1000000 $\ldots$ 19.95\\
    $\smash{\underbrace{0000100}_{\clap{\vphantom{0y}\text{user}}}}\vphantom{0}$ $\ldots$ 
    $\smash{\underbrace{000010000}_{\clap{\vphantom{0y}\text{item}}}}\vphantom{0}$ $\ldots$
    $\smash{\underbrace{0000010}_{\clap{\vphantom{0y}\text{weekday}}}}\vphantom{0}$ $\ldots$
%    $\mathclap{\smash{\underbrace{\hphantom{10.15}}_{\mathrlap{\text{price}}}}}10.15$
    $\smash{\underbrace{10.15}_{\clap{\vphantom{0y}\text{price}}}}\vphantom{0}$
    \end{tabular}
\right]
%\done{Make above not ugly. looks okay, not great solution}
\label{eq:FMinteractionMatrix}
\end{equation}\vspace{1mm}

The basic idea of the factorization machine is then to model \emph{arbitrary} interactions between features. Each feature dimension is associated with a latent representation $\gamma_i$; the model equation is then defined in terms of all \emph{pairs} of (non-zero) features (with feature dimensionality $F$):
\begin{equation}
    f({x}) = \underbrace{w_0 + \vphantom{\sum_{j = i+1}}\sum_{i=1}^F w_i x_i}_{\text{offset and bias terms}} + \underbrace{\sum_{i = 1}^F \sum_{j = i+1}^F \langle \gamma_i, \gamma_j \rangle x_i x_j}_{\text{feature interactions}}
    %\done{Summation goes up to n (here and below), but I don't think I ever defined n?}
    \label{eq:FMequation}
\end{equation}

It is instructive to consider the case where the interaction matrix in \Cref{eq:FMinteractionMatrix} includes \emph{only} a user and item encoding. In such a case, \Cref{eq:FMequation} expands to be 
%identical 
equivalent
to a latent factor model (as in \cref{eq:MFscore}), i.e.,~the only interaction term is $\gamma_u \cdot \gamma_i$ for a user $u$ and item $i$. 

As such, the factorization machine can be viewed as a generalization of a latent factor model, that allows for additional types of interactions to be considered. For example, if we include an additional one-hot feature in \Cref{eq:FMinteractionMatrix} that encodes the \emph{previous} item the user consumed, then the factorization machine
%\done{capitalization?}
will include an expression encoding the compatibility of the next item with the previous one, i.e.,~the model can learn how \emph{contextually relevant} the previous item is compared to the next one. It will be useful to compare this approach to the models we design specifically to handle sequential inputs in \Cref{chap:temporalSequential} (\cref{sec:markovCaseStudies}). \cite{rendle2010factorization} discuss such topics, describing the extent to which the general-purpose factorization machine formulation subsumes various approaches designed to handle specific types of features.

\cite{rendle2010factorization} describe how the model equation of \Cref{eq:FMequation} can be computed efficiently (and how parameter learning can be done efficiently), by showing that the interaction term can be rewritten as
\begin{equation}
    \sum_{i = 1}^F \sum_{j = i + 1}^F \langle \gamma_i, \gamma_j \rangle x_i x_j = \frac{1}{2} \sum_{f = 1}^K \left( \left( \sum_{i = 1}^F \gamma_{i,f} x_i \right)^2 - \sum_{i = 1}^F \gamma_{i,f}^2 x_i^2 \right),
\end{equation}
which allows for computation 
%that is 
in
$O(KF)$ (the dimension of the latent factors multiplied by the feature dimensionality).




\subsection{Neural Factorization Machines}
%and deep factorization machines 
%nice figure: https://d2l.ai/chapter_recommender-systems/deepfm.html
%
As we saw in \Cref{chap:modelbasedRecsys} (\cref{sec:deepRec}), deep learning-based models can potentially be used to improve the performance of traditional recommender systems, essentially by learning complex non-linear relationships among latent features.
%an in particular, multilayer per
Likewise, \emph{Neural Factorization Machines} \citep{he2017neuralB} generalize factorization machines by using a
%use 
multilayer perceptron 
%to improve the performance of factorization machines. 
learn complex non-linear feature interactions.
The idea is similar to what we presented in \Cref{chap:modelbasedRecsys} (\cref{subsec:MLPRec}): just as user and item embeddings were combined by \cite{he2017neural} to develop neural collaborative filtering,
%(\cref{sec_?}), 
%above, 
%\cite{guo2017deepfm} 
\citet{he2017neuralB}
combine the embeddings of several terms (users, items, previous items, etc.).
%via a multilayer perceptron to develop deep factorization machines. 
The main additional component compared to neural collaborative filtering is a \emph{pooling} operation which aggregates the pairwise interactions among latent representations into a single feature vector, so that they can be passed to a multilayer perceptron.
%; the model is a strict generalization of factorization machines 
%In practice, this MLP is trained in parallel to a traditional factorization machine, and the outputs are combined in 
%\done{doesn't make sense since we haven't discussed FMs yet! Move to next section}

\paragraph{Wide and Deep learning for recommender systems}
%
%Wide \& Deep Learning for Recommender Systems
%Although more focused on the setting of feature-based recommendation (as we discuss in \Cref{chap:contentStructure}) 
The model architecture of the \emph{Wide \& Deep} model of \cite{cheng2016wide},
while not precisely a factorization machine,
is inspired by the setting of factorization machines, as well as 
%follows fairly naturally from 
the neural collaborative filtering model from \Cref{chap:modelbasedRecsys}. \cite{cheng2016wide} note that while neural networks can potentially learn complex interactions among latent features, they may nevertheless struggle to learn trivial but useful pairwise interactions among features. The wide and deep architecture essentially extends an architecture like the one above (the \emph{deep} component) by adding a path that allows the model to `circumvent' the multilayer part of the model using simple linear interactions (the \emph{wide} component). The wide component is based on a simple linear model (i.e.,~$x \cdot \theta$), which includes (among other components) simple binary combinations of features. This allows the wide component to focus on important but simple feature interactions, while the deep component can focus on more complex interactions.

%\done{something wrong with cite below since it's the same one as above}

Finally we mention
%the model of 
\cite{guo2017deepfm} (DeepFM), 
which adopts the same wide \& deep architecture from \cite{cheng2016wide}. 
Their presentation is more closely built around, and adopts the specific components of, the factorization machine architecture from \Cref{sec:factorization}. 
%but combines it with the specific components of factorization machine.
%\done{not great. Fine}
%which is an alternate architecture for deep learning-based factorization machines which is closely based on the wide \& deep architecture of \cite{cheng2016wide}.

%neural collaborative filtering
% https://arxiv.org/pdf/1708.05031.pdf
%Neural Factorization Machines for Sparse Predictive Analytics
% https://arxiv.org/pdf/1708.05027.pdf
%DeepFM: A Factorization-Machine based Neural Network for CTR Prediction
% https://arxiv.org/pdf/1703.04247.pdf
%Wide & Deep Learning for Recommender Systems
% https://arxiv.org/pdf/1606.07792.pdf

% \paragraph{Wide and Deep Factorization Machines}




%\done{missing one of the papers from the table in the previous chapter. Looks fine?}

%\done{moved from previous chapter}

%\done{Move all DL to here, or delete table from recsys chapter, or something else? Changed}


\Cref{tab:dlRecExamps} summarizes a few deep learning-based recommendation models (including some from \Cref{chap:modelbasedRecsys}); note that this is only a small sample intended to cover the variety of techniques and architectures involved (see e.g.~\cite{zhang2019deep} for a more complete survey).

%\done{References above or below tables? Needs to be consistent. I think above. Above}
\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Deep learning-based recommendation techniques.]{Deep learning-based recommendation techniques. References: \protect\cite{he2017neural,he2017neuralB,cheng2016wide,guo2017deepfm,cheng2016wide,sedhain2015autorec}.}
\label{tab:dlRecExamps}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{lXX}
% % \hrule\\
% \textbf{Reference} & \textbf{Technique} & \textbf{Description}\\
% \cite{he2017neural} & neural collaborative filtering & Uses multilayer perceptrons to learn complex interactions between user and item latent factors.\\
% \cite{guo2017deepfm} & deep factorization machines & Uses multilayer perceptrons to train factorization machines. \\
% \cite{he2017neuralB} & neural factorization machines & Trains factorization machines using the \emph{Wide \& Deep} approach from \cite{cheng2016wide}.\\
% %\done{say what the point of wide&deep is compared to MLP}
% \cite{sedhain2015autorec} & AutoRec & Learns compressed representations of item (or user) interaction vectors; the compressed representations can be used to estimate scores associated with unobserved interactions.  \\
% \end{tabularx}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.25\textwidth}X}
\toprule
\textbf{Ref.} & \textbf{Technique} & \textbf{Description}\\
\midrule
H17
%\cite{he2017neural}
& neural collaborative filtering & Uses multilayer perceptrons to learn complex interactions between user and item latent factors (\cref{chap:modelbasedRecsys}).\\
HC17
& neural factorization machines & Similar to the above, but using multilayer perceptrons within a factorization machine framework. \\
%\cite{guo2017deepfm} 
C16 & wide \& deep learning for recommendation & Includes a `wide' component to help the model capture pairwise feature interactions directly, allowing the deep component to focus on more complex hidden interactions. \\
%\done{move to FM section}
%\cite{he2017neuralB} 
G17
& deep factorization machines & 
%Uses multilayer perceptrons to train factorization machines. \\
%Uses the wide \& deep architecture to capture complex inter %factorization machines.\\
Similar to the above; incorporates the wide \& deep architecture with the specific components of factorization machines.\\
%\done{descrciption of deepFM doesn't match text}
%\done{haven't covered FMs yet. Move}
%using the \emph{Wide \& Deep} approach from \cite{cheng2016wide}.\\
%\done{say what the point of wide&deep is compared to MLP}
%\cite{sedhain2015autorec}
S15
& AutoRec & Learns compressed representations of item (or user) interaction vectors; the compressed representations can be used to estimate scores associated with unobserved interactions (\cref{chap:modelbasedRecsys}).  \\
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{any others to add here? Is fine}
%\done{not referenced}
%\done{table formatting}
%\done{table row separation? Looks okay}
%\done{make some table entries point to next section. Moved}


\subsection{Factorization Machines in Python with \emph{FastFM}}
\label{sec:fastFM}

%As we shall see later, 
As we saw above, factorization machines are a highly flexible, general-purpose technique to incorporate numerical or categorical 
%As we shall see later, the factorization machine is a highly flexible approach that can 
features into recommender systems. Later in this chapter, and in \Cref{chap:temporalSequential} we'll explore specific types of dynamics
%\done{make sure above is true. True enough}
that can be captured via factorization machines, but for the moment we'll explore an implementation of a `vanilla' factorization machine via the \emph{FastFM} library \citep{bayer2016fastfm} (in this case using \emph{Goodreads} data, as in Project~\ref{proj:2}).
%\done{Say what dataset it is?}
%\done{reference earlier project?}

First, we read our dataset and construct a mapping from each user to a specific index (from $0$ to $|U| - 1$);
%\done{notation. Looks right}
this index will be used to associate each user and item with a feature dimension in our one-hot encoding (as in \cref{eq:FMinteractionMatrix}):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
userIDs,itemIDs = {},{}
for d in data:
    u, i = d['user_id'], d['book_id']
    if not u in userIDs: userIDs[u] = len(userIDs)
    if not i in itemIDs: itemIDs[i] = len(itemIDs)
    
nUsers, nItems = len(userIDs), len(itemIDs)
\end{lstlisting}
\end{minipage}
%\done{No import statement? What do I normally do? Fine I think}

Next, we build our matrix of features associated with each interaction. Each feature is simply the concatenation of a (one-hot encoding of) a user ID and an item ID. Note that we use a sparse data structure (\texttt{lil\_matrix})
%\done{underscore looks kinda funny. Looks fine}
%\done{Do I use texttt elsewhere? Yes}
to represent the feature matrix. Although we only use user and item IDs here, these feature vectors could straightforwardly be extended to include other features, such as those we explore later in the chapter: 

%\done{Add comment: each row is like a row from design matrix \eq whatever}
\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last,escapechar=\%]
X = scipy.sparse.lil_matrix((len(data), nUsers + nItems))
for i in range(len(data)):
    user = userIDs[data[i]['user_id']]
    item = itemIDs[data[i]['book_id']]
    X[i,user] = 1 # Essentially a row from %$\Cref{eq:FMinteractionMatrix}$%
    X[i,nUsers + item] = 1

y = scipy.array([d['rating'] for d in data])
\end{lstlisting}
\end{minipage}

Finally, we split the data into training and test fractions, fit the model, and compute its predictions on the test set:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
X_train, y_train = X[:2000000], y[:2000000]
X_test, y_test = X[2000000:], y[2000000:]

fm = fastFM.als.FMRegression(n_iter=1000, init_stdev=0.1, rank=2, l2_reg_w=0.1, l2_reg_V=0.5)

fm.fit(X_train, y_train)
y_pred = fm.predict(X_test)
\end{lstlisting}
\end{minipage}
%\done{Explain above parameters}
%https://ibayer.github.io/fastFM/tutorial.html
%Better to look at the "help" page

The model has several tunable parameters: \texttt{n\_iter} controls the number of iterations; \texttt{init\_stdev} controls the standard deviation of random parameter initialization; \texttt{rank} controls the number of latent factors ($K$); \texttt{l2\_reg\_w} and \texttt{l2\_reg\_V} control the regularization for the model's linear and pairwise terms (similar to $\lambda_1$ and $\lambda_2$ in \Cref{eq:latentreg3}).

\section{Cold-Start Recommendation}
\label{sec:coldStart}

So far, the recommendation approaches we have developed have depended on having detailed \emph{interaction histories} associated with users and items. Naturally, we cannot find similar users (as in \cref{sec:neighborhood}) to a user who has no purchase history; likewise, we cannot estimate latent parameters $\gamma_u$ (or even a bias $\beta_u$) for a user who has never rated or purchased any items (as in \cref{subsec:matrixFactor}).

As such, we need to develop recommendation approaches that can be useful in so-called \emph{cold-start} scenarios. Depending on the setting, either users or items may be `cold' (i.e.,~have zero associated interactions). 

We'll investigate two categories of approach to deal with cold-start problems. First, one may attempt to deal with cold-start situations via the use of \emph{side-information} about users or items. Side information could range from product images, to text, or social interactions. In each case,
%(including more complex approaches we develop later in the book), 
side information gives clues as to the properties of an item, whether by learning user preferences toward observed item features (\cref{sec:coldStartSide}),
harvesting weaker signals such as the preferences of a user's friends (\cref{subsec:socialBPR}), 
or by using item features to estimate item latent factors (\cref{subsec:VBPR}).
%\done{not great}
We explore some of the simpler methods below, but revisit the use of side-information throughout the book.
Second, we'll explore methods that directly seek to 
elicit 
%\done{elicit or solicit? Looks right}
preferences from new users through surveys (\cref{subsec:coldStartSurvey}).

\subsection{Addressing Cold-Start Problems with Side Information}
\label{sec:coldStartSide}
%Pairwise Preference Regression for Cold-start
%Recommendation (feature based)

In the absence of historical interaction data associated with users or items, one option is to resort to secondary signals. 
\cite{park2009pairwise} consider cold-start settings in the context of movie recommendation. For movies, associated features are available such as the release year, genre, (etc.),~which can be encoded (for example) as a one-hot vector. For users, demographic features are used such as a user's age, gender, occupation, or location. These features are captured via user and item feature vectors $x_u$ and $z_i$ for each user $u$ and item $i$.

Recall that at the beginning of \Cref{sec:setup}, we discussed the differences between recommendation and regression and argued that recommendation was fundamentally different from simple linear regression on user and item features. Critically, we argued that recommender systems must model the \emph{interaction} between users and items, in order to be able to meaningfully personalize predictions for each user.

In order to capture interactions, \cite{park2009pairwise} use what is termed a \emph{bilinear} model (we briefly mentioned bilinearity in \Cref{sec:onlineUpdates}), 
%\done{bilinear already defined?}
to estimate the compatibility between user and item features. The model parameters can then be described via a matrix $W$, and user/item compatibility can be written as
\begin{equation}
    s_{u,i} = x_u W z_i^T = \sum_{a = 1}^{|x_u|} \sum_{b = 1}^{|z_i|} x_{u,a} z_{i,b} W_{a,b},
\end{equation}
Here, unlike the linear regression model from (e.g.)~\Cref{eq:whyRecDifferent}, $W$ now encodes how user features should \emph{interact} with item features. That is, a parameter $W_{a,b}$ encodes the extent to which the $a^{\text{th}}$
%\done{consistent way of writing this. Always like this}
user feature is compatible with the $b^{\text{th}}$ item feature. So, the model can learn (for example) the extent to which users in the 35-50 age demographic will respond positively to the teen romance genre.

Both $x_u$ and $z_i$ include a constant feature. These features (or rather the corresponding entries in $W$) roughly fill the role of bias terms (i.e.,~$\alpha$, $\beta_u$ and $\beta_i$ in \Cref{eq:MFscore}), that is, they allow the model to learn the extent to which users in a certain demographic, or movies of a certain genre, tend to yield higher or lower ratings than others. 

The model is trained so that the compatibility $s_{u,i}$ should align with observed interactions (e.g.~ratings). \cite{park2009pairwise} achieve this using a specific type of pairwise loss (i.e.,~a loss that considers two items at a time, similar to the BPR loss of \Cref{eq:BPRobjective}), though this is an implementation detail that is not critical to the main idea of the method.

Ultimately, the method is evaluated on two movie datasets (\emph{MovieLens} and \emph{EachMovie}). Cold users and items in these datasets are simulated, simply by withholding interactions from a subset of users and items at training time, with interactions from those users being used to evaluate the system at test time. Experiments show that when considering cold users and/or items, the method outperforms alternatives that don't make use of features.

% \paragraph{Other types of cold-start recommendation}
%\done{Capitalize paragraph-level headings? No}


\subsection{Addressing Cold-Start Problems with Surveys}
\label{subsec:coldStartSurvey}

An alternative to relying on side information in user cold-start settings is simply to directly solicit the preferences of new users once they first interact with the system.

\cite{rashid2002getting} investigated strategies for generating initial user surveys.
%, in order to most quickly learn the preference dynamics of new users. 
`Surveys' in this setting simply consist of collecting ratings about an informative set of items
%in order to 
in order to most quickly learn the preference dynamics of new users.
Several strategies are investigated for selecting informative items. Surfacing \emph{popular} items has the advantage that users will generally have interacted with them (and can thus provide an informed opinion), though opinions may not be informative if all users generally like the most popular items.
In contrast an \emph{entropy}-based strategy selects items where opinions are highly varied: each rating provides more information, but users may be unable to rate a large number of unknown items. Beyond these they explore hybrid strategies that balance popularity and entropy, as well as \emph{personalized} strategies that (once a few known items are found) survey similar items that a user is likely to have interacted with.

\cite{zhou2011functional} investigate more elaborate strategies based on the same principle, where a decision tree is constructed over items (where a user can provide positive, negative, or `unknown' feedback at each step) to iteratively choose the most informative items to surface to the user.

%\footnote{
Of course this setting is only appropriate for settings where users can realistically be expected to already have experience with a reasonable fraction of the items (\cite{rashid2002getting} and \cite{zhou2011functional} consider movie recommendation scenarios).
%}
%
%
% Functional Matrix Factorizations for Cold-Start
% Recommendation (interview based) 
%
%
%\done{figure out where text below goes. Drop}
%
% Obviously there is significant room to improve upon the above system, for example we might desire a model that simultaneously leverages features for cold users, and interactions for non-cold users, and learns to smoothly transition between the two groups.
%
We'll revisit the topic of cold-start recommendation regularly in later chapters, as well as `cool-start' settings (where we have only a few interactions per user or item), as we develop systems that operate on features from sequences, text, and images. Whether explicitly designed for cold-start or not, such methods often seek to use side information to circumvent the paucity of available interaction data.


\section{Multisided Recommendation}

So far, our view of recommendation and personalization has consisted of maximizing some predicted utility for each user, e.g.~estimate their ratings or which items they'll interact with. Furthermore, every user has predictions made independently of each other.

Such a setting seems natural when considering contexts such as movie recommendation, but there are several cases where such models are inappropriate. For example, recommendation on an online dating platform would require quite different assumptions. For instance, the problem has symmetries in the sense that the users being recommended are also receiving recommendations---as such, users must be interested in their matches, but the matches must have a reasonable chance of reciprocating. Likewise we must ensure not only that everyone receives recommendations, but that everyone \emph{is recommended to somebody}.

These types of problems are referred to as \emph{multisided} or \emph{multistakeholder} recommendations~\citep{abdollahpouri2017recommender}. Such constraints appear in several settings, many of which we will visit throughout the book. In \Cref{subsec:groupRec} we will look at \emph{group} recommendation, where recommendations must simultaneously satisfy the interests of multiple users in a group. And in \Cref{sec:advertising}, we will consider advertising settings, where we must consider not only user preferences but also the budgets of individual advertisers (which prevent us from recommending the most compatible ads to everyone). Finally we'll revisit the topic in depth in \Cref{chap:consequence}, where we consider issues of fairness, calibration, balance, etc. For example, when recommending movies we might want reasonable coverage of different genres \citep{steck2018calibrated}, or when recommending authors, we might want our recommendations not to be too narrow in terms of gender or nationality \citep{ekstrand2018exploring}.

For now, we'll consider two specific examples of multistakeholder recommendation, namely online dating and bartering (i.e.,~recommending trading partners).

%Multistakeholder Recommendation
%\cite{abdollahpouri2017recommender}

\subsection{Online Dating}
\label{subsec:onlineDating}

\cite{pizzato2010recon} studied recommendation in the context of online dating. Online dating has several constraints not present in the types of recommendation problems we've seen so far, in particular due to the fact that the users receiving recommendations are 
%being recommended are the same ones receiving recommendations. 
the same ones that are \emph{being recommended}.

\cite{pizzato2010recon} consider the specific objective of \emph{reciprocal communication}, which is partly motivated by a specific mechanism in the data they study (from a large Australian online dating website). 
%\done{a few words about what their data is?}
That is, a recommendation of a user $v$ to a user $u$ should be considered successful only if $u$ messages $v$, \emph{and} $v$ responds to their message.

Actual compatibility scores $f(u,v)$ in \cite{pizzato2010recon} are estimated using a fairly simple feature-based strategy that looks for a match between $u$'s preferences and $v$'s attributes (some of which may be matched strictly, e.g.~if a user has historically expressed interest in only a certain gender).  Following this the \emph{reciprocal compatibility} is simply the harmonic mean of the two compatibility scores:
\begin{equation}
    \text{reciprocal compatibility}(u,v) = \frac{2}{f(u,v)^{-1} + f(v,u)^{-1}}.
\end{equation}
The harmonic mean here is preferable to (e.g.)~the arithmetic mean as it does not allow either user's preference to `dominate' the compatibility estimate, i.e.,~two users are only compatible if \emph{both} have high compatibility scores for each other.


Beyond the notion of reciprocity considered in \cite{pizzato2010recon}, online dating has certain `balance' or `diversity' constraints not yet seen in other problems, for example we cannot identify a user with `ideal' characteristics and recommend them to everyone (which might be perfectly reasonable for, say, movies); instead, the system only has utility if users both receive, and appear in, recommendations. We consider this type of constraint further when considering online advertising problems in \Cref{sec:matching}.
%\done{Moved this paragraph, doesn't really fit}
%\done{Mention we'll see these types of constraints later, e.g. in diversity and advertising}

\subsection{Bartering Platforms}
\label{subsec:bartering}

\cite{rappaz2017bartering} considered the problem of generating recommendation systems for \emph{bartering} platforms, i.e.,~settings in which users exchange goods.

They study several settings in which products are exchanged, including CDs and DVDs, though most of their analysis centers around three datasets, of books (from \emph{bookmooch.com}), beers (from \emph{ratebeer.com}), and video games (from \emph{reddit.com/r/gameswap}).

On each of these websites, users have both a `wishlist' $W_u$ and a `give-away' list $G_u$, i.e.,~sets of items they wish to give or receive. Given this constraint on the problem, one might think that recommending compatible trades is as simple as identifying compatible pairs. However surprisingly the data reveals that `eligible' swapping partners are incredibly rare, and the vast majority of logged trades occur between items that were not expressly included in a user's wishlist; thus there is a need to build a system that can model likely trading partners via latent preferences. They also note that users repeatedly trade with the same partners, indicating that there is a social component to trading.

%\done{One of these equations seems wrong. U already owns I in figure, but not in eq. I think fix figure}

Given the above two factors, the basic model combines a standard latent factor representation with a social term. Given a user $u$, an item $i$, and a potential trading partner $v$, their compatibility is modeled as
\begin{equation}
    f(u,v,i) = \gamma_u \cdot \gamma_i + S_{u,v}.
    \label{eq:barteringCompat}
\end{equation}
Here $\gamma_u$ and $\gamma_i$ are low-rank factors as in \Cref{eq:MFscore}, whereas $S_{u,v}$ is a (potentially full-rank) matrix $S \in \mathbb R^{|U|\times |U|}$;
%\done{is the number of users |U|? Yes}
although $S$ potentially encodes a large number of parameters, in practice it is very sparse (at least in their datasets) as the number of observed trading partners is limited.

Note that the above model captures only the interest of one user toward another's item; to model reciprocal interest, \cite{rappaz2017bartering} simply captures the average of interest in both directions (\cref{fig:reciprocalInterest}):
\begin{equation}
    f(u,i,v,j) = \frac{1}{2}(f(u,v,i) + f(v,u,j)).
    \label{eq:barteringCompat2}
\end{equation}
Other aggregation functions besides the arithmetic mean (such as the harmonic mean) can be used, though the arithmetic mean proved the most effective, indicating that a weak preference from one user can be made up for by a strong preference from another (in contrast to the online dating scenario from \Cref{subsec:onlineDating}). The model also includes a temporal term encoding timepoints when certain users are particularly active and certain items are particularly popular, though we leave discussion of temporal models to \Cref{chap:temporalSequential}.


\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
%  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=8mm]
%\node[fsnode,on chain,draw,circle] (f1) [label=left: 1, label={[label distance=0.7mm]90:women:}] {};
\node[fsnode,on chain,draw,circle] (f1) [label=left: $u$,label={[label distance=2mm, rectangle]90:users:}] {};
\node[fsnode,on chain,draw,circle] (f2) [label=left: $v$] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=8mm]
%\node[ssnode,on chain,draw,circle] (s1) [label=right: 1, label={[label distance=2mm]90:men:}] {};
\node[ssnode,on chain,draw,circle] (s1) [label=right: $i$,label={[label distance=2mm, rectangle]90:items:}] {};
\node[ssnode,on chain,draw,circle] (s2) [label=right: $j$] {};
\end{scope}

% \node[above=of f1,yshift=2mm,rectangle] (lu) {users:};
% \node[above=of s1,distance=2mm,rectangle] (li) {items:};

% \node[above=of f1] (w1) {women};
% \node[above=of s1] (m1) {men};

\draw[thick] (s1) edge[-] node[above,rectangle,draw=none] {swaps} (f1);
% \node [pos=0.66,above=s1] {Transpose};
\draw[dotted,thick] (s1) edge[-] node[right,xshift=2mm,rectangle,draw=none] {owns} (f2);
\draw[dotted,thick] (s2) edge[-] (f1);
\draw[thick] (s2) edge[-] node[below,rectangle,draw=none] {swaps} (f2);




% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}

%$\text{Compatibility}(u,v,i,j) = \frac{1}{2} (f(u,v,i) + f(v,u,j))$

    \caption[Reciprocal interest in bartering settings.]{Basic idea behind reciprocal interest, as in \protect\cite{rappaz2017bartering}; in bartering settings a strong preference from one user compensates for a potentially weaker one from the other. Compatibility is a function of the social cohesion between $u$ and $v$, as well as their interest in the desired items (\protect\Cref{eq:barteringCompat,eq:barteringCompat2}).
    \label{fig:reciprocalInterest}
    }

\end{figure}
%\done{Make this figure}

%See other examples from multisided fairness paper

Ultimately, the method is evaluated in terms of its ability to assign higher scores to observed interactions compared to non-observed ones (i.e.,~using a BPR-like training and evaluation scheme, as in \Cref{subsec:BPR}). The main conclusion of the experiments is that several components are important in bartering settings: reciprocal interest, a social history of trades, as well as temporal `trends' across users and items.


\section{Group- and Socially-Aware Recommendation}
\label{sec:groupSocialRec}

%\done{Could also add bundling here, if it's cast more as set-based recommendation. Nah}

%\done{intro paragraph}

Our opinions and decisions are influenced by our social connections. 
%Many of the interaction datasets we've been studying so far include additional signals in the form of social interaction data.
%Within the context of interaction data, possible sources of influence could include:
% \begin{itemize}
%     \item 
Within the context of recommendation,
One possible dynamic is that of \emph{social trust}:
%For example, users my \emph{trust} their friends' opinions, 
%as such 
%mean
the fact that a friend has liked or purchased an item is a strong predictor of a user's future behavior (\cref{subsec:soRec}).
%     \item 
    %Satisfy group preferences
Alternately, 
in some contexts recommendations must satisfy several users' interests simultaneously (e.g.~selecting a movie for a group of users to watch), so that a user's preferences should be balanced against those of their friends (\cref{subsec:groupRec}).
%     \item 
% \end{itemize}

More prosaically, social connections can simply be a way to harvest additional interaction data to improve the accuracy or cold-start performance of recommender systems. Given a paucity of data from a particular user, 
%we can use interactions from their social network to
interactions from their social network can act as weak signals to augment the amount of data to be used for training.
%\done{ugly paragraph}
We explore a few representative approaches below, which we summarize in \Cref{tab:socialRec}.


\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Comparison of socially-aware recommendation techniques.]{Comparison of socially-aware recommendation techniques. References: \protect\cite{ma2008sorec,zhao2014leveraging,o2001polylens,amer2009group,pan2013gbpr}.}
\label{tab:socialRec}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{lXX}
% % \hrule\\
% \textbf{Reference} & \textbf{Dataset} & \textbf{Description}\\ 
% \cite{ma2008sorec} & Social trust data from \emph{epinions} & Trust links help to regularize $\gamma_u$, which must simultaneously explain rating and trust factors.\\
% \cite{zhao2014leveraging} & \emph{epinions}, \emph{Ciao}, \emph{Delicious}, \emph{LibraryThing} &
% Friends' interactions act as additional \emph{implicit signals} for recommendation.\\
% % \hrule
% \cite{o2001polylens} & \emph{MovieLens} & Studies the desirable characteristics of good interfaces for group recoommendation.\\ 
% \cite{amer2009group} & \emph{MovieLens} & Designs measures of \emph{group consensus} and proposes recommendation algorithms to maximize them.\\
% \cite{pan2013gbpr} & \emph{MovieLens}, \emph{Netflix} & Treats group preferences as weak signals to design pairwise sampling strategies for BPR.
% \end{tabularx}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.24\textwidth}X}
%\done{Adjust fixed column width by eye, all tables. Look okay}
% \hrule\\
\toprule
\textbf{Ref.} & \textbf{Social Data} & \textbf{Description}\\ 
\midrule
%\cite{ma2008sorec} 
M08
& 
%Social trust data from 
\emph{Epinions} & Trust links help to regularize $\gamma_u$, which must simultaneously explain rating and trust factors (\cref{subsec:soRec}).\\
%\cite{zhao2014leveraging} 
Z14
& \emph{Epinions}, \emph{Delicious}, \emph{Ciao},  \emph{LibraryThing} &
Friends' interactions act as additional \emph{implicit signals} for recommendation (\cref{subsec:socialBPR}).\\
% \hrule
%\cite{o2001polylens} 
O01
& \emph{MovieLens} & Studies the desirable characteristics of good interfaces for group recommendation (\cref{subsec:groupRec}).\\ 
%\cite{amer2009group} 
A-Y09
& \emph{MovieLens} & Designs measures of \emph{group consensus} and proposes recommendation algorithms to maximize them (\cref{subsec:groupRec}).\\
%\cite{pan2013gbpr} 
PC13
& \emph{MovieLens}, \emph{Netflix} & Treats group preferences as weak signals to design pairwise sampling strategies for BPR (\cref{subsec:groupBPR}).\\
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{not refereced}
%\done{table formatting}


\subsection{Socially-Aware Recommendation}
\label{subsec:soRec}

Several approaches have sought to incorporate signals from social networks to improve recommendation. The basic idea behind doing so is that social connections will help us to circumvent \emph{sparsity} issues in interaction data. That is, even if a user has only a small number of observed interactions, we can (to some degree) leverage the interactions of their friends, whose opinions they are likely to trust.

Conceptually, the typical approach behind socially-aware recommendation is to use social connections as a form of \emph{regularizer}, which states that a user's preferences should be \emph{similar to} those of their connections in a social network. For example, given a user with few interactions, we might assume that their preferences align with the (average of) their friends; this is a possibly better assumption than the regularizer of \Cref{eq:latentreg2}, which in practice will essentially discard user latent factors ($\gamma_u$) for users with few interactions.


%SoRec: Social Recommendation Using
%Probabilistic Matrix Factorization
An early attempt to incorporate social networks into recommender systems extended the basic framework of a latent factor model \citep{ma2008sorec}. They looked at data from \emph{Epinions}, which in addition to interaction data in the form of ratings (much like \Cref{eq:interactionMatrix}), includes a network of `trust' and `distrust' relationships. Unlike a typical social network, these are \emph{signed} relationships, whereby a user explicitly indicates that they `trust' (1) or distrust (-1) another user. That is, in addition to our interaction matrix we have a (directed) adjacency matrix:
\begin{equation}
    A = \underbrace{\left[ \begin{tabular}{ccccc}
    1 & $\cdot$ & $\cdot$ & -1 & 1\\
    $\cdot$ & 1 & -1 & $\cdot$ & $\cdot$\\
    $\cdot$ & 1 & 1 & -1 & $\cdot$\\
    1 & $\cdot$ & 1 & $\cdot$ & 1\\
    -1 & -1 & $\cdot$ & 1 & 1\\
    \end{tabular} \right]}_{\let\scriptstyle\textstyle
    \substack{\text{users}}} \left. \mathclap{\begin{tabular}{c} \\ \\ \\ \\ \\ \end{tabular}} \right\rbrace \text{users}.
    %\done{In other equations like this, make sure the underbrace text is the right size}
    \label{eq:trustMatrix}
\end{equation}
Ultimately though, the distrust relationships are not used in \cite{ma2008sorec}, as it is argued that the semantics of `distrust' are somewhat more complex than (e.g.)~users having different preference dimensions.

%\done{Make shared parameter figure (copy from text) and put here?}

Thus, give a rating matrix $R$, and an adjacency matrix $A$, we want to predict ratings in $R$ in such a way that $A$ informs us about each user's likely latent preferences. The basic idea is to make use of a \emph{shared parameter} $\gamma_u$ for each user. For rating data, $\gamma_u$ is no different from a user latent factor in a matrix factorization model, i.e.,~it is combined with an item latent factor and used to predict the rating, in this case via a sigmoid function:\footnote{The specific choice of the sigmoid function is an implementation detail, and ratings are scaled to be in the range $[0,1]$ to accommodate this choice.}
\begin{equation}
    r_{u,i} = \sigma(\gamma_u \cdot \gamma_i).
\end{equation}
Next, the parameter $\gamma_u$ is \emph{re-used} to predict trust relationships in $A$:
\begin{equation}
    a_{u,v} = \sigma(\gamma_u \cdot \gamma'_v).
\end{equation}
The original paper allows $A$ to be a weighted matrix, indicating varying degrees of social trust, but for simplicity we assume here that $A$ contains only trust (1) and not-trust (0) values.




Note that while $\gamma_u$ is a shared parameter, $\gamma'_v$ is not; since the matrix $A$ is directed, $\gamma_u$ can be thought of as explaining why $u$ trusts others, whereas $\gamma'_v$ explains why $v$ \emph{is trusted by} others.
%\done{confusing, does the above help? Is okay}
%
In practice we are usually not interested in predicting entries $a_{u,v}$; rather, the trust relationships are additional data that should help us to calibrate $\gamma_u$ more efficiently.  This idea is depicted in \Cref{fig:sorecIdea}.
%For example, if a user has few observed ratings, $\gamma_u$ can still be estimated via their trust relationships.
%, giving us an initial estimate of their rating behavior in cold- (or cool-)start settings.
%\done{Did I already make this argument above? Dropped}

\begin{figure}
    \centering
\begin{tikzpicture}
 %\draw[dashed] (0,0) rectangle (5,4) node[pos=.5] (A) {Dataset $\lbrace X; y \rbrace$};
% \node [below=of A,yshift=0em] {Estimate $\theta$};
\node [style=circle,draw] (gamma1) {$\gamma_i$};
\node [right=of gamma1, circle, fill, inner sep=0.1em] (pt1) {};
\node [style=circle,draw,right=of pt1] (gamma2) {$\gamma_u$};
\node [right=of gamma2, circle, fill, inner sep=0.1em] (pt2) {};
\node [style=circle,draw,right=of pt2] (psi) {$\gamma'_v$};
\node [below=of pt1] (out1) {$r_{u,i} = \sigma(\gamma_u \cdot \gamma_i)$};
\node [below=of pt2] (out2) {$a_{u,v} = \sigma(\gamma_u \cdot \gamma'_v)$};
\draw[->] (pt1) -- (out1);
\draw[->] (pt2) -- (out2);
\draw (gamma1) -- (pt1);
\draw (pt1) -- (gamma2);
\draw (gamma2) -- (pt2);
\draw (pt2) -- (psi);

\node[style=dashed,draw, inner sep=3mm, yshift=0.2em, fit=(gamma1) (gamma2)] (Fit1) {};
\node[style=dashed,draw, inner sep=3mm, yshift=-0.2em, fit=(gamma2) (psi)] (Fit2) {};

\node[above=of Fit1,yshift=-3em] (label1) {ratings};
\node[above=of Fit2,yshift=-2.6em] (label2) {social links};

\end{tikzpicture}
    \caption[Shared parameters in socially-aware recommendation.]{Social recommendation techniques often make use of a \emph{shared parameter} (in this case $\gamma_u$) that simultaneously explains rating dimensions and social connections. In this way, social links can estimate preference dimensions even for users with few historical ratings.}
    \label{fig:sorecIdea}
    %\done{Reference this somewhere!}
\end{figure}

The overall objective then takes the form
\begin{equation}
    \underbrace{\sum_{(u,i) \in R} (r_{u,i} - \sigma(\gamma_u \cdot \gamma_i))^2}_{\text{rating prediction error}} + \lambda^{(\text{trust})}\underbrace{\sum_{(u,v) \in A} (a_{u,v} - \sigma(\gamma_u \cdot \gamma'_v))^2}_{\text{trust prediction error}} + \lambda\| \gamma \|_2^2,
\end{equation}
%\done{Parentheses don't add up in above equation}
where $\lambda^{(\text{trust})}$ trades-off the importance of predicting the trust network (versus predicting ratings). Ultimately, experiments in \cite{ma2008sorec} show that the trust network helps to predict ratings more accurately than matrix factorization alone. Of course, it should be noted that trust relationships on \emph{Epinions} 
%\done{capitalization, italicization?}
are very closely tied to opinion dimensions, presumably moreso than in other social networks.

Note that the above is essentially a more complex form of cold-start (or `cool-start')
%\done{make sure not to say the above too many times. Deleted above}
recommendation (as we saw in \Cref{sec:coldStart}), in the sense that we are leveraging a form of side-information (social connections) to make up for a paucity of interaction data. In the case of a user who has never rated an item (but has social connections), the system can still reasonably estimate $\gamma_u$ from the preference dimensions of $u$'s friends.


\subsection{Social Bayesian Personalized Ranking}
\label{subsec:socialBPR}

%Leveraging Social Connections to Improve Personalized
%Ranking for Collaborative Filtering

Above we showed how matrix factorization frameworks can be extended to incorporate signals from social networks. The intuition behind the idea simply stated that users are likely to have preference dimensions aligned with other users who they `trust.'

Next, we'll see how this idea can be adapted to predict interactions (rather than ratings), by incorporating social connections into the Bayesian Personalized Ranking framework from \Cref{subsec:BPR}.

Conceptually, using social links to predict
%Personalized ranking 
interactions relies on
a possibly different assumption than we made above. Whereas our previous intuition above was based on some notion of trust, here we are simply assuming that a user is more likely to interact with items (e.g.~to watch movies or read books) if their friends have previously interacted with them.

\cite{zhao2014leveraging} attempted to adapt the assumptions made by Bayesian Personalized Ranking (BPR) to datasets involving social connections. Recall that BPR makes the assumption that a user's compatibility with items they've interacted with ($x_{u,i}$) should be higher than their compatibility with items they haven't interacted with ($x_{u,j}$), which is captured via a sigmoid function:
\begin{equation}
    x_{u,i} \geq x_{u,j} \quad \rightarrow \quad \sigma(x_{u,i} - x_{u,j}) \text{\ should be maximized}
\end{equation}
(see \cref{eq:BPRobjective}). To adapt this to settings involving a social network, \cite{zhao2014leveraging} assume a third type of feedback: for a user $u$, in adition to \emph{positive} feedback $i$, and \emph{negative} feedback $j$ (as with BPR), we also have \emph{social} feedback $k$, which consists of items consumed by $u$'s connections in a social network.

\cite{zhao2014leveraging} test two assumptions about how this social feedback should be incorporated. The first states that social interactions are weaker than positive interactions, but still stronger than negative interactions, essentially stating that users are somewhat more likely to interact with items their friends have interacted with:
\begin{equation}
    \underbrace{x_{u,i}\vphantom{x_{u,j}}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,k}\vphantom{x_{u,j}}}_{\mathclap{\text{social}}}; \quad \underbrace{x_{u,k}\vphantom{x_{u,j}}}_{\mathclap{\text{social}}} \geq \underbrace{x_{u,j}}_{\mathclap{\text{negative}}}.
    \label{eq:sbpr1}
\end{equation}
An alternate hypothesis states the opposite: if our friends have interacted with an item but we haven't, this might instead be a signal that we know about the item, but deliberately chose not to interact with it; in this instance we drop the second assumption from \Cref{eq:sbpr1} and replace it with a weaker assumption:
\begin{equation}
        \underbrace{x_{u,i}\vphantom{x_{u,j}}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,k}\vphantom{x_{u,j}}}_{\mathclap{\text{social}}}; \quad \underbrace{x_{u,i}\vphantom{x_{u,j}}}_{\mathclap{\text{positive}}} \geq \underbrace{x_{u,j}}_{\mathclap{\text{negative}}}.
    \label{eq:sbpr2}
\end{equation}
Note that neither of these assumptions is `better' than the other; rather they are simply hypotheses that must be tested by determining which best fits real datasets.

To train the model, a BPR-like objective (\cref{eq:BPRobjective}) is used, but which now involves two terms. E.g.~using the assumption from \Cref{eq:sbpr1}:
\begin{equation}
    \sum_{(u,i,k) \in \mathcal T} \log \sigma(x_{u,i} - x_{u,k}) + \sum_{(u,k,j) \in \mathcal T} \log \sigma(x_{u,k} - x_{u,j}) + \| \gamma \|_2^2,
\end{equation}
where $\mathcal T$ is a training set consisting of positive, negative, and social feedback ($i$, $j$, and $k$) for each user $u$.

Ultimately, \cite{zhao2014leveraging} show that both models outperform alternatives that don't leverage social connections. Several datasets besides \emph{Epinions}
%\done{italicize, capitalize?}
are used, including data from \emph{Ciao}, \emph{Delicious}, and \emph{LibraryThing} (product reviews, social bookmarks, and books).
Overall, the assumption from \Cref{eq:sbpr2} slightly outperforms that of \Cref{eq:sbpr1} on all datasets.



\subsection{Group-Aware Recommendation}
\label{subsec:groupRec}

Somewhat related to the topic of social recommendation is the idea of \emph{group} recommendation, where recommendations should be made collectively to a group of users, rather than to an individual. 
%However, 
%Unlike 
%As with
%social recommendation, 
%where the goal is normally to use social signals to improve 
%the goal is sometimes to use group signals to improve recommendation accuracy,
Understanding group dynamics can help to improve recommendation accuracy, though another goal of the approaches below is also
%the 
%accuracy,
%of recommendations each individual receives, 
%though many
%the goal of group-aware recommendation is generally 
to
design 
%new 
evaluation criteria 
%(and
%produce recommendations)
%that mutually satisfy a group of individuals.
that correspond to group satisfaction.

% PolyLens: A Recommender System for
% Groups of Users (basically just has an interface for group recommendations)

Early work on this topic includes \emph{PolyLens}
%\done{italics. Fine}
\citep{o2001polylens}, though this work was mostly concerned with designing user interfaces for the purpose of group recommendation, rather than using group data to improve recommendation performance. Although focused on interface-building, the work shows the utility of group-based recommenders, which could help users to find items that are compatible with their group's mutual interests.
%\done{is that what it actually says? Close enough}

Later, \cite{amer2009group} attempted to formalize the notion of group recommendation, by defining useful objectives that define how compatible a set of items is with a group. Given a pre-defined compatibility function $f(u,i)$ (e.g.~the output of a latent factor model), a simple attempt to define group compatibilty between a group of users $\mathcal G$ and an item $i$ might consist of computing 
\begin{equation}
    \text{Average compatibilty of $i$:} \quad \mathit{rel}(\mathcal G, i) = \frac{1}{|\mathcal G|} \sum_{u \in \mathcal G} f(u,i).
    \label{eq:groupRel1}
\end{equation}
Alternately, we could define the compatibility as that of the \emph{least} compatible user in the group, known as \emph{least misery}:
\begin{equation}
\text{Least misery:} \quad \mathit{rel}(\mathcal G, i) = \min_{u \in \mathcal G} f(u,i).
    %2) Least-Misery: rel(G, i) = Min(relevance(u, i))
    \label{eq:groupRel2}
\end{equation}
The latter is preferable in settings where users have constraints, e.g.~to avoid recommending a steakhouse to a group of users, some of whom are vegetarian.
%\citep{amer2009group}.

\cite{amer2009group} argue that in addition to maximizing relevance (or minimizing misery), it is also important that the group has some degree of \emph{consensus} about the quality of an item. That is, users in a group should not drastically disagree about $f(u,i)$, separately from their actual scores. Two disagreement functions proposed are the average pairwise disagreement:
\begin{equation}
    \mathit{dis}(\mathcal G, i) = \frac{2}{|\mathcal G|(|\mathcal G| - 1)} \sum_{(u,v) \in \mathcal G, u \neq v} (| f(u,i) - f(v,i) |),
    \label{eq:groupDis1}
\end{equation}
and disagreement variance:
\begin{equation}
    \mathit{dis}(\mathcal G, i) = \frac{1}{|\mathcal G|} \sum_{u \in \mathcal G} \Bigl(f(u,i) - \underbrace{\frac{1}{|\mathcal G|} \sum_{v \in \mathcal G} f(v,i)}_{\mathclap{\text{average compatibility in $\mathcal{G}$}}}\Bigr)^2.
    \label{eq:groupDis2}
\end{equation}
%\done{Both are called "dis". Should one be called "var"? Check paper. I think it's right as they're both disagreement functions. Looks right}

Finally, \cite{amer2009group} argue that a group \emph{consensus function} should be a combination of both of these factors, i.e.,~a relevance function (\cref{eq:groupRel1} or (\ref{eq:groupRel2}))
and a disagreement function
(\cref{eq:groupDis1} or (\ref{eq:groupDis2})):
%\done{maybe use lighter-weight equation references within parentheses?}
\begin{equation}
    \mathcal{F}(\mathcal G, i) = w_1 \times \mathit{rel}(\mathcal G, i) + w_2 \times (1 - \mathit{dis}(\mathcal G, i)),
\end{equation}
where $w_1$ and $w_2$ trade-off the relative importance of the two terms. \emph{Group recommendation} then consists of finding suitable items (e.g.~movies that no user in the group has seen) that maximize a (tuned) group consensus function.

Beyond defining notions of group consensus, \cite{amer2009group} show how to efficiently select items that maximize the above criteria (note the large number of comparisons involved when performing optimization na\"ively).
%
They also demonstrate experimentally (via a \emph{Mechanical Turk}-based user study of movie recommendations) that both \emph{relevance} and \emph{disagreement} are simultaneously important to achieve satisfaction within a group.


\subsection{Group Bayesian Personalized Ranking}
\label{subsec:groupBPR}
%\done{could be paragraph, not much to say here. No}

Much like \cite{zhao2014leveraging} incorporated social links into Bayesian Personalized Ranking by treating friends' interactions as additional implicit signals, 
`Group BPR' \citep{pan2013gbpr}
seeks to treat group preferences as a form of implicit signal that can be used within a BPR framework.
%  GBPR: Group preference based bayesian
% personalized ranking for one-class collaborative filtering \cite{pan2013gbpr}


While BPR assumes that a user $u$'s compatibility with an observed interaction $i$ is greater than their compatibility with an unobserved interaction $j$ (i.e.,~$x_{u,i} > x_{u,j}$, as in \Cref{eq:bprOpt0}),
%\done{compare notation to BPR section. Looks fine}
\cite{pan2013gbpr} assume that \emph{group} preference acts as a similar form of implicit feedback.
As with Social BPR (\cref{subsec:socialBPR}), the goal of Group BPR is essentially to leverage weak signals from related users as a way of harvesting implicit pairwise preference feedback.

%\done{Make sure various acronyms are defined, probably put them in notation page}

Specifically, if a group of users $\mathcal G$ has interacted with some item $i$, their \emph{mutual preference} toward the item, defined as
\begin{equation}
    x_{\mathcal G, i} = \frac{1}{|\mathcal G|} \sum_{g \in \mathcal G} x_{g,i}
\end{equation}
is assumed to be greater than a user $u$'s preference toward an unseen item $j$ (i.e.,~$x_{u,j}$). This notion of group preference is combined with the pairwise preference model from \Cref{eq:bprOpt0}, resulting in a preference model of the form:
\begin{equation}
    \rho x_{\mathcal G, i} + (1 - \rho) x_{u,i} >  x_{u,j},
    \label{eq:gbprSample}
\end{equation}
where $\rho$ is a hyperparameter controlling the relative importance of individual versus group preference.

Interestingly, the training data used for evaluation by \cite{pan2013gbpr} consists of `standard' interaction datasets that do not contain explicit groups; rather, groups are sampled randomly among users who have consumed a particular item.
As such, Group BPR is perhaps best thought of as a different means of leveraging explicit and implicit signals in implicit feedback settings, rather than as a group method as such.

Experiments in \cite{pan2013gbpr} show that sampling pairwise preferences as in \Cref{eq:gbprSample} can improve performance over standard BPR on various benchmark datasets.
%\done{boring. Is fine}


%\subsection{Price}
\section{Price Dynamics in Recommender Systems}
%\done{Change below paragraphs to subsections. Have already I think?}
\label{sec:priceRecSys}

In spite of the obvious impact price has on user decisions, there is surprisingly little work that seeks to incorporate price features into personalized predictive models.
%
Partly this owes to the lack of suitable datasets: the vast majority of the datasets we've study so far (concerned with movies, books, restaurants, etc.)~include few useful features from which to build a model of price.


% %Developing a price-sensitive recommender system to improve accuracy and business performance of eCommerce applications (initial attempt to include price, doesn't seem super relevant)
% Early 
% %\done{not early. Removed}
% attempts at modeling price within the context of recommender systems explored the overall difficulty of
% successfully incorporating price dynamics 
% %into recommender systems.
% \cite{umberto2015developing}.
% %\done{fold into intro sentence}




Even in datasets that include a price variable, it is not obvious how this variable should be incorporated in to the types of algorithms we've seen so far (the difficulty of incorporating these variables is discussed in e.g.~\cite{umberto2015developing}). 
%\done{not beautiful. Fine}
Na\"ively one might think that price might be incorporated into (e.g.)~a factorization machine (\cref{sec:factorization}) much like any other feature. While such a feature might help in cold-start settings, it is unlikely to improve predictive performance in general: to the extent that price explains significant variability in user preferences or item properties, it may already be captured by user or item latent factors. This form of ineffectiveness often comes as a surprise when implementing content-aware models: the features that explain the most variance (price, brand, genre, etc.)~are precisely those that latent factor models already capture, and add little predictive capacity (see \cref{box:whenUseful}).
%\done{somewhat contradicts the claims of the first paper presented. Who cares}
A notable exception to this is features that are not \emph{static}:
%We explore this scenario more broadly 
%However, 
while a simple feature like the price of an item may already be `baked in' to a latent factor representation, what our current models cannot tell us is how a user would react to a \emph{change} in price. As such, much of the research we'll explore below is concerned with questions of price variability, and modeling the impact that a change in price will have on user preference.



\begin{figure}
\caption{When is side-information useful for recommendation?
\label{box:whenUseful}}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
\begin{itemize}
\item Most of the settings we've considered in this chapter have essentially been forms of \emph{cold-start}. In other words, features compensate for a lack of historical interaction data (from either the user or item side).
%, either as a form of regularizer, 
\item Features are \emph{unlikely} to be particularly useful in `warmer' settings: even if a feature (price, brand, genre) explains variance, high-variance dimensions will already be captured by latent terms (i.e.,~$\gamma_u$ and $\gamma_i$).
\item An exception to the above is features which are not \emph{static}. We study price variability in this chapter, and temporal dynamics more broadly in \Cref{chap:temporalSequential}. Latent terms will struggle to capture this type of variability unless it is explicitly modeled.
\item Another important use of features is for model \emph{interpretability}: even features that yield a modest improvement in predictive performance may help us to understand the underlying dynamics of a particular problem better than we can from latent representations. We discuss such notions of interpretability (in the context of text-aware models) in \Cref{chap:text}.
\end{itemize}
\end{minipage}
}
\end{figure}
%\done{reference. Referenced in price section}



%An Empirical Analysis of Algorithmic Pricing on Amazon Marketplace. Attempts to empirically detect algorithmic pricing among best-sellers on Amazon, and analyses their characteristics.
%https://mislove.org/publications/Amazon-WWW.pdf
%Not really relevant to personalizadion, though could cover in intro



The models covered in this section are summarized in \Cref{tab:price}.



\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Comparison of price-aware recommendation techniques.]{Comparison of price-aware recommendation techniques. References: \protect\cite{ge2011cost,guo2017understanding,hu2018learning,wan2017modeling}.}
\label{tab:price}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{lXX}
% % \hrule\\
% \textbf{Reference} & \textbf{Dataset} & \textbf{Description}\\
% \cite{ge2011cost} & (Proprietary) data of travel-tour purchases & Incorporates price and time constraints into travel-tour recommendations\\
% \cite{guo2017understanding} & Various \emph{Amazon} categories & Disentangles interactions in terms of preferences versus price compatibility.\\
% \cite{wan2017modeling} & Purchases from Seattle grocery stores & Estimates how purchase decisions (item choice, quantity, etc.)~are affected by price fluctuations.\\
% \cite{hu2018learning} & Purchase and browse data from \emph{Etsy} & Forecasts a user's target purchase price from a sequence of browsed items.\\
% % \hrule
% \end{tabularx}
%\begin{tabularx}{\linewidth}{lXX}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.25\textwidth}X}
% \hrule\\
\toprule
\textbf{Ref.} & \textbf{Price Data} & \textbf{Description}\\
\midrule
%\cite{ge2011cost}
%\done{Do table entries always enter in full stop? Yes}
G11
& (Proprietary) data of travel tour purchases & Incorporates price and time constraints into travel tour recommendations (\cref{subsec:disentanglePrice}).\\
%\cite{guo2017understanding} 
G17
& Various \emph{Amazon} categories & Disentangles interactions in terms of preferences versus price compatibility (\cref{subsec:disentanglePrice}).\\
%\cite{hu2018learning} 
H18
& Purchase and browse data from \emph{Etsy} & Forecasts a user's target purchase price from a sequence of browsed items (\cref{subsec:willing2pay}).\\
%\cite{wan2017modeling} 
W17
& Purchases from Seattle grocery stores & Estimates how purchase decisions (item choice, quantity, etc.)~are affected by price fluctuations (\cref{subsec:sensElas}).\\
\bottomrule
% \hrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{table formatting}
%\done{not referenced}



\subsection{Disentangling Prices and Preferences}
\label{subsec:disentanglePrice}

\cite{ge2011cost}
%\cite{ge2014cost}
consider price from the perspective of a user who wants recommendations that satisfy a \emph{budget constraint}.
%https://viscenter.uncc.edu/sites/viscenter.uncc.edu/files/CVC-UNCC-14-06.pdf
They consider this problem in the context of recommending `travel tours,' where a user has constraints in terms of time (the length of the vacation), and the amount they are able to spend. They note however, that while the length and price of an actual travel package is observed, a user's constraints may not be, and as such must be modeled (or estimated) based on their historical activities.

To achieve this type of price-aware recommendation, \cite{ge2011cost} consider a modification of a latent factor model that includes both a preference compatibility, and a price-compatibility term:
%\done{no need to cite this all the time. Don't see the problem}
\begin{equation}
    f(u,i) = \underbrace{S(C_u, C_i)}_{\mathclap{\text{price compatibility}}} \cdot \overbrace{\gamma_u \cdot \gamma_i}^{\mathclap{\text{user preference}}}.
\end{equation}
$\gamma_u$ and $\gamma_i$ are user and item (travel tour)-related latent factors, as in \Cref{eq:MFscore}. $C_u$ and $C_i$ are cost-related factors; the cost for the tour ($C_i$) is assumed to be an observed, two-dimensional vector, encoding both time and price; $C_u$ is assumed to be a corresponding latent compatibility encoding the user's price constraints; $S$ is then a compatibility function, such as the (negative) Euclidean distance:
\begin{equation}
    S(C_u, C_i) = 1 - \| C_u - C_i \|_2^2.
    \label{eq:price1}
\end{equation}
A few improvements to this basic model are proposed (including in a follow-up paper \citep{ge2014cost}); for example, different training strategies are proposed based on different types of explicit and implicit feedback; and the user factor $C_u$ is carefully regularized (since e.g.~a trivial $\ell_2$ regularizer
%\done{Do I use \ell_2 elsewhere? Yes}
would center user price constraints around zero).


Experiments (on a proprietary dataset of historical travel tour interactions) show that the model outperforms variants that fail to consider price information.

%\done{Maybe have paragraph here and raise levels of other subsections. Fine, doesn't need paragraph label}
\cite{guo2017understanding} also build a model to separately capture price and preference dynamics. Although the specific method is somewhat different from those we've discussed (a form of Poisson Factorization, see e.g.~\cite{gopalan2013scalable}), this approach has a common goal with \cite{ge2011cost} of separating price and preference concerns. In essence, latent item properties $\gamma_i$ are responsible for estimating ratings via $\gamma^{(\text{rating})}_u \cdot \gamma_i$ and also price compatibility via $\gamma^{(\text{price})} \cdot \gamma_i$; compatible items are then those that satisfy both of these concerns. This idea is similar to the use of shared parameters for social recommendation as in \Cref{fig:sorecIdea}.
%\done{If make shared parameter figure, could reference here}
%\done{Rough paragraph, doesn't really fit anywhere. It's okay}
%\done{Add moral of the story about the point of disentangling price / preference concerns. Think it's obvious enough}

\subsection{Estimating Willing-to-Pay Prices within Sessions}
\label{subsec:willing2pay}
%\done{This really belongs in the above section since it's basically the same idea. I think it's okay}

\cite{hu2018learning} also considered the effect that price has on users' purchasing decisions, but did so at the level of individual browsing sessions. That is, the sequence of products a user browses might provide some clue as to their purchase intent or their `willing-to-pay' amount, e.g.~if they are comparison shopping among items within a certain price range.

Like \cite{ge2011cost}, the basic model of \cite{hu2018learning} extends a latent factor
%\done{latent-factor or latent factor?}
model to incorporate a price-compatibility term:
\begin{equation}
    f(u,i) = \gamma_u \cdot \gamma_i + \alpha_u C(u, p_i).
\end{equation}
Here $C(u, p_i)$ encodes the compatibility between the user $u$ and the price $p_i$ of the item $i$, and $\alpha_u$ is a personalized measure of user $u$'s sensitivity toward this term.

A trivial price-compatibility term might take the form
\begin{equation}
    C(u, p_i) = \exp\left( -\omega (b_u - p_i)^2 \right)
\end{equation}
%\done{is this the correct omega (lowercase)? check paper. Yes}
where $p_i$ is the price of the item and $b_u$ is a latent estimate of the user's budget; $\omega$ controls the bandwidth of the compatibility function. This function is essentially a variant of \Cref{eq:price1}.

To incorporate session dynamics, \cite{hu2018learning} model 
%a 
price-compatibility 
%term based on 
in terms of
a feature vector $\rho_i$, which is a one-hot encoding representing the 
%price
\emph{quantile} of the price $p_i$ compared to the prices of previously-viewed items in the session;\footnote{For example, $\rho_i = [0,0,0,1]$ would indicate that the price $p_i$ was among the top 25\% of browsed prices.} then the price-compatibility is merely $C(u, p_i) = \theta \cdot \rho_i$. This is further extended by using a \emph{mixture model}, which essentially says that there could be different parameters $\theta_g$ for different (latent) users groups:
\begin{equation}
    C(u, p_i) = \sum_g \underbrace{\frac{e^{\psi_{u,g}}}{\sum_{g'}e^{\psi_{u,g'}}}}_{\mathclap{\text{extent to which $u$ belongs to group $g$}}} \overbrace{\theta_g \cdot \rho_i}^{\mathclap{\text{price-compatibility model for group $g$}}}.
\end{equation}

%What 
\cite{hu2018learning} ultimately find that there are several different classes of user (based on latent group membership $\psi_{u,g}$): some tend to gradually browse toward more expensive items, others gradually browse toward cheaper items, 

%Cost-Aware Collaborative Filtering for Travel Tour Recommendations


%\done{put all tables near where they're referenced, or somewhere else good}

\subsection{Price Sensitivity and Price Elasticity}
\label{subsec:sensElas}

Notions such as \emph{price sensitivity}, and \emph{price elasticity} (defined as the change in purchase quantity given a change in price) are well understood in economics and marketing \citep{case2007principles}. Understanding such factors can help to guide custom marketing and promotion strategies \citep{zhang2004customizing,zhang2009effectiveness}.

However they are less well understood in terms of their effectiveness in a \emph{predictive} setting, i.e.,~in terms of how price should be used to understand and forecast user actions (or to make recommendations).

Part of the reason that price has received relatively little attention (at least in academic literature) is presumably the lack of useful available data; even when price is observed, it is confounded by numerous other factors, such as brand or manifest aspects of a product; moreover even when price data is available, one rarely has \emph{historical} data on price that allows for measurement of the impact of a \emph{change} in price.

\cite{wan2017modeling} studied price in the context of grocery recommendation. Their research was mostly based on real transaction data from a physical grocery store (in Seattle), though was also validated based public data from \emph{dunnhumby}. Both datasets contain price measurements, and critically measurements of price \emph{variation} over time. As such the main questions center around the extent to which purchase decisions are affected by changes in price.

In the context of grocery shopping, potential questions include:
\begin{itemize}
    \item Will a reduction in price cause a user to buy a \emph{category} of product they otherwise wouldn't have (e.g.~would they buy milk at a discount if it wasn't on their shopping list)?
    \item Will a reduction in price cause users to buy a \emph{specific item} that they otherwise wouldn't have (e.g.~would they buy a different brand of milk because of a discount)?
    \item Will a reduction in price cause users to buy a \emph{larger quantity} of an item than they otherwise would have?
\end{itemize}
To study these questions, prediction is broken down into three subsequent choices:
\begin{multline}
    p_u(\text{buy $q$ units of an item $i$ from category $c$}) = \\
    \begin{tabular}{l}
      $\hphantom{\times} p^{(\text{category})}_u(\text{buy a product from category $c$})$ \\
      $\times p^{(\text{item})}_u(\text{buy product $i$\ } | \text{\ buying from category $c$})$\\
      $\times p^{(\text{quantity})}_u(\text{buy $q$ units\ } | \text{\ buying item $i$})$.
    \end{tabular}
\end{multline}

Each of these three prediction tasks (category, item, and quantity prediction) is based on a predictor $f(u,c,t)$, $f(u,i,t)$, and $f(u,q,t|i)$; the underlying method behind each is a latent factor model, as in \Cref{eq:MFscore}, including additional features associated with the time (e.g.~what day of the week the trip occurs on). Each is passed through a different activation function, for example quantity prediction is modeled via a \emph{Poisson} function:
\begin{equation}
    p(\text{quantity} = q | \text{buying item $i$}) = \frac{f(u,q,t|i)^{q-1}\exp(-f(u,q,t|i))}{(q-1)!}.
\end{equation}

Next, the change in purchase probability due to price is captured (for each of the three models) using a simple feature encoding the price at a particular point in time. Specifically (e.g.~for quantity)
\begin{equation}
    f'(u,q,t|i) = \underbrace{f(u,q,t|i)}_{\mathllap{\text{$u$'s compatibility with $q$ units of item $i$ at time $t$}}\hspace{-20mm}} + \beta_{u,q} \log \underbrace{P_i(t)}_{\mathclap{\text{price of item $i$ at time $t$}}},
\end{equation}
where $P_i(t)$ is the price of the item at time $t$. $\beta_{u,q}$ is 
%then a bias term 
a coefficient
%which encodes 
encoding
the price sensitivity, i.e.,~the extent to which a particular user $u$, when purchasing $q$ units of an item, will react to changes in price (e.g.~a negative value of $\beta_{u,q}$ would indicate that a user is less likely to purchase a particular quantity given a price increase). All parameters are learned by training on purchase data, using a BPR-like training scheme.

%\done{Talk about WARP / WARP 2.0 loss (not here, I just rememebered). No}

% \emph{Price-elasticity} now reflects how much preference changes given a change in price, e.g.~for a particular item $i$:
% \begin{equation}
%     e_{i,u}(t) = \frac{df'(u,i,t)}{f(u,i,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx (1 - f'(u,i,t))\beta_{u,i}(t).
% \end{equation}
%\done{Redo this, fix notation, \beta is the output of some model, not a linear coefficient. Dropped}
% A related notion, \emph{cross-elasticity}, measures the extent to which a change in $i$'s price will change a user's compatibility toward (e.g.~probability of purchasing) a \emph{different} product $j$:
% \begin{equation}
%     e_{i,j,u}(t) = \frac{df'(u,j,t)}{f(u,j,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx -f(u,i,t)\beta_{u,i}(t).
% \end{equation}
%\done{check! same as paper}
% Compare to equality in https://mengtingwan.github.io/paper/www17_mwan.pdf
% (after eq 15)
%a(\sum_(j neq a) j)b_i  =   \sum_{j neq a} ja b_i

\emph{Price-elasticity} now reflects how much preference changes given a change in price, e.g.~for a particular item $i$; or 
% \begin{equation}
%     e_{i,u}(t) = \frac{df'(u,i,t)}{f(u,i,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx (1 - f'(u,i,t))\beta_{u,i}(t).
% \end{equation}
%\done{Redo this, fix notation, \beta is the output of some model, not a linear coefficient. Dropped}
% A related notion, 
\emph{cross-elasticity} measures the extent to which a change in $i$'s price will change a user's compatibility toward (e.g.~probability of purchasing) a \emph{different} product $j$.
% \begin{equation}
%     e_{i,j,u}(t) = \frac{df'(u,j,t)}{f(u,j,t)} \bigg/ \frac{dP_i(t)}{P_i(t)} \approx -f(u,i,t)\beta_{u,i}(t).
% \end{equation}
%\done{check! same as paper}
% Compare to equality in https://mengtingwan.github.io/paper/www17_mwan.pdf
% (after eq 15)
%a(\sum_(j neq a) j)b_i  =   \sum_{j neq a} ja b_i
Price-elasticity and cross-elasticity are \emph{measurements} after the model has been trained.
%
% Note that the above equations (price-elasticity and cross-elasticity) are \emph{measurements} after the model has been trained.
%\done{Crappy sentence. Fine}
The main finding of the model is that price-elasticity applies mostly to product choice, but not to category choice or quantity (i.e.,~a change in price may cause users to buy a \emph{different} brand of eggs, but will not cause them to buy eggs when they otherwise wouldn't have).
%\done{a few other findings}

%\done{Write f's as PROBABILITIES. Dropped}


%Others:
%Shopper: A probabilistic model of consumer choice with substitutes and complements (Ruiz, Athey, and Blei): focused on counterfactual queries about changes in price, and includes some study of complementary items. Somewhat similar to above, also for groceries.
%\cite{ruiz2020shopper}
\cite{ruiz2020shopper} develop a somewhat similar model of consumer choice, also in a setting of grocery purchases. Like the above model, \cite{ruiz2020shopper} attempt to disentangle the various effects of item popularity, user preference, and price dynamics, though also include additional terms involving seasonal effects. The main goal of the paper is to answer `counterfactual' queries about price (i.e.,~what would the user have done if the price had been different?).
%~in the presence of substitutable and complementary items.
%They do so by analyzing 
By modeling how users will react to changes in price, they argue that the model is also able to detect interactions between products, namely in terms of which items are likely to be substitutable and complementary.
%https://arxiv.org/pdf/1711.03560.pdf
%\done{Fine, but no mention of why I'm not talking about this paper more}
%
%
%\subsection{Optimizing Promotion Strategies}
%\done{include papers below. Quite limited}
%
%

Finally, we mention attempts to use 
similar ideas within the context of 
%designing customized promotions
dynamic pricing.
\cite{jiang2015redesigning} seek to combine ideas from 
%ideas from from 
pricing and recommendation in order to design optimal (i.e.,~profit-maximizing) promotion strategies.
%Redesigning promotion strategy for E-commerce Competitiveness through pricing and recommendation (also somewhat related to price sensitivity, attempts to use such ideas to design profit-maximizing promotions.
%
%An Empirical Analysis of Algorithmic Pricing on Amazon Marketplace. Attempts to empirically detect algorithmic pricing among best-sellers on Amazon, and analyses their characteristics.
Conversely,
\cite{chen2016empirical} analyze the characteristics of sellers on \emph{Amazon}, in order to automatically detect the presence of algorithmic pricing.
%\done{not the best, could drop. Otherwise check that it's accurate}

%\candrop{this}


\section{Other Contextual Features in Recommendation}
%\done{probably make subsection and reduce these to paragraphs. Nowhere to put though}

In this chapter, we have attempted a high-level treatment of the various ways features can be incorporated to develop richer models of interaction data. Naturally, 
%a complete presentation of all feature modalities is impossible, 
we cannot give a complete presentation of all feature modalities,
and as such we've only sought to cover some of the most common scenarios. Below we briefly survey a few of the other main directions for content-based recommendation.

\subsection{Music and Audio}

A number of scenarios we'll explore later in the book involve interactions with music and audio data. 
%For instance, we'll see sequential modeling 
%\done{modeling vs modelling. Modeling}
%techniques for playlist generation in \Cref{chap:temporalSequential} (\cref{sec:PRME}), and case-studies on diversity of music preferences in \Cref{chap:consequence} (\cref{sec_?}).
%Two specific scenarios we'll explore later include (1) 
First,
interactions with music have significant sequential context, i.e.,~the next interaction or recommendation should relate to the characteristics of the previous song (\cref{chap:temporalSequential}, \Cref{sec:PRME}).
%\done{is music actually mentioned in sec:PRME? Yes}
Second, music recommendations must carefully balance familiarity and novelty (\cref{chap:consequence}, \Cref{sec:spotify}). 
%The latter concerns tie into our broader discussion 

%\section{Music}

Given our 
%specific 
focus
%interests 
in this chapter, we 
%focus briefly on the 
briefly explore the
specific semantics of interaction data and the challenges involved in building content-aware models. We refer to e.g.~\cite{celma2008music} for a deeper survey of music recommendation.

Interaction signals in music can be quite different from those associated with the types of data we've seen so far (e.g.~ratings and purchases). Feedback associated with music is often \emph{implicit}, and can be weak and noisy: for instance we may know as little as whether a user finished a song or whether they skipped it (see e.g.~\cite{pampalk2005dynamic}). Handling such signals is difficult as they do not obviously map to `positive' and `negative' signals.
%implicit feedback

Unlike most of the data we've studied so far, music interactions are also highly driven by \emph{repeat consumption} (see e.g.~\cite{anderson2014dynamics}). This requires specific techniques to understand under what conditions users might seek novelty versus more familiar options.
%
%repeat consumption




% Thesis: Music recommendation and discovery in the long tail
%http://www.mtg.upf.edu/static/media/PhD_ocelma.pdf

%\done{Does this belong here, or should put it in sequential recommender systems section? Or incorporate audio features into the visual recommendation chapter? Is fine}

% Features derived from music

% Interaction patterns such as skips and completed listens



%https://dl.acm.org/doi/pdf/10.1145/2647868.2654940?casa_token=UtHgvyXPnEIAAAAA:wVMpUDix7muDjz3gpw_pq1PIxExffT9gjXxZBf8oymBo8Dydpj6qBfvO1xjD7CNoqKTz3PbediLn4A
%Quite straightforward  - item embedding is learned from DNN

Extracting useful features from audio is also challenging.
\cite{wang2014improving} note the difficulty of directly using high-dimensional audio features (e.g.~based on spectrogram-based representations of audio) within a traditional feature-based recommender system. The solution they propose is to use a neural network-based representation (essentially a multilayer perceptron) to learn embeddings of songs that are useful for recommendation, i.e.,:
\begin{equation}
    \gamma_i = \mathit{MLP}(x_i).
\end{equation}

%Deep content-based music recommendation
\cite{van2013deep} adopt a similar approach 
%to content 
%based encoding of music, though their approach is 
based on a convolutional neural network (CNN) which operates on an audio \emph{spectrogram}. Spectrograms are two-dimensional time/frequency representations of audio, so the approach is methodologically similar to using CNNs to develop content-aware models from images. We forego detailed discussion but note the similarity to image-based recommendation approaches such as those we develop in \Cref{chap:visual}.
%\done{repetitive? "forego". Fine}
%https://papers.nips.cc/paper/2013/file/b3ba8f1bee1238a2f37603d90b58898d-Paper.pdf


%We forego detailed presentation of this type of technique 

% Improving Content-based and Hybrid Music
% Recommendation using Deep Learning
% \cite{wang2014improving}

Like other recommendation domains, research in music recommendation has partly been driven by data. Popular music datasets include the \emph{Million Song Dataset} and the \emph{Taste Profile Dataset} \citep{mcfee2012million}, containing rich audio and interaction data, though various studies also make use of proprietary sources including data from \emph{YouTube}~\citep{anderson2014dynamics} and \emph{Spotify}~\citep{anderson2020algorithmic}.

% Million song dataset
% %\cite{bertin2011million}
% \cite{mcfee2012million}





\subsection{Recommendation in Location-Based Networks}
\label{subsec:location}
%\done{And something else...? No}



%https://link.springer.com/content/pdf/10.1007/s10707-014-0220-8.pdf

%\done{Refer to this section from location section in sequential modeling}
%\done{and maybe from first project? No}

Several attempts have been made to incorporate geographical features into recommender systems. Actions are often guided by geographical constraints, whether due to a user operating within a certain geographical region, or due to sequential actions being highly localized.
%\done{bad opening sentence. Fine}

\cite{bao2015recommendations} survey attempts to model recommendations in \emph{location-based social networks}, and highlights some of the main ideas and challenges in modeling this type of data:
\begin{itemize}
    \item Locations are often used as a \emph{contextual} feature, for example, users are likely to visit places (restaurants, hotels, landmarks, etc.)~in the vicinity of those they have recently visited. We explore this type of assumption in the context of sequential recommendation in \Cref{chap:temporalSequential} (\cref{sec:PRME}).
    \item Appropriately extracting features from location data is difficult due to its hierarchical nature. For example, a restaurant belongs to a neighborhood, a city, a state, and a country. Strategies for extracting useful hierarchical representations from GPS data are covered in (e.g.)~\cite{zheng2009mining}.
    \item Recommendation in such settings varies in terms of the data and goals involved. In addition to geo-tagged activities, data may include profile information, relations among users, etc.~\citep{cho2011friendship}.
    \item Other familiar issues, such as cold-start, %scenarios, 
    can be amplified in location-based data, where user activities can be sparse and can grow rapidly.
\end{itemize}

As such, successful solutions to modeling in location-based social networks can incorporate several features, including those we've seen so far, e.g.~how to leverage social connections (as in \cref{sec:groupSocialRec}), and those we'll see in later chapters, such as sequential dynamics. \cite{bao2015recommendations} also highlight a number of data sources that are commonly used for the study of recommendation in location-based networks, including (e.g.)~\emph{Brightkite}, \emph{Gowalla}, \emph{Foursquare}, etc.



%\done{modeling vs modelling. Modeling}


\subsection{Temporal, Textual, and Visual Features}

So far we have covered 
content in recommender systems, 
including features ranging from price, geography, social signals, and audio.
%Segue into next chapter
%\done{Pretty useless paragraph}
In the following chapters we'll revisit content-aware recommendation techniques using features based on temporal and sequential dynamics (\cref{chap:temporalSequential}), text (\cref{chap:text}), and images (\cref{chap:visual}). These modalities require special attention, primarily to deal with the complex semantics of the data and high-dimensional signals involved. Exploring how to build personalized models based on these complex modalities of data is one of the main themes explored throughout the remainder of this book.





\section{Online Advertising}
\label{sec:advertising}

% \subsection{What Makes Ad Recommendation Different?}

Superficially, surfacing advertisements to users seems no different from any other form of personalized recommendation. That is, we can imagine learning user `preferences' and advertisement `properties' from (e.g.)~clicked advertisements in much the same way that any other recommender system is trained.

While ad recommendation does indeed have many similarities to other forms of personalized recommendation, 
%However as we discuss in this chapter, 
there are several properties that demand different solutions compared to what we have seen so far. In particular:
\begin{itemize}
 \item Advertisers have budget constraints. In many recommendation settings we can tolerate considerable imbalance among the items that are recommended (e.g.~a highly popular movie might be recommended to a substantial fraction of all users); this is impossible when recommending ads, given that each advertiser can afford to surface only a limited number of ads (and at the same time, we want to ensure that all advertisers have some ads recommended).
 \item Likewise, each user may only be shown a limited number of ads; while this seems a common enough feature in most recommendation scenarios, it is especially apparent when surfacing ads, as users are unlikely to explicitly request additional ads.
 \item Ad recommendations need to be made immediately. Again, this feature is common enough in many recommender systems, but is especially challenging given the considerations above: we cannot find a globally optimal solution that maximizes utility while satisfying advertisers' budget constraints. Instead, we must develop schemes that make local decisions in a way that approximates the globally optimal solution.
 \item Ad recommendation is highly contextual. Whereas most recommender systems rely heavily on user interaction histories, these are presumably less reliable in ad recommendation scenarios (where users' interactions with ads are extremely sparse), if an interaction history is available at all. As such, one has to rely more heavily on user \emph{context} (e.g.~a user's query to a search engine).
 \item Even if a user is responsive to a certain type of ad, there is diminishing value in repeatedly showing similar ads. Instead, we must sometimes recommend ads with low expected utility in the hope of discovering new user interests (this is the basic principle behind an \emph{explore/exploit} tradeoff).
\end{itemize}

%These are the problems we will investigate in order to build systems for online advertising.
%\done{introduction}

%These are the problems we will investigate in order to build systems for online advertising.
We'll discuss a few of these issues as we briefly investigate systems for online advertising below.
%
Note that in this section we will mostly ignore the question of how `compatibility' between a user and an ad is estimated---this could itself be the output of a recommender system, or could simply be a bid that an advertiser places on a user or query.
Our goal in this section is mostly to 
%introduce the concept of \emph{matching} in 
explore
situations where we have constraints in terms of how many recommendations can be received by a user (or given of an item), and highlight some of the general differences and strategies used to build ad recommenders.

\subsection{Matching Problems}
\label{sec:matching}

%First we will consider the problem of incorporating \emph{matching} constraints into (ad) recommendation problems.
A typical constraint in ad recommendation 
%settings is that
%We want to consider situations where 
%First we consider \emph{matching} constraints, 
is what is known as a \emph{matching} constraint, where
each user can only be shown a fixed number of ads, and each ad can only be shown to a fixed number of users. 
%This is already an oversimplification, as advertisers would usually have a 
% We ignore issues such as as advertiser
% \emph{budgets} and \emph{bids} for now,
%and may (for example) bid different amounts per user or %query (
% though we consider these issues in \Cref{subsec:adwords}.

To begin with, we'll consider the case where each ad is shown to exactly one user, and each user is shown exactly one ad. That is, we would like to select a function $\mathit{ad}(u)$ that maps users to ads, such that
\begin{equation}
    \mathit{ad}(u) = \mathit{ad}(v) \rightarrow u = v.
\end{equation}
Alternately, we could write this as an adjacency matrix $A$ such that $A_{u,a} = 1$ if $\mathit{ad}(u) = a$. Then our constraint could be written as
\begin{equation}
    \underbrace{\forall a \sum_u A_{u,a} = 1}_{\text{each advertiser shows one ad}}; \quad \underbrace{\forall u \sum_a A_{u,a} = 1}_{\text{each user sees one ad}}.
\end{equation}
%\done{realign braces with \mathclap etc. Looks okay}
Then, we would like to choose the mapping that maximizes the utility between users and the ads they are shown:
\begin{equation}
    \max_A \sum_{u,a} A_{u,a} f(u,a)
    \label{eq:matchingObj}
\end{equation}
where $f(u,a)$ is a measure of the compatibility between a user and an ad (e.g.~the output of a recommender system, a click probability, etc.).

This type of problem is known as a \emph{matching problem}. Conceptually, it can be viewed as matching two sets of nodes to form a bipartite graph (\cref{fig:matching}, left), where each possible edge has an associated weight (i.e.,~the compatibility between a user and an ad). Our constraint above now says that every node should be incident on one edge.

\begin{figure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}

\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label=left: 1, label={[label distance=2mm, rectangle]90:users:}] {};
\foreach \i in {2,3,...,6}
  \node[fsnode,on chain] (f\i) [label=left: \i] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label=right: 1, label={[label distance=2mm, rectangle]90:ads:}] {};
\foreach \i in {2,3,...,6}
  \node[ssnode,on chain] (s\i) [label=right: \i] {};
\end{scope}

\draw (s1) -- (f2);
\draw (s2) -- (f5);
\draw (s3) -- (f1);
\draw (s4) -- (f4);
\draw (s5) -- (f6);
\draw (s6) -- (f3);
% \draw (s9) -- (f5);
% \draw (f5) -- (s6);
\end{tikzpicture}
\hspace{4mm}
\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label=left: 1, label={[label distance=2mm, rectangle]90:users:}] {};
% \foreach \i in {2,3,...,6}
%   \node[fsnode,on chain] (f\i) [label=left: \i] {};
\node[fsnode,on chain] (f2) [label=left: 2] {};
\node[fsnode,on chain] (f3) [label=left: 3] {};
\node[fsnode,on chain] (f4) [label=left: 4, fill=white] {};
\node[fsnode,on chain] (f5) [label=left: 5, fill=white] {};
\node[fsnode,on chain] (f6) [label=left: 6, fill=white] {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label=right: 1, label={[label distance=2mm, rectangle]90:ads:}] {};
\foreach \i in {2,3,...,6}
  \node[ssnode,on chain] (s\i) [label=right: \i] {};
\end{scope}

\draw (s5) -- (f3);
\draw (s1) -- (f2);
\draw (s3) -- (f1);
\end{tikzpicture}
    \caption[Ad recommendation as bipartite matching.]{Ad recommendation can be viewed as a \emph{bipartite matching problem} (left), where users are shown a fixed number of ads, and each ad is shown to a fixed number of users (in this figure each ad is shown to exactly one user, and \emph{vice versa}). In an online setting (right), users arrive one at a time; we seek a solution that will be as close as possible to the solution we would have obtained in an offline setting at left.}
    \label{fig:matching}
\end{figure}

Outside of ad recommendation, this type of matching problem appears in many settings, for example it is equivalent to the US's \emph{National Residency Matching Program}, in which medical school students are matched to residency programs: each student can only be matched to a single residency, and each program has a limited number of slots; the matching should be chosen so as to optimize students' preferences for programs (and \emph{vice versa}). Similar problems appear in various resource allocation settings \citep{gusfield1989stable}; the original paper proposing the solution outlined below considered a setting related to college admissions \citep{gale1962college}.
%\done{not beautiful writing. Looks fine}

Although \Cref{eq:matchingObj} is a combinatorial optimization problem, it admits efficient approximation (the so-called `stable marriage' problem \citep{gusfield1989stable}) and a polynomial exact solution (see \cite{kuhn1955hungarian}).
%\done{abrupt ending, but okay}

%The approximation we discuss below is known as \emph{stable marriage}, which draws an analogy between bipartite matching and finding (heterosexual) marriage partners: we seek a solution which, while not necessarily optimal, is `stable,' in the sense that no unmatched pair would have an incentive to `cheat.'

%For each man/woman pair $m$ and $w$, we assume a compatibility $f(m,w)$.\footnote{The original solution assumed \emph{different} compatibility functions for men toward women, and women toward men, though we discard this for the application discussed here.} `Stability' then means that there should be no pairs $(m,w)$ and $(m,w')$ such that $m$ and $w'$ are matched but $f(m,w) > f(m,w')$.
%\done{confusing writing. Dropped}

%and initially assume that there are an equal number of men and women (and that everyone gets married):
%\done{put this somewhere}

% Pseudocode for the stable marriage problem is given below:
% \begin{itemize}
%     \item All men and all women are initially ‘free’ (i.e.,~not engaged)
%     \item while there is a free man $m$, and a woman he has not proposed to
%     \begin{itemize}
%         \item $w = \max_w f(m,w)$
%         \item if ($w$ is free):
%         \begin{itemize}
%             \item $m$ and $w$ become engaged (and are no longer free)
%         \end{itemize}
%         \item else ($w$ is engaged to $m'$):
%         \begin{itemize}
%             \item if $w$ prefers $m$ to $m'$ (i.e.,~$f(m,w) > f(m',w)$):
%             \begin{itemize}
%                 \item $m$ and $w$ become engaged
%                 \item $m'$ becomes free
%             \end{itemize}
%         \end{itemize}
%     \end{itemize}
% \end{itemize}
%\done{actual pseudocode environment, and above is crappy. Drop}
%\done{algorithm is basically the wikipedia one, but where does it really come from? Drop}

% Theorems:
% \begin{itemize}
%     \item The algorithm eventually terminates; each step includes a new proposal, and there are only $n^2$ possible proposals. Furthermore, each iteration improves the matching score of \Cref{eq:matchingObj} (or does nothing).
%     \item The solution is stable. The proof is relatively straihtforward: one assumes an unstable solution in which 
%     %$m$ is married to $w'$ but would prefer to be married to $w$; 
%     $m$ and $w$ would prefer to be married to each other rather than their current spouses $m'$ and $w'$;
%     in this case $m$ must have proposed to $w$ first (since he always proposes to his most compatible match first); so either $w$ was already engaged to somebody preferable, or $w$ accepted and later broke off the engagement; neither of these two scenarios could have ocurred if $m'$ is less compatible.
%     \item Note that a consequence of the above is the possibly surprising conclusion that every matching problem as a stable solution.
%     \item The solution requires $O(n^2)$ steps, since each step includes a proposal, and there are at most $n^2$ possible proposals. In practice the running time is often substantially faster.
%     %\done{What about sorting the ranked lists for each man? Dropped}
%     %\done{Cite for O(nlogn) running time? Dropped}
%     \item The solution is not guanarteed to be optimal, and the optimal solution is not guaranteed to be stable. Examples of both are given in \Cref{fig:matchingOpt}.
% \end{itemize}


% \begin{figure}
%         \centering
% \definecolor{mygrey1}{RGB}{160,160,160}
% \definecolor{mygrey2}{RGB}{120,120,120}

% \begin{tikzpicture}[thin,
% %  every node/.style={draw,circle},
%   fsnode/.style={fill=mygrey1},
%   ssnode/.style={fill=mygrey2},
%   ->,shorten >= 3pt,shorten <= 3pt
% ]

% % User vertices
% \begin{scope}[start chain=going below,node distance=8mm]
% %\node[fsnode,on chain,draw,circle] (f1) [label=left: 1, label={[label distance=0.7mm]90:women:}] {};
% \node[fsnode,on chain,draw,circle] (f1) [label=left: 1,label={[label distance=2mm, rectangle]90:women:}] {};
% \node[fsnode,on chain,draw,circle] (f2) [label=left: 2] {};
% \end{scope}

% % Ad vertices
% \begin{scope}[xshift=2cm,start chain=going below,node distance=8mm]
% %\node[ssnode,on chain,draw,circle] (s1) [label=right: 1, label={[label distance=2mm]90:men:}] {};
% \node[ssnode,on chain,draw,circle] (s1) [label=right: 1,label={[label distance=2mm, rectangle]90:men:}] {};
% \node[ssnode,on chain,draw,circle] (s2) [label=right: 2] {};
% \end{scope}
% %\done{Hardcoded vertical spacing of men/women NOT CORRECT: label. Dropped}

% % \node[above=of f1] (w1) {women};
% % \node[above=of s1] (m1) {men};

% \draw[dotted,thick] (s1) edge[-] (f1);
% % \node [pos=0.66,above=s1] {Transpose};
% \draw[thick] (s1) edge[-] (f2);
% \draw[thick] (s2) edge[-] (f1);
% \draw[dotted,thick] (s2) edge[-] (f2);




% % \draw (s9) -- (f5);
% % \draw (f5) -- (s6);
% \end{tikzpicture}

%     \caption{Examples of matching solutions that are stable but not optimal (left), and optimal but not stable (right).
%     %\done{figure. Dropped}
%     \label{fig:matchingOpt}
%     }

% \end{figure}



%tikz:
%https://tex.stackexchange.com/questions/15088/bipartite-graphs

\subsection{AdWords}
\label{subsec:adwords}

%So far, we have 
Above we
discussed the issue of developing recommendation approaches that consider some notion constraints or `budgets' for the perspective of users and advertisers. While incorporating constraints is potentially useful in a variety of recommendation scenarios, our solution still doesn't fully address the setting of \emph{ad} recommendation; in particular, when recommending ads, we are unlikely to see the entire set of users (or queries) in advance, before having to select advertisements.

As such, we desire algorithms that make decisions (i.e.,~assigns ads to users) one at a time, while still conforming to matching (or budget) constraints, as in \Cref{fig:matching} (right).

Formally, the above describes the distinction between an \emph{offline} and an \emph{online} algorithm, i.e.,~one which sees the entire problem in advance, versus one which must make predictions (or update model parameters) immediately in response to new interactions. We discussed this type of setting briefly in \Cref{sec:onlineUpdates}.

\emph{AdWords} \citep{mehta2007adwords} is a specific instance of this type of recommendation problem developed for \emph{Google's} online advertising platform.

Mostly, the setting follows the one we described above, though includes a few additional components. Specifically,
\begin{itemize}
    \item Each advertiser $a$ has a \emph{bid} $f(q,a)$ that they are willing to make for each query $q$.
    \item The bid generally refers to how much the advertiser will pay \emph{if} the ad is clicked on; this is determined by an estimated \emph{click-through-rate}, $\mathit{ctr}(q,a)$. As such the expected profit would be $f(q,a) \times \mathit{ctr}(q,a)$, which one can think of as being analogous to 
    %the compatibility from 
    user-to-item compatibility (and could be estimated using a model such as one from \Cref{chap:modelbasedRecsys}).
    %\done{edit and check. Fine}
    %\Cref{sec_sectionAbove}.
    \item Each advertiser has a \emph{budget} $b(a)$ (e.g.~for a one-week period).
    \item As in \Cref{sec:matching}, there is a limit on the number of ads that can be returned for each query.
\end{itemize}


Of course the actual implementation of AdWords contains many features not described here, for instance Adwords
%\done{italics}
uses a second-price auction (the winning advertiser pays the amount that the \emph{second} highest bidder bid), and advertisers don't bid on exact queries, but rather are matched using a `broad matching' criteria that can include subsets, supersets, or synonyms of keywords being bid on. We refer to \cite{rajaraman2011mining} for further description of these details, and \cite{mehta2007adwords} for a more detailed technical description.

% \subsection{Reinforcement Learning and Bandit Algorithms}

% So far, when developing matching algorithms in \Cref{sec:matching} and AdWords in \Cref{subsec:adwords},
% %\done{anything else? No}
% we've assumed that a compatibility function between between users and ads was available in advance, and merely considered the question of how ads should be \emph{allocated} to users based on budgets and the given compatibility.

% However, ad recommendation (among other settings) could present unique challenges in terms of how we estimate compatibility in the first place.

% Compared to other recommendation settings, it is unlikely that we could learn user-to-ad compatibility by collecting historical data of ads users have previously interacted with. Some potential issues include:
% \begin{itemize}
%     \item Interaction data is extremely sparse; users likely interact with only a tiny fraction of the advertisements that are surfaced to them, making positive feedback extremely rare.
%     \item Even if a user is interested in a particular topic, there are diminishing returns in repeatedly surfacing similar ads (e.g.~a user searching for rental tuxedos likely remains interested in the topic for a short time).
%     \item Users do not `seek out' ads from which we might learn their interests. Thus we must sometimes surface ads before we have collected evidence that a user is interested in them. 
%     \item Recommendation is highly contextual. E.g.~an ad for rental tuxedos is only worth surfacing if a user is currently searching for a closely related topic.
% \end{itemize}

% %\section{Case Study?}

% % \section{Bandit Algorithms and Reinforcement Learning}

% % A Contextual-Bandit Approach to Personalized News Article Recommendation \cite{li2010contextual}


% The basic setup of a \emph{contextual bandit} (following e.g.~\cite{li2010contextual}) is as follows.
% \begin{itemize}
% \item 
% During each step $t$, we observe a user $u_t$ and a set of actions (or `arms') $\mathcal A_t$. The actions $a \in \mathcal A_t$ reflect, for example, the space of possible advertisements that could be surfaced to a user during a particular timestep.
% %
% \item 
% In the setting of a contextual bandit, each timestep $t$ and action $a$ is associated with a feature vector $x_{a,t}$ representing the current \emph{context}; $x_{a,t}$ is intended to capture any features associated with the user and the action at a particular point in time (which could include information about their search query, etc.).
% %
% \item
% During every timestep, an action is selected, following which a reward $r_{a,t}$ is received. This reward might measure (e.g.)~whether the user clicked on the surfaced advertisement, or the payout received from doing so. Critically, the reward is only observed \emph{for the action $a$ that was selected}, that is, we cannot observe whether the user would have clicked on an ad that \emph{wasn't} shown.
% %
% \item
% Over time, the goal is to devise a strategy for selecting actions in such a way that the total reward is maximized. 
% \end{itemize}

% We avoid full treatment ... though refer to solutions based on RL ... conversational rec.
%\done{dropped}


%\end{itemize}
%\done{itemize or not? Yes I think looks better with items}


% \begin{equation}
%     E(r_{a,t}|x_{a,t}) = x_{a,t} \cdot \theta_u
% \end{equation}

% \subsection{News Recommendation}
% %\done{detached section, maybe drop. Dropped}
% Doesn't exactly go here but might go somewhere?

% Turning down the noise

% News search engines and the challenge to traditional journalistic
% roles



% \section{User Interfaces for Recommendation}
%\done{detached section, maybe drop. Dropped}
% Controlling Spotify Recommendations:
% Effects of Personal Characteristics on Music Recommender User
% Interfaces \cite{jin2018effects}
%file:///home/jmcauley/Downloads/UMAP2018__draft.pdf



% \section{Conversation and Explanation}

% \subsection{Conversation}

% Improving recommender
% systems with adaptive conversational strategies
%Early paper

% Enhancing the diversity of conversational collaborative recommendations: a comparison

% Towards Conversational Recommender Systems
%https://www.kdd.org/kdd2016/papers/files/rfp0063-christakopoulouA.pdf
%Is this the paper I'm thinking of that introduces the dataset?

% A Ranking Optimization Approach to Latent Linear Critiquing in Conversational Recommender System


%
%Content-based recommendation systems, Pazzani paper
%\done{Maybe cite somewhere (not here). Not important}

% \section{Deployment Considerations}
%Active learning in recommender systems, Recommender Systems Handbook
%\done{Maybe name section "Interactive Recommendations" or similar. No}






% \section{Exercises}

\begin{exercises}
 \item %Fit a factorization machine with various features
 In \Cref{sec:factorization} we introduced the factorization machine  as a general-purpose technique for incorporating features into recommender systems. In this exercise we'll incorporate a few features into a factorization machine to measure the extent to which they improve recommendation performance. \emph{You could use any dataset for this exercise, so long as it includes a few features.} For example, using our beer dataset (as in \cref{sec:simpleTransforms}) we might include features such as:
 \begin{itemize}
     \item Simple numerical features such as the ABV of the beer or the age of the user;
     \item The category of the item (one-hot encoded);
     \item The timestamp; this will require some care to encode, e.g.~you might use an encoding of the season or day of week.
 \end{itemize}
 %\done{Why don't bullet points look the same as in other exercises? Look fine}
 Using a few of these features (or similar features on another dataset), measure the extent to which 
 %performance is improved by adding features to the recommender.
 they boost recommendation performance over a model which does not include them.\footnote{Do not be surprised if the improvement is minimal; we'll investigate this more in Exercise~\ref{ex:FM2}.}
 %\done{reword a little}
 \label{ex:FM1}
 \item 
 \label{ex:FM2}
 In \Cref{sec:coldStart}  we discussed the potential value of incorporating features (side information) into recommender systems, and argued that features may be most informative in \emph{cold-start} scenarios. To assess this, conduct the following experiment (using your model from Exercise~\ref{ex:FM1}):
 \begin{itemize}
     \item For each user (or item) in the test set, count how many times that user appeared in the training set;
     \item Plot the testing performance (y) of your model as a function of how many times that user appeared in the training set (x); this type of plot is shown in \Cref{fig:rmseNtrain};
     \item Generate the same plot both with and without additional features in your factorization machine.
 \end{itemize}
 Naturally, our expectation is that performance will improve as users (or items) are less `cold' (i.e.,~more training interactions). However, we expect the performance degradation to be more mild when features can compensate for a lack of historical interactions.
 \item 
 \label{ex:sorec1}
 Before implementing a socially-aware recommender, 
 %here we'll 
 test the hypothesis that friends actually interact with more (or less) similar  items compared to randomly chosen sets of users. 
 Use any dataset that contains social interactions (such as those in \Cref{tab:socialRec}).
 This 
 %could be measured 
 hypothesis could be tested
 in various ways, e.g.~by computing the average Jaccard (or cosine, etc.)~similarity between randomly chosen pairs of users, versus randomly chosen pairs of friends.
%  \begin{itemize}
%      \item Compute the average Jaccard (or cosine, etc.)~similarity between randomly chosen pairs of users, versus randomly chosen pairs of friends.
%      %\done{only a single item?}
%  \end{itemize}
 \item In \Cref{sec:groupSocialRec} we explored a few ways to incorporate social signals into recommender systems.
 %, and in particular one-class recommender systems (\cref{sec:socialBPR}).
 %\done{Do I call them "one class" or something else? Removed mention of one-class}
 For the most part,
 %\done{is it most or all of them? Doesn't matter}
 these techniques amounted to ways of sampling negative feedback instances, e.g.~we might be more 
 %(\cref{eq:sbpr1}) 
 or less 
 %(\cref{eq:sbpr2}) 
 likely to interact with negative instances that our friends have already interacted with (\cref{eq:sbpr1,eq:sbpr2}). 
 %(e.g.~\ref{_}, as in \Cref{sec_?}) 
 Experiment with these sampling strategies (e.g.~using the same dataset from Exercise~\ref{ex:sorec1}) and determine which (if any) lead to improved performance over a traditional BPR implementation.
 %\done{BPR defined or do I always write it out? Should I here? Yes, it's used a bit}
%  \item Incorporate price into some dataset (amazon?) in a few ways
\end{exercises}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/rmseNtrain}
\caption[Performance demonstration in cold-start settings.]{Performance (MSE) as a function of user coolness (defined as the number of times that user was seen during training). \emph{Goodreads} `Graphic Novels' data.
\label{fig:rmseNtrain}
}
%\done{style}
%\done{refer to figure}
\end{SCfigure}


\refstepcounter{projectCounter}
\label{proj:coldStart}
\section*{Project \theprojectCounter: Cold-Start Recommendation on \emph{Amazon}}
\addcontentsline{toc}{section}{Project \theprojectCounter: Cold-Start Recommendation on \emph{Amazon}}
%\done{Replace with a specific category}

As we argued in \Cref{sec:coldStart},
%\done{did we? Yes}
one of the main reasons to incorporate features into personalized recommendation approaches is to improve their performance in \emph{cold-start} settings. Here, we'll look at cold-start recommendation problems using data from \emph{Amazon}, following data from (e.g.)~\cite{ni2019justifying}.
%\done{not ideal reference for dataset}
We select this dataset as it includes various types of item metadata (prices, brands, categories, etc.)~that can potentially be useful in cold-start settings, though this project could be completed using any dataset that includes user or item metadata. For this project you'll likely want to select a specific sub-category (e.g.~\emph{Musical Instruments}, as we saw in \Cref{sec:jaccard}) which has informative features, and which is of an appropriate scale to build your model.

We'll build our system for cold-start recommendation via the following steps:



\begin{enumerate}
    \item Start by implementing a `vanilla' factorization machine to solve the prediction problem, i.e.,~without incorporating any side-information.
    Note that this problem could be cast either as one of rating prediction (as in \cref{subsec:matrixFactor}) or as purchase prediction (as in \cref{sec:implicit}).
    %\done{Same text as next project. Is fine}
    \item To build models for cold-start recommendation, it is useful to develop some evaluation metrics specifically for the purpose of evaluating cold-start (and `cool-start') performance. To do so, try plotting the performance on the test set \emph{as a function of the number of times the item appears in the training set} (as in Exercise~\ref{ex:FM2})---this type of plot is shown in \Cref{fig:rmseNtrain} for a `standard' latent factor model. Our hypothesis when incorporating side-information into recommendations is that performance improvements will be largest for the coldest items, and less useful for items with longer interaction histories. We'll use this same type of plot to compare models in the following.
    %\done{Make plot of this, but for a normal model}
    \item Several features could potentially be useful in cold-start scenarios. Consider how to encode the following: (a) the brand of the item; (b) the category (or categories) the item belongs to; and (c) the price of the item. Features describing relationships among items (`people who viewed X also viewed Y,' etc.)~may also be useful (in much the same way as we considered user relationships in \Cref{sec:groupSocialRec}). 
\end{enumerate}

For each cold-start feature you include, compare the performance of the model with and without that feature.

It may be worth revisiting this project as we explore more complex feature modalities in the later chapters,
%You might also 
for example you might
consider more complex features based on the text of the product description, temporal information, or visual features. We'll revisit these topics in 
%\Cref{chap:temporalSequential,chap:text,chap:visual}.
\Crefrange{chap:temporalSequential}{chap:visual}.
%\done{Needs something else to finish the project. Fine}





\chapter{Temporal and Sequential Models}
\label{chap:temporalSequential}

%So far, we have
Throughout \Cref{chap:modelbasedRecsys,chap:contentStructure} we gradually developed more refined arguments about the role that features (or `side-information') play when modeling user interaction data. Our initial argument (started in \Cref{subsec:matrixFactor}) was that latent user and item representations are sufficient to capture complex preference dynamics, and whatever features are most predictive (i.e.,~explain the most variability) of interactions will automatically emerge via our latent representations.

Later, (e.g.~in \cref{sec:coldStart}) we refined our argument, 
%arguing 
noting
that side-information \emph{can} be useful, especially in settings where there is a paucity of interaction data from which to learn high-dimensional latent representations (i.e.,~the \emph{cold-start} setting).

However in both of the above cases we still assumed that our model of users and items was \emph{stationary}. In practice, preferences and interactions can be non-stationary for a variety of reasons. For instance, no matter how many interactions we observe, and how many latent factors we fit, 
%a model such as the one we developed in \Cref{sec_?} cannot tell us that 
the models we've developed so far would struggle to tell us that
a user might only buy swimsuits in summer, or that they are unlikely to watch the third film in a series until they have already seen the second.

In this chapter we explore techniques to build personalized models around user behavior that has \emph{temporal and sequential dependencies}. Most straightforwardly, we might simply treat temporal and sequential information as additional features that can be modeled, e.g.~via the framework of a factorization machine as we developed in \Cref{chap:contentStructure} (and indeed we will explore temporal models based on this type of approach). However as we will see in this chapter, a variety of complex and subtle temporal dynamics could be at play, for example:
\begin{itemize}
    \item Dynamics could apply to users (e.g.~users may `grow out' of certain movies), items (e.g.~a movie's special effects may become dated over time), or the \emph{zeitgeist} of the entire community may shift over time.
    \item Temporal dynamics exist at multiple scales, e.g.~when modeling heart-rate data (\cref{sec:caseStudyHR}), dynamics can be short-term (e.g.~a user running), medium-term (e.g.~a user getting tired) or long-term (e.g.~a user becoming more fit).
    \item In addition to short- and long-term dynamics, dynamics can also be periodic (e.g.~weekly or seasonal trends), or prone to bursts and outliers (e.g.~purchases around a major holiday).
\end{itemize}

Our main focus in this chapter is to explore a wide variety of (personalized) models for temporally-evolving data. As we'll see, dynamics such as those above require carefully-designed models (rather than simply including a temporal feature in a general-purpose model). As such, we focus on understanding the overall process and design considerations when building personalized models with temporal dynamics.

\section{Introduction to Regression with Time Series}
\label{sec:introTime}
%\done{This would be better in regression chapter, but probably only do that if regression + classification become two separate chapters. No. Too much in that chapter}
%\done{Note that actually sequential recommender systmes *are* autoregressive, so this is a perfectely reasonably place to discuss autoregression}

Before investigating more sophisticated models of temporal and sequential data, it is instructive to consider how much progress can be made with the techniques we have already developed.

To do so, we'll consider developing predictors that estimate the next value in a sequence (or the next several values). In the simplest case, we are given a sequence of observations $y = \lbrace y_1 \ldots y_n \rbrace$, from which we would like to predict the next value ($y_{n+1}$). For example, one might want to estimate website traffic on the basis of historical traffic patterns.

To solve such a problem using a regression approach (or a classification approach for a binary outcome), we can imagine constructing features based on the previous observations ($\lbrace y_1 \ldots y_n \rbrace$) in order to predict the next one. Presumably, once we observe the true value of $y_{n+1}$, we would like to predict $y_{n+2}$, and so forth. Note that this blurs the line somewhat between `features' and `labels,' since the label for one prediction becomes a feature for the next.

\paragraph{Autoregression}
This procedure is known as \emph{autoregression} (referring to the fact that we are regressing on the same data that was used for prediction). As usual, we are typically interested in defining our regressor (or classifier) such that it minimizes some error between the predictions and the true values. That is, we want to define a predictor $f(y_1 \ldots y_n)$ that estimates the next value ($y_{n+1}$) in the sequence so as to minimize (e.g.)~a Mean Squared Error:
%\done{Capitalization?}
\begin{equation}
    \frac{1}{n} \sum_{i = 1}^{n} (f(y_1 \ldots y_i) - y_{i+1})^2.
\end{equation}

Trivially, we might imagine several na\"ive techniques for estimating the next value in a sequence, e.g.~we could predict the next value as a weighted sum of previous values:
\begin{align}
    \text{moving average:}\quad f(y_1 \ldots y_n) & = \frac{1}{K} \sum_{k = 0}^{K - 1} y_{n - k} \label{eq:movingAverage}\\
    \text{weighted moving average:}\quad f(y_1 \ldots y_n) & = \frac{\sum_{k = 0}^{K - 1} (K - k) \cdot y_{n - k}}{\sum_{k=1}^K k}. \label{eq:weightedMovingAverage}
\end{align}
%\done{Check equation, make sure weights are right and that denom matches weight. Looks right}
Although simple, one can imagine how these averages could potentially be better predictors than always predicting the next value to be equal to the previous one.
%(for example, see Exercise~\ref{ex_autoRegression1}). 
These types of trivial predictors can also be used to plot trends in noisy data (\cref{fig:slidingWindows}); averages over larger intervals (i.e.,~larger values of $K$) will produce smoother summaries of the data. Such averages can be efficiently computed for successive values via a dynamic programming solution; code to generate the plots in \Cref{fig:slidingWindows} is presented below:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
xSum = sum(x[:wSize]) # Given data x and y to plot, and a window size wSize
ySum = sum(y[:wSize]) # Sum of first wSize values
xSliding = []
ySliding = []

for i in range(wSize,len(x)-1):
    xSum += x[i] - x[i-wSize] # Strip off oldest value and add newest one
    ySum += y[i] - y[i-wSize]
    xSliding.append(xSum / wSize)
    ySliding.append(ySum / wSize)
\end{lstlisting}
\end{minipage}

%\done{sliding window figure is clipped}
\begin{SCfigure}
\includegraphics[scale=0.75]{fig/slidingWindows.pdf}
\caption[Moving-average plots of \emph{Goodreads}
%\done{italicize? Yes}
Fantasy novel ratings.]{Moving-average plots of $\sim$1 week of \emph{Goodreads}
%\done{italicize? Yes}
Fantasy novel ratings. Although not particularly effective as predictors, moving averages can be used to plot data and summarize overall trends. \label{fig:slidingWindows}}
\end{SCfigure}
%\done{Probably add code here. No}

%\done{`Autoregression' is never defined. At least name this a paragraph or subsection}

The two trivial strategies above are heuristics for predicting the next value, capturing the intuition that it should be similar to recently observed values (\cref{eq:movingAverage}), and possibly that more recent values should be more predictive than less recent ones (\cref{eq:weightedMovingAverage}). However, a better strategy might be to \emph{learn} which of the recent values are the most predictive, i.e.,
\begin{equation}
    f(y_i \ldots y_n) = \sum_{k=0}^{K - 1} \theta_k y_{n - k}.
    \label{eq:autoRegression}
\end{equation}
The values $\theta_k$ now determine which of the previous $k$ values are most
%predictive. 
related to the next one.
For example, in periodic data (e.g.~network traffic, seasonal purchases), the most predictive values may include recent observations, observations from 
%7 days ago, 
the same day last week,
observations from the same day in the previous month, etc.

%\done{Include example from some actual dataset? Could use e.g. Bay Area Bike Share, though dataset seems not to be public}

For example, training a simple autoregressive model on a dataset of hourly measurements of bike rentals in the Bay Area\footnote{Each observation $y_h$ is a measurement of how many trips were taken during hour $h$.} yields the following model:
%\done{Fix equation alignment}
\begin{equation}
%0.47105067, -0.28432565, 0.10555845,  0.01445134, -0.02131945, 0.17501989,  0.539696351
{
\setlength{\tabcolsep}{0.2pt}
    y_{n} = \begin{tabular}{rl}
        0.471 & $y_{n-1}$\\
      - 0.284 & $y_{n-2}$\\
      + 0.106 & $y_{n-3}$\\
      + 0.014 & $y_{n-4}$\\
      - 0.021 & $y_{n-5}$\\
      + 0.175 & $y_{n-24}$\\
      + 0.540 & $y_{n-24\times 7}$
      \end{tabular}
}
\end{equation}
Here we see that the two most predictive observations are those from the previous hour ($y_{n-1}$) and from exactly one week ago ($y_{n - 24\times 7}$). Observe that we didn't include \emph{every} previous observation up to $24\times 7$ hours ago as a feature, rather we only included those previous observations that we expected to be predictive.

Note that although the solution in \Cref{eq:autoRegression} uses \emph{only} previous values in the sequence, one can of course include other features associated with the current timepoint or previous values, just like in a normal regression model.

%\done{Check indentation level for all lstlistings}

Although a simple approach to regression and classification of time series data, the basic idea behind autoregression (i.e.,~to use previous observations to predict future values in a sequence) will reappear in many of the more complex models we develop, especially as we develop sequential models in \Cref{sec:markovCaseStudies,sec:rnn}.

\section{Temporal Dynamics in Recommender Systems}
\label{sec:temporalRec}

Several attempts have been made to improve recommendations by incorporating temporal dynamics. There are countless reasons why preferences, purchases, or interactions may change over time, or more simply why knowing the current timestamp may help us to more accurately predict the next interaction.
Consider for example the following scenarios which could cause changes in movie ratings or interactions over time:
\begin{enumerate}
    \item Users who favor special effects may give lower ratings as a movie's special effects become dated.
    \label{temporalItem3}
    \item Users may give higher ratings to older movies, e.g.~due to feelings of nostalgia.
    \label{temporalItem1}
    \item Alternately, ratings of older movies may represent a biased sample of items that users had explicitly searched for (versus newer items which a user selected due to their being surfaced on a landing page).
    \label{temporalItem2}
    \item A mundane change to a user interface, such as modifying the tool-tip text associated with a certain rating, may alter the rating distribution.
    \label{temporalItem10}
    \item A family member may borrow a user's account, and temporarily consume movies quite different from the account's typical activities. 
    \label{temporalItem4}
    \item Users may binge-watch a series, dominating their interaction patterns for a short period.
    \label{temporalItem5}
    \item Action blockbusters may be more favored during summer (or Christmas movies during Christmas).
    \label{temporalItem6}
    \item Users may want to consume (or avoid) content very similar to what they have previously interacted with.
    \label{temporalItem7}
    \item Users may gradually develop an appreciation for certain characteristics of a movie as they consume more content from that genre.
    \label{temporalItem8}
    \item Users may be anchored by external forces, i.e.,~the \emph{zeitgeist} of what is currently popular in their community.
    \label{temporalItem9}
\end{enumerate}
%INCLUDE ITEMS BELOW
% Changes in the interface
% • People give higher ratings to older  movies (or, people
% who watch older movies are a biased sample)
% • The community’s preferences gradually change over time
% • My girlfriend starts using my Netflix account one day
% • I binge watch all 144 episodes of buffy one week and
% then revert to my normal behavior
% • I become a “connoisseur” of a certain type of movie
% • Anchoring, public perception, seasonal effects, etc.
The above dynamics are quite varied in their sources and scale: Effects~\ref{temporalItem3} and \ref{temporalItem1} are gradual and long-term; \ref{temporalItem2} and \ref{temporalItem10} owe to vagaries of a changing user interface; \ref{temporalItem4} and \ref{temporalItem5} are `bursty' or short-term; \ref{temporalItem6} is seasonal; \ref{temporalItem7} is sequential; \ref{temporalItem8} owes to user growth; and \ref{temporalItem9} due to a changing community. One can imagine many other sources, especially in different settings subject to social dynamics, price variability, fashion, etc.
%
As we will see, understanding such dynamics is often key to making successful recommendations. 
%Some may seem mundane or trivial: e.g.~a change in UI or 
Some may scarcely seem like `temporal dynamics' at all: e.g.~a change to the user interface has little to do with user or item evolution. Nevertheless, we'll argue that modeling even these trivial or mundane dynamics proves critical in order to disentangle them from the `real' real personalization dynamics in the data.

%\paragraph{Methods for temporal recommendation}
\subsection{Methods for Temporal Recommendation}

Methods for temporal recommendation fall broadly into two classes. The first make use of the actual \emph{timestamps} of events. Each interaction $(u,i)$ is augmented with a timestamp $(u,i,t)$, and the goal is to understand how ratings $r_{u,i,t}$ vary over time. That is, our goal is to extend models such as the one in \Cref{subsec:matrixFactor} (\cref{eq:MFscore}) so that parameters vary as a function of time, e.g.
\begin{equation}
    r_{u,i,t} = \alpha(t) + \beta_u(t) + \beta_i(t) + \gamma_u(t) \cdot \gamma_i(t).
    \label{eq:latentFactorTime}
\end{equation}
Modeling temporal dynamics in this way is effective at capturing long-term shifts of community preferences over time. Such a model can also capture short-term or `bursty' dynamics, such as purchase patterns being affected by external events, or periodic events, such as purchases being higher at a particular time of day, day of week, or season.

A second class of methods discards the specific timestamps, but preserves only the \emph{sequence} (or order) of events. Thus the goal is generally to predict the next action as a function of the previous one, i.e.,
\begin{equation}
    p(\text{user $u$ interacts with item $i$} \mid \text{they previously interacted with item $j$}).
    \label{eq:sequenceModelExamp}
\end{equation}
This type of model makes the assumption that the important temporal information is captured in the \emph{context} provided by the most recent event. This is useful in highly-contextual settings, such as predicting the next song a user will listen to, or other items they will place in their basket, etc. In such settings knowing the most recent actions (or most recent few actions) is more informative than knowing the specific timestamp.

These two classes of model are quite orthogonal, both in terms of the settings where they are effective, as well as the techniques involved. Below we'll explore both settings via several case-studies. First we'll explore parametric temporal models (as in \cref{eq:latentFactorTime}) through the example of the \emph{Netflix prize}, where temporal dynamics 
%\done{italics}
are carefully modeled to capture a wide variety of application-specific dynamics. We explore sequential models (as in \cref{eq:sequenceModelExamp} in \Cref{sec:markovCaseStudies} via several examples, starting with online shopping scenarios; these settings tend to be highly contextual, where the context of recent activities is often more informative than long-term historical activities. 

Later, 
we'll introduce \emph{recurrent networks} as general-purpose models of sequence data (\cref{sec:rnn}), and as a potential approach to capture complex dynamics in evolving interaction sequences. Such models have the potential to overcome the limitations of traditional sequential models by learning complex semantics that persist over many steps. Such models (and variants based on different sequential architectures) arguably represent the current state-of-the-art for general purpose recommendation. 
%Our discussion on recurrent networks leads us toward \Cref{chap:text}, where such 
%\done{In this chapter or not? Yes}

%\done{Wrote above late, see if it fits. Is fine}

%\done{Borrowed paragraphs. Not used}
%Some early works that deal with temporal dynamics do so in
%terms of concept drift 
%[23, 37, 39]. 
%Such models are able to account
%for short-term temporal effects (‘noise’), and long-term changes
%in user behavior (‘drift’), for example due to the presence of new
%products within a community.

%Sophisticated models of such temporal dynamics proved critical
%in obtaining state-of-the-art performance on the Netflix challenge
%[3], most famously in [20]. As discussed in [20], few previous
%works had dealt with temporal dynamics, other than a few notable
%exceptions [2, 10, 35]. Around the same time, ‘adaptive neighborhood’ models were proposed [24], that address the problem of
%iteratively training recommender systems whose parameters ought
%to change over time.

% Better performance may be obtained by modeling large-scale
% global changes at the level of entire communities [41], or by developing separate models for short term changes (e.g. due to external
% events), and long-term trends [40].
% Other works that study temporal dynamics at the level of products and communities include [29], where the authors studied how
% existing ratings within communities may influence new users, and
% how community dynamics evolve; and [13], who studied how users
% are influenced by previous ratings of the same product

%\done{Introduce why we're talking about this. Not sure what this was?}

\subsection[Case Study: Temporal Recommendation and the Netflix Prize]
{Case Study: Temporal Recommendation and the Netflix Prize} 
%\cite{koren2009collaborative}}
\label{sec:caseStudyNetflixTemporal}
%\done{Should include citations in section headings? No}

%\done{Several typos below. Looks fine}

Careful modeling of temporal dynamics was one of the key features characterizing the strongest solutions to the Netflix Prize (which we described at the beginning of \Cref{chap:modelbasedRecsys}). Several of the ideas that proved effective for modeling temporal dynamics on Netflix specifically are covered in \cite{koren2009collaborative} (`Collaborative Filtering with Temporal Dynamics'), which we summarize here.

As a motivating example, consider the two plots shown in \Cref{fig:netflixTemporal}.
%\footnote{These two plots are based on similar figures from \cite{koren2009collaborative}; the purported trends are somewhat more pronounced in the original paper, possible due to differences in sampling schemes.}
%\done{need this footnote? No}
At top, we see ratings over time averaged across weekly bins. We see several points where ratings appear to increase suddenly, followed by plateaus of relatively stable ratings; for example at around week 210, average ratings appear to jump from around 3.5 stars to 3.6 stars.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/netflix1.pdf}\\[2mm]
    \includegraphics[scale=0.75]{fig/netflix2.pdf}
    \caption[Temporal dynamics on Netflix.]{Temporal dynamics on Netflix. The top plot shows ratings averaged across each week over the lifetime of the dataset; the bottom plot shows how ratings change for newly-introduced movies, showing that ratings gradually increase during the first few weeks the movie is on Netflix. These plots reveal a combination of sudden and gradual trends in movie ratings over time.     \label{fig:netflixTemporal}}
\end{figure}
%\done{Plots don't fit; should be shrunk width-wise}

Such long-term, population-level changes could owe to several explanations. Changes in rating patterns could be due to a changing user base, or from certain movies being added to Netflix; such changes could be due to world events exogenous to Netflix. Or, the change could owe to a factor as simple as a change in Netflix's User Interface (UI), causing users to rate movies differently.

The bottom plot in \Cref{fig:netflixTemporal} shows another temporal trend, demonstrating that individual movies receive higher ratings the longer they have been available on Netflix. Again, such trends could be due to a variety of factors: users may be favorably biased toward older movies (e.g.~by nostalgia); or again it could be a function of the UI: a user who specifically sought out an older movie may view it more favorably than a user who had 
%been recommended 
discovered
the same movie from the front page.

Whatever the underlying cause of these trends, they account for significant variability in the observed ratings. And as such, we should model these dynamics to predict ratings more accurately. Na\"ively, one might simply discard (e.g.)~older data that does not correspond to 
%the predominant 
current
rating trends. But, a more effective model would attempt to account for the differences between newer and older data, while learning from both.

To model the kinds of long term trends captured in \Cref{fig:netflixTemporal}, \cite{koren2009collaborative} first focus only on temporally evolving bias terms, i.e.,
\begin{equation}
    b_{u,i}(t) = \alpha + \beta_u(t) + \beta_i(t).
\end{equation}
Starting with item biases, one can capture long-term, gradual variation simply by having different bias terms for different periods, i.e.,
\begin{equation}
    \beta_i(t) = \beta_i + \beta_{i,\mathrm{bin}(t)}.
    \label{eq:bins30}
\end{equation}
\cite{koren2009collaborative} 
%the authors 
suggest $\sim 30$ bins corresponding to about 10 weeks each in the case of the Netflix data.

This basic idea of separating bias terms into bins could likewise be applied to capture periodic trends, much like we encoded periodic terms in \Cref{sec:temporalFeatures}. Here the bias term would again take the form of several bins:
\begin{equation}
    \beta_i(t) = \beta_i + \beta_{i,\mathrm{bin}(t)} + \beta_{i,\mathrm{period}(t)},
    \label{eq:netflixPeriodic}
\end{equation}
where $\mathrm{period}(t)$ could represent periodic effects at the level of different days of the week, or months of the year (etc.).

The above ideas are effective for modeling long-term and periodic dynamics, but are quite expensive: e.g.~the model in \Cref{eq:bins30} requires an additional 30 parameters \emph{per item}; this is affordable for items on Netflix since the average item has over 5,000 ratings (100 million ratings of 17,770 titles). However this would likely not be possible for users, as the average user has only around 200 ratings.
% 480k users
Thus one needs a way of parameterizing user temporal dynamics that is considerably cheaper (i.e.,~involves fewer parameters).

A solution suggested in \cite{koren2009collaborative} is to use an `expressive deviation' term for each user:
\begin{equation}
    \mathrm{dev}_u(t) = \underbrace{\mathrm{sign}(t - t_u)}_{\mathclap{\text{before $(-1)$ or after $(1)$ the mean date}}} \cdot |t - t_u |^x.
    \label{eq:expressiveDev}
\end{equation}
The term $t_u$ represents the mean date amongst a particular user $u$'s ratings, so that the term $(t - t_u)$ represents whether a particular point in time $t$ is before or after the midpoint of the user's rating lifetime.
%
The expressive deviation term is depicted in \Cref{fig:expressiveDev}; the exponent of 0.4 was found to work well on Netflix data. 
%\done{sentence}
The deviation term augments the user bias term via a user-specific scaling term $\alpha_u$, essentially controlling how strongly the deviation term applies to a specific user:
\begin{equation}
    \beta_u(t) = \beta_u + \alpha_u \cdot \text{dev}_u(t),
    \label{eq:temporalBeta}
\end{equation}
e.g.~a negative value of $\alpha_u$ would mean the user's ratings trend down over time, whereas a value $\alpha_u \simeq 0$ would mean that the user's overall bias is not subject to temporal variation.
%\done{and sentence below. Done, I think}
A similar strategy can also be used to capture variation at the level of individual latent dimensions, e.g.
\begin{equation}
    \gamma_{u,k}(t) = \gamma_{u,k} + \alpha_{u,k} \cdot \text{dev}_u(t) + \gamma_{u,k,t}.
    \label{eq:temporalGamma}
\end{equation}
Note that the deviation terms in \Cref{eq:temporalBeta,eq:temporalGamma}
%\done{check all crefs with multiple terms. They should render right}
add only a single term per user (\cref{eq:temporalBeta}) or a single term per factor (\cref{eq:temporalGamma}).
The final term $\gamma_{u,k,t}$ in \Cref{eq:temporalGamma} (a temporally-evolving term applied to a specific factor for a specific user) models highly-local preference dynamics and can be used to model (e.g.)~day-specific variability; this term is however highly expensive (in terms of the number of parameters introduced) so may only be available for some users.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/expressiveDev.pdf}
\caption{Expressive deviation term from \protect\cite{koren2009collaborative}. 
\label{fig:expressiveDev}}
\end{SCfigure}


When combined with the carefully-chosen deviation term of \Cref{eq:expressiveDev} this allows the model to capture quite complex dynamics while adding only a modest number of parameters; this basic design philosophy (a carefully-designed parametric model that costs only a small number of parameters) will prove a common theme when designing temporal models (\cref{sec:whatTeach}).


While the expressive deviation term is effective for users who have few interactions, for users who have more it can be useful to fit a more complex model. To do so, a spline function can be used to model gradual shifts in user biases. A spline function smoothly interpolates between a series of control points via the following function:
\begin{equation}
    \beta_u(t) = \beta_u + \frac{\sum_{l = 1}^{k_u} e^{\gamma|t - t_l^u|}b_{t_l}^u}{\sum_{l = 1}^{k_u}e^{-\gamma|t - t_l^u|}}.
\end{equation}
Here $k_u$ is the number of control points for user $u$ (which grows with the number of ratings that user has entered); $t^u$ is a series of uniformly spaced time-points for each user; and $b_{t_l}^u$ is the bias associated with each control point. This type of interpolation is depicted in \Cref{fig:splines}.

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/splines.pdf}
\caption{Spline interpolation of temporally evolving user bias. \label{fig:splines}}
\end{SCfigure}

The above term is a reasonably flexible way to capture gradual drift in user preferences, though still can't handle sudden changes. \cite{koren2009collaborative} address this with a `per day' user bias $\beta_{u,t}$, which can be useful for particular days in which users have a lot of activity. Note that while such a bias is unlikely to be helpful when predicting future events; instead, by modeling outliers in this way, the model can essentially learn to `ignore' events that are not useful for prediction. This too is a common design principle when building temporal models: the goal is not so much to forecast \emph{future} trends but rather to adjust and account for \emph{past} events appropriately.

The above ideas capture the essential components covered in \cite{koren2009collaborative}; 
%though 
their work also includes an exploratory study of several alternative modeling approaches, including incorporating temporal dynamics into neighborhood-based models like those 
%of \Cref{eq:similarityRecommender?}.
of \Cref{chap:recsysIntro}.

%It should be noted though that the goal of this model was to be effective on Netflix, rather than to capture temporal dynamics in general settings.
%\done{Pasted this text in here, see if it fits. Cut this but added one sentence below}

%\done{Still needs personalized latent factors etc. Done, but only one sentence above}



\subsection{What can Netflix Teach us about Temporal Models?}
\label{sec:whatTeach}
%\done{Capitalization, overall organization}
%\done{Cite "lessons from the Netflix prize" paper}
The model developed in the above case study involved several decisions that are quite specific to Netflix, and indeed the model was designed with the explicit goal of achieving strong performance on a single dataset. As such, many of the specific choices (such as the specific parametric form of the expressive deviation term) may not generalize to other settings.
Nevertheless, there are several important lessons in the above study that apply generally when developing temporal models of user behavior:
\begin{itemize}
    % \item First, developing bespoke models around specific types of dynamics is somewhat the norm when designing temporal models. As we will see 
    \item Successful solutions to the \emph{Netflix Prize} highlighted the critical importance of temporal dynamics in recommendation settings.
    %\item 
    While the approaches we explored in \Cref{sec:caseStudyNetflixTemporal} are quite Netflix-specific, they highlight a general philosophy followed when building temporal models: 
    temporal models tend to be carefully designed around the dynamics of specific datasets and applications. 
    %as we'll see in the following sections, different temporal models abound to capture dynamics in different settings.
    \item One reason for the proliferation of many different hand-crafted temporal models
    %(rather than more `general purpose' models as we developed in \Cref{?,?}) 
    is that temporal models are \emph{expensive}
    in terms of the number of parameters required. 
    As such a main focus when building temporal models is that of model \emph{parsimony}: to prevent the parameter space from exploding, one must carefully choose models that capture the desired dynamics with as few parameters as possible.
    \item Temporal dynamics may range from `lofty' concepts such as users becoming nostalgic toward older movies, to more mundane sources of variation, such an overall shift in average ratings across the community. Both are important to model as both explain variance in the data: untangling even mundane dynamics is a critical step toward extracting meaningful personalization signals.
    \item Some specific lessons from Netflix generalize quite well, especially the importance placed on temporally evolving bias terms: the vast majority of temporal variation often owes to shifts in item popularity, user activity (etc.),~and can be captured via evolving $\beta_u$ or $\beta_i$; temporally evolving latent factors ($\gamma_u$ or $\gamma_i$) 
    %\done{i comes before u or u comes before i? u before i}
    play less of a role, or otherwise are simply too expensive to model.
    \end{itemize}
    %\item 
    
Finally, we highlight that our goal in developing temporal models is typically not to \emph{forecast} long-term trends. For example our model of opinion dynamics on \emph{Netflix} tells us nothing about what will be popular next year. Rather the goal is typically to 
    %adjust for 
    %compensate
    account for
    discrepancies across different time periods so that interactions across time can be meaningfully compared. The resulting model thus gives a more accurate sense of \emph{current} rating dynamics, if not future trends. See also \cite{bell2007lessons} for further discussion.
%
    % \item One reason for the proliferation of many different hand-crafted temporal models (rather than more `general purpose' models as we developed in \Cref{?,?}) is that temporal models are \emph{expensive}, in terms of the number of parameters required. 
    %As such, models are designed to capture the required dynamics while maintaining model parsimony (i.e.,~using as few parameters as possible).
    % \item Modeling temporal dynamics is not about \emph{forecasting} so much as it is about determining how to properly adjust, re-weight, or discount past events.
% \end{itemize}


%\done{spacing before code blocks doesn't generally look good (too close to text above)}

% \begin{figure}
% \caption{
% The models we discussed when exploring temporal dynamics on \emph{Netflix} teach us several general lessons about building temporal models in other settings.
% \label{box:netflixLessons}}
% %\done{not referenced. Removed}
% %\done{Not good caption. Removed}
% \fbox{
% \begin{minipage}{0.95\linewidth}
% \begin{itemize}
%     \item Successful solutions to the \emph{Netflix Prize} highlighted the critical importance of temporal dynamics in recommendation settings.
%     %\item 
%     While the approaches we explored in \Cref{sec:caseStudyNetflixTemporal} are quite Netflix-specific, they highlight a general philosophy followed when building temporal models: temporal models tend to be carefully designed around the dynamics of specific datasets and applications.
%     \item A main focus when building temporal models is that of model \emph{parsimony}: to prevent the parameter space from exploding, one must carefully choose models that capture the desired dynamics with as few parameters as possible.
%     \item Temporal dynamics may range from `lofty' concepts such as users becoming nostalgic toward older movies, to more mundane sources of variation, such an overall shift in average ratings across the community. Both are important to model as both explain variance in the data: untangling even mundane dynamics is a critical step toward extracting meaningful personalization signals.
%     \item The goal of temporal models is typically not to forecast long-term trends. E.g.~our model of opinion dynamics on \emph{Netflix} tells us nothing about what will be popular next year. Rather the goal is typically to 
%     %adjust for 
%     %compensate
%     account for
%     discrepancies across different time periods so that interactions across time can be meaningfully compared. The resulting model thus gives a more accurate sense of \emph{current} rating dynamics, if not future trends.
%     \end{itemize}
% \end{minipage}
% }
% \end{figure}
%\done{repeats the text somewhat. Could expand the text and call this a summary. Removed}
%\done{Make sure to reference figure. Removed}




% \subsection{What can Netflix Teach us about Temporal Models?}
% %\done{Capitalization, overall organization}
% %\done{Cite "lessons from the Netflix prize" paper}
% The model developed in the above case study involved several decisions that are quite specific to Netflix, and indeed the model was designed with the explicit goal of achieving strong performance on a single dataset. As such, many of the specific choices (such as the specific parametric form of the expressive deviation term) may not generalize to other settings.
% Nevertheless, there are several important lessons in the above study that apply general when developing temporal models of user behavior:
% \begin{itemize}
%     \item First, developing bespoke models around specific types of dynamics is somewhat the norm when designing temporal models. As we will see in the following sections, different temporal models abound to capture dynamics in different settings.
%     \item One reason for the proliferation of many different hand-crafted temporal models (rather than more `general purpose' models as we developed in \Cref{?,?}) is that temporal models are \emph{expensive}, in terms of the number of parameters required. As such, models are designed to capture the required dynamics while maintaining model parsimony (i.e.,~using as few parameters as possible).
%     \item Modeling temporal dynamics is not about \emph{forecasting} so much as it is about determining how to properly adjust, re-weight, or discount past events.
%     \item Some specific lessons from Netflix generalize quite well, especially the importance placed on temporally evolving bias terms: the vast majority of temporal variation often owes to shifts in item popularity, user activity (etc.),~and can be captured via evolving $\beta_u$ or $\beta_i$; temporally evolving latent factors ($\gamma_u$ or $\gamma_i$) 
%     %\done{i comes before u or u comes before i? u before i}
%     play less of a role, or otherwise are simply too expensive to model.
% \end{itemize}


% \begin{figure}
% \caption{
% The models we discussed when exploring temporal dynamics on \emph{Netflix} teach us several general lessons about building temporal models in other settings.
% \label{box:netflixLessons}}
% %\done{not referenced. Removed}
% %\done{Not good caption}
% \fbox{
% \begin{minipage}{0.95\linewidth}
% \begin{itemize}
%     \item Successful solutions to the \emph{Netflix Prize} highlighted the critical importance of temporal dynamics in recommendation settings.
%     %\item 
%     While the approaches we explored in \Cref{sec:caseStudyNetflixTemporal} are quite Netflix-specific, they highlight a general philosophy followed when building temporal models: temporal models tend to be carefully designed around the dynamics of specific datasets and applications.
%     \item A main focus when building temporal models is that of model \emph{parsimony}: to prevent the parameter space from exploding, one must carefully choose models that capture the desired dynamics with as few parameters as possible.
%     \item Temporal dynamics may range from `lofty' concepts such as users becoming nostalgic toward older movies, to more mundane sources of variation, such an overall shift in average ratings across the community. Both are important to model as both explain variance in the data: untangling even mundane dynamics is a critical step toward extracting meaningful personalization signals.
%     \item The goal of temporal models is typically not to forecast long-term trends. E.g.~our model of opinion dynamics on \emph{Netflix} tells us nothing about what will be popular next year. Rather the goal is typically to 
%     %adjust for 
%     %compensate
%     account for
%     discrepancies across different time periods so that interactions across time can be meaningfully compared. The resulting model thus gives a more accurate sense of \emph{current} rating dynamics, if not future trends.
%     \end{itemize}
% \end{minipage}
% }
% \end{figure}
%\done{repeats the text somewhat. Could expand the text and call this a summary}
%\done{Make sure to reference figure}


\section{Other Approaches to Temporal Dynamics}
\label{sec:otherTemporal}
%\done{Weird section name. It's fine}

Our case study of temporal dynamics on \emph{Netflix} demonstrated the general approach of building temporal models by carefully hand-crafting model components to account for the dynamics in a particular dataset or setting. Naturally, temporal dynamics can vary widely in other settings, demanding different modeling approaches. Below we outline a few of the main categories of temporal models and present a few specific examples which we summarize in \Cref{tab:temporalRec}.

%\done{intro}
%Our sample of the temporal recommendation approaches discussed here is summarized in \Cref{tab:temporalRec}.

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Comparison of temporally-aware recommendation techniques.]{Comparison of temporally-aware recommendation techniques. References: \protect\cite{koren2009collaborative,ding2005time,xiang2010temporal,mcauley2013amateurs}.}
\label{tab:temporalRec}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{lXX}
% % \hrule\\
% \textbf{Reference} & \textbf{Dataset} & \textbf{Description}\\ 
% \cite{ma2008sorec} & Social trust data from \emph{epinions} & Trust links help to regularize $\gamma_u$, which must simultaneously explain rating and trust factors.\\
% \cite{zhao2014leveraging} & \emph{epinions}, \emph{Ciao}, \emph{Delicious}, \emph{LibraryThing} &
% Friends' interactions act as additional \emph{implicit signals} for recommendation.\\
% % \hrule
% \cite{o2001polylens} & \emph{MovieLens} & Studies the desirable characteristics of good interfaces for group recoommendation.\\ 
% \cite{amer2009group} & \emph{MovieLens} & Designs measures of \emph{group consensus} and proposes recommendation algorithms to maximize them.\\
% \cite{pan2013gbpr} & \emph{MovieLens}, \emph{Netflix} & Treats group preferences as weak signals to design pairwise sampling strategies for BPR.
% \end{tabularx}
% \begin{tabularx}{\linewidth}{l>{\raggedright}p{0.22\textwidth}X}
\begin{tabularx}{\linewidth}{llX}
% \hrule\\
\toprule
\textbf{Ref.} & \textbf{Temporal Signal} & \textbf{Description}\\ 
\midrule
%\cite{koren2009collaborative} 
K09
& Various & Uses a variety of parameteric functions and temporal bins, mostly to capture gradual drift in item biases and preferences (\cref{sec:caseStudyNetflixTemporal}).\\
%\cite{ding2005time}
DL05
& Recency & Recent interactions are weighted more highly when determining relatedness against historical interactions (\cref{subsec:longterm}).\\
%\cite{xiang2010temporal}
X10
& Sessions & Interactions within the same \emph{session} are used as an additional signal of relatedness among items (\cref{subsec:shortTerm}).\\
%\cite{mcauley2013amateurs} 
M13
& Acquired tastes & Users acquire tastes toward certain items due to repeated exposure to related items (\cref{subsec:userEvolve}).\\
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{not refereced}
%\done{table formatting}

\subsection{Long-Term Dynamics of Opinions}
\label{subsec:longterm}

Other than the methods we discussed 
%in \Cref{sec:caseStudyNetflixTemporal} 
above
for temporal recommendation on \emph{Netflix}, several papers studied the notion of gradually evolving concepts within the context of preferences and opinions.

Early works that deal with temporal dynamics explore the notion of \emph{concept drift} \citep{widmer1996learning,tsymbal2004problem}; early works on this topic are concerned with systems for classification in settings with temporally-evolving data. 
Simple approaches consist of (e.g.)~taking only a \emph{window} of recent examples during training (much like we saw in \Cref{fig:slidingWindows}). More sophisticated approaches allow the context window size to adapt based on how `stable' a particular concept is; or to reuse concepts that recur periodically; or to distinguish drifting concepts from noise. Models based on these ideas, along with theoretical results, are discussed in \cite{widmer1996learning}, among others.

%terms of concept drift 
%[23, 37, 39]. 
%Such models are able to account
%for short-term temporal effects (‘noise’), and long-term changes
%in user behavior (‘drift’), for example due to the presence of new
%products within a community.

Among temporal techniques for recommendation, early approaches incorporate temporal factors into heuristic techniques, such as the model from \Cref{eq:simPredict}. For example in \cite{ding2005time}, the basic idea is to weight related items in \Cref{eq:simPredict} so that recent interactions are weighted more highly:
\begin{equation}
    r(u,i) = \frac{\sum_{j \in I_u} R_{u,j} \cdot \mathrm{Sim}(i,j) \cdot f(t_{u,j})}{\sum_{j \in I_u} \mathrm{Sim}(i,j) \cdot f(t_{u,j})}.
    \label{eq:timeWeight}
    %\label{eq:simPredict2}
\end{equation}
%\done{consistent subscripts (comma or not). No comma, not sure what this is about}
Here $t_{u,j}$ is the timestamp associated with the rating $R_{u,j}$, and $f(t_{u,j})$ is a monotone function of the timestamp. For example, relevance can decay exponentially for older items:
\begin{equation}
    f(t) = e^{-\lambda \cdot t}
\end{equation}
%\done{Connecting sentence or description of results}



\cite{godes2012sequential}, while less focused on predictive modeling, attempt to characterize the long-term dynamics of opinions through online reviews. They studied book reviews on \emph{Amazon}. Like our study of \emph{Netflix} data (especially \Cref{fig:netflixTemporal}), they study how ratings evolve over time, as well as how they evolve in terms of the age of a book (time since first review). The dynamics are quite different from those on Netflix, where both show a \emph{decreasing} trend over time (in contrast to the `nostalgia' effect we observed in \Cref{fig:netflixTemporal}).

In addition to a difference in domain (books versus movies), \emph{online reviews} potentially have quite different dynamics than ratings on \emph{Netflix}, which \cite{godes2012sequential} discuss in detail. For example, given that users see each other's reviews, they may be guided by social effects, e.g.~users may enter a review only if they perceive that it will affect the average rating \citep{wu2008public}. \cite{godes2012sequential} also discuss the importance of \emph{self-selection}, where users who assign more value to a product tend to purchase it earlier (and thus are responsible for more positive early reviews) \citep{li2008self}. \cite{godes2012sequential} argue that more complex dynamics may be at play once we appropriately control for these effects; ultimately their study again reveals that temporal dynamics can differ drastically as a function of a particular setting or dataset.
%However \cite{godes2012sequential} argue that the dynamics are possibly more nuanced, and that once one appropriately controls for the gradual decline across 
%\done{Dropped}

%Sequential and temporal dynamics of online opinion

\subsection{Short-Term Dynamics and Session-Based Recommendation}
\label{subsec:shortTerm}

%Temporal recommendation on graphs via long- and short-term
%preference fusion


%\done{Beer review table needs adjusting column widths}


Most of the models of temporal dynamics we've discussed so far capture notions such as gradual drift, where models used a small number of parameters to describe gradually
evolving
%\done{sentence garbled}
parametric functions (\cref{fig:expressiveDev}), sequential bins spanning several months (\cref{eq:bins30}), or periodic effects (\cref{eq:netflixPeriodic}). While effective in the settings they were designed for, such models are limited to capturing broad, global trends. We touched briefly on models of short-term dynamics using per-day biases \citep{koren2009collaborative}, though such terms are essentially forms of outlier detection.
%upon `stort-term' dynamics briefly 
%(see e.g.~\Cref{eq_?}), though terms (which apply to a specific day or interaction) are essentially forms of outlier detection, as we argued in \Cref{sec_?}.

Another pattern of short-term temporal variation arises due to users'
%notion of 
specific context within an interaction
\emph{session}. 
%\done{ugly sentence. Okay}
Sessions may have 
%specific characteristics that differ from 
a specific, narrow focus that can be useful to predict future interactions in the short term, but which differ from users' overall patterns and can be discarded in the long term. `Sessions' can be extracted in various ways, though are typically based on some simple heuristic such as setting session boundaries based on a threshold between successive interaction timestamps.

\cite{xiang2010temporal} attempt to combine models of long- and short-term dynamics by incorporating user sessions. Their model is based on random walk-based methods similar to those we studied in \Cref{sec:randomWalk}. Here, `sessions' simply become additional nodes in the interaction graph, as in \Cref{fig:STG}. Edges now connect user-to-item interactions and session-to-item interactions. In terms of our random walk model, this means there are two mechanisms for information to propagate between related items. User-to-item and session-to-item terms are associated with different interaction weights $\eta_u$ and $\eta_s$; roughly speaking, these control the relative importance of 
long-term user-level dynamics and
short-term session-level dynamics.
%and 
%long-term user-level dynamics. 
The model can also be tuned in terms of the granularity at which interactions are divided into sessions (with shorter sessions capturing more local dynamics). \cite{xiang2010temporal} show the value of this notion of short-term dynamics on user bookmark data (from \emph{CiteULike} and \emph{Delicious}).
%\done[style]


\begin{SCfigure}
        \centering
\definecolor{mygrey1}{RGB}{160,160,160}
\definecolor{mygrey2}{RGB}{120,120,120}
\definecolor{mygrey3}{RGB}{200,200,200}

\begin{tikzpicture}[thin,
  every node/.style={draw,circle},
  fsnode/.style={fill=mygrey1},
  ssnode/.style={fill=mygrey2},
  gsnode/.style={fill=mygrey3},
  ->,shorten >= 3pt,shorten <= 3pt
]

% User vertices
\begin{scope}[start chain=going below,node distance=4mm]
\node[fsnode,on chain] (f1) [label={[label distance=2mm, rectangle]90:users:}] {};
\foreach \i in {2,3,...,6}
  %\node[fsnode,on chain] (f\i) [label=left: \i] {};
  \node[fsnode,on chain] (f\i) {};
\end{scope}

% Ad vertices
\begin{scope}[xshift=2cm,start chain=going below,node distance=4mm]
\node[ssnode,on chain] (s1) [label={[label distance=2mm, rectangle]90:items:}] {};
\foreach \i in {2,3,...,6}
  %\node[ssnode,on chain] (s\i) [label=right: \i] {};
  \node[ssnode,on chain] (s\i) {};
\end{scope}

\begin{scope}[xshift=4cm,start chain=going below,node distance=4mm]
\node[gsnode,on chain] (g1) [label={[label distance=2mm, rectangle]90:sessions:}] {};
\foreach \i in {2,3,...,6}
  %\node[gsnode,on chain] (g\i) [label=right: \i] {};
  \node[gsnode,on chain] (g\i) {};
\end{scope}

\draw (s1) edge[-] node[above,rectangle,draw=none] {$\eta_s$} (g1);
\draw (s2) edge[-] (g1);
\draw (s3) edge[-] (g3);
\draw (s4) edge[-] (g3);
\draw (s5) edge[-] (g3);
\draw (s2) edge[-] (g2);
\draw (s3) edge[-] (g2);
\draw (s5) edge[-] (g4);
\draw (s5) edge[-] (g5);
\draw (s6) edge[-] (g5);
\draw (s6) edge[-] (g6);

\draw (f1) edge[-] node[above,rectangle,draw=none] {$\eta_u$} (s1);
\draw (f1) edge[-] (s2);
\draw (f1) edge[-] (s3);
\draw (f2) edge[-] (s3);
\draw (f2) edge[-] (s4);
\draw (f2) edge[-] (s5);
\draw (f3) edge[-] (s5);
% \draw (f4) edge[-] (s4);
\draw (f4) edge[-] (s5);
\draw (f5) edge[-] (s6);
\draw (f6) edge[-] (s6);
\draw (f6) edge[-] (s5);
% \draw (f5) edge[-] (g4);

% \draw (s9) -- (f5);
% \draw (f5) -- (s6); 
\end{tikzpicture}

    \caption[Session-based Temporal Graph \protect\citep{xiang2010temporal}. ]{Session-based Temporal Graph \protect\citep{xiang2010temporal}. Each session describes a sequence of interactions from a specific user.}
    \label{fig:STG}
\end{SCfigure}

%\done{\protect cites in figure}
 

We revisit session-based recommenders in more depth in \Cref{sec:markov} and \Cref{sec:rnnRec} when we explore methods based on Markov chains and recurrent networks. Like \cite{xiang2010temporal}, such models have a general goal of combining local dynamics from recent interactions with longer-term features extracted from users' interaction histories.
%\done{make sure this reference points forwards}

%\done{Basically learns models about sessions similar to models about users. I think is clear enough}

\subsection{User-Level Temporal Evolution}
\label{subsec:userEvolve}

Most of the sources of temporal dynamics we've explored so far owe to shifting properties of \emph{items} (or how items are perceived over time, due to e.g.~nostalgia), or gradual shifts that apply to the community as a whole (including trivial dynamics such as a change in the user interface, which don't apply specifically to any user or item). Temporal drift could likewise occur at the level of individual users, for example due to a user gradually gaining more experience with a certain type of item.

\cite{mcauley2013amateurs} sought to model this notion of `acquired tastes' in recommendation datasets. They noted that in many settings, a user's preference toward a certain type of item may change due to the very act of consuming items of that type. This setting was motivated by data of beer and wine reviews (including the same datasets we've been using in examples throughout this book), though applies in various other settings: e.g.~a user will likely have a different opinion of \emph{Seven Samurai} depending on whether it is the first or the fiftieth drama they've watched. 

Note that this notion of temporal dynamics is not attached to an item, nor is it attached to the community as a whole (i.e.,~it is not a function of the precise timestamp); rather it is a function of a specific user's \emph{expertise} at the time of their rating. \cite{mcauley2013amateurs} capture this type of dynamic by fitting two variables: $\gamma_u^E$ captures user latent factors for users at different experience levels (where $E$ belongs to a discrete set, e.g.~$E \in \lbrace 1 \ldots 5 \rbrace$). $E$ is then fit for each user as a function of time (i.e.,~$E(u,t)$), with an additional constraint that $E(u,t) > E(u,t') \rightarrow t > t'$, i.e.,~a user's `experience level' must be non-decreasing over time.\footnote{This constraint forces the model to learn parameters resembling experience levels, in the sense that the model is forced to discover systematic `stages' through which many users progress in common.} 
%Both parameters ($\gamma_u$ and $E$) are fit so as to optimize the 
%Modeling the evolution of user expertise through online reviews

In addition to improving performance on certain datasets where acquired tastes play a key role, \cite{mcauley2013amateurs} argue that such a model can also be used to understand which specific types of items require expertise or experience for users to fully appreciate. Examples include IPAs (India Pale Ales), which users tend to gradually develop a preference toward, versus so-called `adjunct lagers' (e.g.~\emph{Bud Light}), which experienced users tend to dislike.

%\done{could say more or cite follow-up papers}


\section{Personalized Markov Chains}
\label{sec:markov}

The temporal models we saw in \Cref{sec:temporalRec} directly modeled (or extracted features from) the \emph{timestamps} associated with each interaction. We showed how features extracted from timestamps could include factors like seasonality, the day of the week, or nostalgia effects (etc.)~on the \emph{Netflix} dataset, and later we'll revisit such models in the context of the temporal dynamics of fashion (\cref{subsec:VBPR}). 

However in many settings, the best predictor of what a user will do next is simply what they did \emph{last}. For example, if you click on a winter coat, then I should recommend other winter coats, regardless of whether those items are currently in-season.

Even trivial models such as the item-to-item recommenders which we saw in \Cref{sec:neighborhood} implicitly made this assumption. For example, the recommendations 
%made using 
%from
%features 
such as `people who viewed X also viewed Y' 
%in \Cref{fig:alsoviewed} 
can be made based purely on the context of what is currently being viewed; the user's historical interactions---or their preferences---are not considered. In contrast the approaches we explore below generally try to combine both personal and contextual factors.

\paragraph{Markov chains}
The assumption described above---that the next action is conditionally independent\footnote{Two variables $a$ and $b$ are said to be \emph{conditionally independent} given a third variable $c$ if $p(a,b|c) = p(a|c)p(b|c)$. Essentially $c$ `explains' any dependence $a$ and $b$. In the case of Markov Chains, this assumption implies that the most recent event is sufficient to explain the next action's dependence on the history.
%\done{Quick writing. Looks okay}
} of the interaction history given the previous action---describes exactly the setting of a \emph{Markov Chain}. 
%\done{strictly speaking, I suppose a discrete time markov chain. Who cares}
%\done{could have a box about conditional independence}
%\done{Clean up probability notation}
Formally, given a sequence of interactions (among a discrete set of items $i \in I$) $i^{(1)} \ldots i^{(t)}$, a Markov Chain
%\done{Do I use this notation for the item set? No, just I}
assumes that the probability of the next interaction given the history can be written purely in terms of the previous interaction:
\begin{equation}
    p(i^{(t+1)} = i \mid i^{(t)} \ldots i^{(1)}) = p(i^{(t+1)} = i \mid i^{(t)})
    \label{eq:markov1}
\end{equation}

\emph{Personalized} Markov Chains generalize \Cref{eq:markov1} by allowing the probability of the next item to depend on both the previous item and the identity of the user $u$. That is, for a given user we have:
\begin{equation}
    p(i_u^{(t+1)} = i \mid i_u^{(t)} \ldots i_u^{1}) = p(i_u^{(t+1)} = i \mid i_u^{(t)}).
\end{equation}

What this means in practice is 
%just 
that when predicting a user's next action, our prediction should be a function of their previous action as well as their preference dimensions. 
Most of the time we can ignore the formalism of Markov Chains and more simply state that we are trying to fit a function of the form
\begin{equation}
    f(\overbrace{u,i}^{\mathclap{\text{score associated with the next interaction}}}|\underbrace{i_u^{(t-1)}}_{\mathclap{\text{given the user's previous interaction}}}).
    \label{eq:fMarkov}
\end{equation}
That is, where our previous models took a user and item as inputs (i.e.,~$f(u,i)$), or a user, item, and timestamp ($f(u,i,t)$), we now wish to model a user, an item, and the user's previous interaction. We might fit this function via a rating estimation framework (as in \cref{subsec:matrixFactor}), or a personalized ranking framework (as in \cref{subsec:BPR}), etc.

The key challenge in fitting models of the form in \Cref{eq:fMarkov} is that techniques like matrix factorization can no longer be straightforwardly applied. Whereas we previously modeled user/item interactions by factorizing a $U\times I$ matrix we must now factorize a $U \times I \times I$ \emph{tensor} (i.e.,~interactions between the user, the item, and the previous item).

We'll explore this idea by investigating specific
%\done{Wrong word. Looks okay}
implementations
from case studies in the following section.

\section{Case Studies: Markov-Chain Models for Recommendation}
\label{sec:markovCaseStudies}
%\done{Style for case study titles. Looks okay}

%\done{Markov Chain or Markov-Chain. Markov chain}

Below we describe various attempts to extend latent factor recommendation approaches to incorporate signals from the previous item.
%(though some incorporate additional information such as social signals). 
The main challenges involved include handling the large state space of interactions between users, items, and previous items; understanding the semantics that describe sequential relationships among items; and incorporating sequential dynamics with additional signals such as social information.

The models covered in this section are summarized in \Cref{tab:markovTable}.

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Markov-Chain models for personalized recommendation.]{Markov-Chain models for personalized recommendation. References: \protect\cite{rendle2010factorizing,cai2017spmc,feng2015personalized,he2017translation}.}
\label{tab:markovTable}
%\done{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{|XXX|}
% \textbf{Method} & \textbf{Reference} & \textbf{Description}\\
% FPMC & \cite{rendle2010factorizing} & The next item should be compatible with the user, as well as the previous item.\\
% SPMC & \cite{cai2017spmc} & Extends FPMC, incorporating a social term which states that the next item should be similar to our \emph{friends'} previous items.\\
% PRME & \cite{feng2015personalized} & Similar to FPMC, but measures compatibility via similarity in a metric space.\\
% Translation-based Recommendation & \cite{he2017translation} & Replaces the fixed user embedding $\gamma_u$ with a translation operation in latent item space.\\
% \end{tabularx}
% \begin{tabularx}{\linewidth}{|XXX|}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.34\textwidth}X}
\toprule
%\textbf{Method} & \textbf{Reference} & \textbf{Description}\\
\textbf{Ref.} & \textbf{Method} & \textbf{Description}\\
\midrule
%\cite{rendle2010factorizing}
R10 &
Factorized Personalized Markov Chains (FPMC)  & The next item should be compatible with the user, as well as the previous item (\cref{sec:FPMC}).\\
%\cite{cai2017spmc}
C17 &
Socially-Aware Personalized Markov Chains (SPMC) & Extends FPMC, incorporating a social term which states that the next item should be similar to our \emph{friends'} previous items (\cref{subsec:SocialSeqRec}).\\
%\cite{feng2015personalized}
F15 &
Personalized Ranking Metric Embedding (PRME) & Similar to FPMC, but measures compatibility via similarity in a metric space (\cref{sec:PRME}).\\
%\cite{he2017translation}
H17 &
Translation-based Recommendation & Replaces the fixed user embedding $\gamma_u$ with a translation operation in latent item space (\cref{sec:transRec}).\\
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{Style: table rows and items should end in full stop? Yes}

%\done{check all hlines}

%\done{Add extra table for temporal methods}

\subsection{Factorized Personalized Markov Chains}
\label{sec:FPMC}


%\done{Mention that FPMC is handled by Factorization Machines. Done}

%Cover: Factorized Personalized Markov Chains
An early paper that used Markov Chains for personalized recommendation was \emph{Factorizing Personalized Markov Chains for Next-Basket Recommendation} \citep{rendle2010factorizing}. 
%The paper was specifically concerned with predicting what 
Factorized Personalized Markov Chains (FPMC for short) predict what
items a user will purchase next, based on the items in their previous basket; customer basket data from \emph{Rossmann} (a German drugstore) was used to train and evaluate the model.

The basic premise of FPMC is that the contents of the previous basket should help to predict the contents of the next one, but also that the basket contents should be personalized to the user. This is achieved by fitting a function of the form
\begin{equation}
    %f(u,i;j)
    f(i|u,j)
    \label{eq:fuij}
\end{equation}
where $u$ is a user, $i$ is a potential item to be recommended, and $j$ is an item from the user's previous basket.\footnote{Although the original paper uses basket data, baskets are mostly a complication needed to handle their specific dataset. It is more straightforward to present the work by simply considering item sequences (which is how the method is often adopted by other papers). Where we write a term for a previous item, in \protect\cite{rendle2010factorizing} that expression would usually be replaced by a sum over items in the previous basket}. 

The paper discusses the difficulty of modeling sparse interactions of the form in \Cref{eq:fuij}, and explains how this might be addressed with tensor decomposition. The decomposition used is essentially a generalization of the matrix factorization schemes we saw in \Cref{chap:modelbasedRecsys}, where 
%$f(u,i;j)$ 
$f(i|u,j)$
decomposes into a series of pairwise factors:
\begin{equation}
    %f(u,i;j) = 
    f(i|u,j) = 
    \underbrace{\gamma^{(\mathit{ui})}_u \cdot \gamma^{(\mathit{iu})}_i\vphantom{\gamma^{(\mathit{iu})}_j}}_{f(i|u)} + \underbrace{\gamma^{(\mathit{ij})}_i \cdot \gamma^{(\mathit{ji})}_j\vphantom{\gamma^{(\mathit{iu})}_j}}_{f(i|j)} + \underbrace{\gamma^{(\mathit{uj})}_u \cdot \gamma^{(\mathit{ju})}_j\vphantom{\gamma^{(\mathit{iu})}_j}}_{f(u,j)}.
\end{equation}
The three terms above denote the user's compatibility with the next item ($\gamma^{(\mathit{ui})}_u \cdot \gamma^{(\mathit{iu})}_i$), the next item's compatibility with the previous item ($\gamma^{(\mathit{ij})}_i \cdot \gamma^{(ji)}_j$), and the user's compatibility with the previous item ($\gamma^{(\mathit{uj})}_u \cdot \gamma^{(\mathit{ju})}_j$). In practice, the latter expression cancels out when optimizing the model using a BPR-like framework (as we see in \Cref{eq:FPMC_BPR}, below), which is perhaps intuitive as the expression doesn't include the candidate item $i$; as such 
%that 
the factorization can be rewritten
\begin{equation}
    %f(u,i;j) = 
    f(i|u,j) =
    \overbrace{\gamma^{(\mathit{ui})}_u \cdot \gamma^{(\mathit{iu})}_i}^{\mathclap{\text{user's compatibility with next item}}} + \underbrace{\gamma^{(\mathit{ij})}_i \cdot \gamma^{(\mathit{ji})}_j}_{\mathclap{\text{next item's compatibility with previous item}}}.
    \label{eq:FPMC}
\end{equation}
%\done{Check that superscripts are consistent. Looks fine}
Intuitively, this factorization simply states that the next item should be compatible with both the user and the previous item consumed. Note that (as indicated by superscripts), item parameters are not shared between the terms $\gamma^{(\mathit{ui})}$, $\gamma^{(\mathit{iu})}$, $\gamma^{(\mathit{ij})}$, $\gamma^{(\mathit{ji})}$, i.e.,~we use separate sets of factors when modeling how an item interacts with a user vs.~another item.

%\done{Long section headings overlap with page numbers in header text. Check.}

Ultimately, the model is optimized using a BPR-like framework (\cref{subsec:BPR}), i.e., using a contrastive loss of the form
\begin{equation}
    % \sigma( f(u,i;j) - f(u,i';j) ),
    \sigma( f(i|u,j) - f(i'|u,j) ),
    \label{eq:FPMC_BPR}
\end{equation}
where $i'$ is a sampled negative item that the user did not consume. 
 
Experiments compare two variants of FPMC, which exclude either the sequential or the personal term.
%, which are essentially ablations. 
%Regular Matrix Factorization (MF), includes only the first term in \Cref{eq:FPMC}; 
That is, they model $f(i|u) = \gamma_u \cdot \gamma_i$ or $f(i|j) = \gamma_i \cdot \gamma_j$.
Excluding the sequential term reduces to regular matrix factorization (MF), as in \Cref{subsec:matrixFactor}. 
Excluding the personal term (which they term Factorized Markov Chains, or FMC), 
%includes only the second. 
captures `global' sequential dynamics that are common to all users.
The experiments thus measure (albeit on a specific dataset) the extent to which future actions can be explained by the previous action, versus overall historical preferences.

Ultimately FPMC outperforms both variants, though interestingly FMC and MF outperform each other under different conditions. Importantly, FMC is particularly effective in \emph{sparse} settings (i.e.,~few interactions per user/item) whereas MF works better on dense data.
%\done{Do I ever define ablation? No but it's gone}
%\done{(vs.~MF and FMC). Define FMC and MF}

We note briefly that methods like factorization machines (\cref{chap:contentStructure}, \Cref{sec:factorization}) can relatively straightforwardly be used to implement FPMC-like models: in addition to embedding user and item encodings,
%(as in \cref{eq_?}), 
one can simply augment the representation to include the previous item. We leave this as an exercise (Exercise~\ref{ex:seq1}), though also show how to implement FPMC in \emph{Tensorflow}
%\done{style}
in \Cref{subsec:fpmcTensor}.

\subsection{Socially-Aware Sequential Recommendation}
\label{subsec:SocialSeqRec}

Just as we saw how Bayesian Personalied Ranking can be augmented by sampling signals from social interactions in \Cref{subsec:socialBPR}, sequential models like FPMC can also be improved by leveraging social information.

\emph{Socially-Aware Personalized Markov Chains} (SPMC) \citep{cai2017spmc} extend FPMC (and Social BPR) by merging both temporal and social signals. The basic idea is to extend \Cref{eq:FPMC} from FPMC based on the reasoning that the user's next item should be similar to those that their friends have recently consumed:
%SPMC: Socially-Aware Personalized Markov Chains for Sparse Sequential recommendation
\begin{equation}
    % f(u,i;j) =
    f(i|u,j) =
    \gamma^{(\mathit{ui})}_u \cdot \gamma^{(\mathit{iu})}_i + 
    \gamma^{(\mathit{ij})}_i \cdot \gamma^{(\mathit{ji})}_j +
    |\mathcal S|^{-\alpha} \sum_{(v,k) \in \mathcal S} \underbrace{\sigma(\gamma^{(\mathit{uv})}_u \cdot \gamma^{(\mathit{uv})}_v)}_{\hspace{-15mm}\mathrlap{\text{similarity between $u$ and $v$}}} \overbrace{(\gamma^{(\mathit{ik})}_i \cdot \gamma^{(\mathit{ik})}_k)}^{\mathllap{\text{next item's compatibility with friend's previous item}}\hspace{-15mm}}.
    \label{eq:SBPR}
\end{equation}
%\done{check rendering. Looks okay}
%Here we are summing over $(v,k) \in \mathcal S$, is the set
Here the set $\mathcal S$ 
%\done{check notation against SBPR. Looks fine}
consists of only the most recent interactions $k$ by each of $u$'s friends $v$. The first term inside the summation $\sigma(\gamma^{(\mathit{uv})}_u \cdot \gamma^{(\mathit{uv})}_v)$ measures the similarity between $u$ and $v$, so that we only consider the effect of social influence if $u$ and $v$ are sufficiently similar. 
The term $|\mathcal S|^{-\alpha}$ normalizes the expression so that social influence does not saturate the other terms for users with a large number of friends. Note that there is no term $\gamma^{(\mathit{vu})}$ or $\gamma^{(\mathit{ki})}$ in \Cref{eq:SBPR} (i.e.,~only one set of representations is learned, rather than asymmetric representations as in \Cref{eq:FPMC}); this is done simply to reduce the number of model parameters.
%
\cite{cai2017spmc} show that including both temporal and social terms improves predictive performance over FPMC and Social BPR.
%\done{boring ending. Is fine}



\subsection{Locality-Based Sequential Recommendation}
\label{sec:PRME}

In \Cref{sec:whyInner}, we briefly 
%discussed how 
suggested that
different aggregation functions (besides the inner product) could be useful in various contexts. This proves to be the case in various sequential recommendation settings, where sequential actions follow some notion of \emph{locality}.

In \cite{feng2015personalized}, sequential recommendation schemes were studied in the setting of Point-of-Interest recommendation. In such a setting, the context of the previous action is particularly informative, since the following action is likely to be (geographically) close. We briefly touched upon this assumption 
%\done{did we? Yes}
when introducing location-based social networks in \Cref{chap:contentStructure} (\cref{subsec:location}).

If the actual semantics of the problem demand some notion of locality, then arguably similarity in the latent space should also be based on locality (rather than, say, an inner product).

The framework of \cite{feng2015personalized}, \emph{Personalized Ranking Metric Embedding} (PRME), models %transitions
sequential compatibility
%probabilities 
using an expression of the form:
%\done{equation spacing is bad, I guess because the previous line is long?}
\begin{equation}
    f(i|j) = -d(\gamma_i - \gamma_j)^2 = - \| \gamma_i - \gamma_j \|_2^2.
    \label{eq:PRME_sequential}
\end{equation}
% where $Z(i)$ is a normalization term:
% \begin{equation}
%     Z(i) = \sum_{k = 1}^{|I|}  e^{- \| \gamma_i - \gamma_k \|_2^2}
% \end{equation}
Note two differences between this model (PRME) and FPMC (\cref{sec:FPMC}):
\begin{itemize}
    \item The main difference is the use a of distance (actually a squared distance) function, so that sequential activities exhibit locality in the latent space.
    \item Unlike FPMC, which used separate latent spaces $\gamma^{(\mathit{ij})}$ and $\gamma^{(\mathit{ji})}$ for the next and previous item, PRME uses only a single latent space (which saves parameters).
    % \item Finally, the normalization factor allows the next item to be interpreted as a probability, though this is an implementation detail.
\end{itemize}
Like FPMC, PRME also includes an expression encoding the compatibility between the user and the item (again using a distance function), and also trains the model using a BPR-like framework (i.e.,~including a negative item $i'$ as in \Cref{eq:FPMC_BPR}). Other specific details include an explicit feature encoding geographical distance (based on latitude and longitude), and a temporal feature which down-weights the influence of the sequential term if sequential events are temporally far apart.

Though the authors of PRME argue that Euclidean distance is a more natural way of comparing sequential items (and show that PRME outperforms FPMC for POI recommendation), it should be noted that whether one similarity function is `better' than the other largely depends on the semantics of the specific problem and dataset.

Another paper makes use of a similar model for personalized playlist generation \citep{chen2012playlist}. Like PRME, their model (Factorized Markov Embeddings, or FME) notes that sequential songs in playlists tend to be highly localized, such that a metric embedding is possibly well-motivated. The compatibility function given a user $u$, song $i$, and previous song $j$ takes the form%
%\done{equation also has bad spacing above, check all of these}
\begin{equation}
    %f(u,i;j) = 
    f(i|u,j) =
    - d( \gamma^{(\mathrm{start})}_i - \gamma^{(\mathrm{end})}_j )^2 + \gamma_u \cdot \gamma'_i.
    \label{eq:FME_sequential}
    %\done{f(i|u,j) or f(i,u|j)? Though how I've done it is pretty sloppy}
\end{equation}
Note a few differences between FME and PRME:
\begin{itemize}
    \item FME uses a separate embedding for the next song ($\gamma^{(\mathrm{start})}$) and the previous song ($\gamma^{(\mathrm{end})}$). The basic idea being that songs in playlists should not just be highly local, but should gradually `transition' from one song to the next, so that the `start point' of the next song in latent space should be similar to the `end point' of the previous song (\cref{fig:translation}).
    \item FME uses a combination of both a distance function (for compatibility with the previous item) and an inner product (for compatibility with the user) in \Cref{eq:FME_sequential}. Again, this demonstrates that the correct choice of compatibility function is highly dependent on problem semantics.
\end{itemize}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/translationIdea.pdf}
\caption[Translation-based sequential models.]{Some sequential models use the principle of translation to model sequential transitions between items. \label{fig:translation}}
%\done{Add a second figure for transrec. don't bother, it's so similar}
\end{SCfigure}

\subsection{Translation-Based Recommendation}
\label{sec:transRec}

A third class of models for sequential recommendation are also based on the principle of translation. \cite{he2017translation} built recommender systems by borrowing ideas from knowledge-base completion.

Several techniques for knowledge base completion are based on the principle of learning low-dimensional embeddings that describe relationships among entities \citep{bordes2013translating,wang2014knowledge,10.5555/2886521.2886624}. The basic idea is to represent both entities and relationships as vectors in a low-dimensional space, such that a relation vector encodes how to `translate' between entities. For example, we might seek to learn vectors describing entities such as \emph{`Alan Turing'} and \emph{`England'}, such that given a vector describing the relation \emph{`born in'} we should have
\begin{equation}
    d(\overrightarrow{\text{Alan Turing}} + \overrightarrow{\text{born in}}, \overrightarrow{\text{England}}) \simeq 0,
\end{equation}
where $d$ is a Euclidean distance.

\emph{Translation-based recommendation} \citep{he2017translation} adapts this type of approach to personalized recommendation. Whereas for knowledge graph completion relations tell us how to traverse the space of entities, in a recommendation setting, \emph{items} fulfil the role of entities, and \emph{users} traverse the space of items. 
%Then, given two items $i$ and $j$ that are consumed in sequence, we should have
Then, given the previous item $j$ and the next item $i$ to be consumed in sequence,\footnote{In the original paper $j$ is the \emph{next} item and $i$ is the previous, though we reverse the order to maintain 
%consistency 
notation
across all methods. } we should have
\begin{equation}
    d(\gamma_j + \gamma_u, \gamma_i) \simeq 0.
\end{equation}

Training such a model is quite similar to how we trained FME and PRME in \Cref{sec:PRME}: that is, we fit a compatibility function between a user, an item, and a previous item (much like \Cref{eq:FME_sequential}):
\begin{equation}
    %f(j|u,i) = \beta_j - \| \gamma_i + \gamma_u - \gamma_j \|_2,
    %f(u,i;j) = \beta_i - \| \gamma_j + \gamma_u - \gamma_i \|_2,
    f(i|u,j) = \beta_i - \| \gamma_j + \gamma_u - \gamma_i \|_2,
\end{equation}
where 
$\beta_i$
%$\beta_j$ 
is incorporated so that the method is capable of capturing overall item popularity as well as preferences. \cite{he2017translation} further constrains item representations to live on a unit ball (i.e.,~$\|\gamma_i\|_2^2 = 1$), which was found to be effective in the knowledge graph completion settings above.

Conceptually, the above model corresponds to users following a `trajectory' through their interactions over time (\cref{fig:translation}).
In principle, this ought to mean that related items (e.g.~sequential songs in a playlist) should be aligned to form a chain of equally-spaced items in the latent space. In practice, such complex dynamics are unlikely to emerge from the model; rather, like other temporal modeling approaches, the model benefits from its parsimony (i.e.,~it has much fewer parameters than other sequential models, due to the use of only a single latent space) but is still able to capture common sequential patterns even in sparse datasets.


\subsection{FPMC in \emph{Tensorflow}}
\label{subsec:fpmcTensor}
%\done{formatting}

Although several of the above models can be implemented via an appropriately designed factorization machine (Exercises~\ref{ex:seq1} and \ref{ex:fism}), it is worth briefly describing how a sequential model might be implemented `from scratch;' this will be useful when implementing variants that do not straightforwardly map to existing architectures (such as the factorization machine of \Cref{sec:factorization}) or libraries.

Here we implement the Factorized Personalized Markov Chain (FPMC) method from \Cref{sec:FPMC}, though the code can straightforwardly be adapted to implement 
%any of the 
other
sequential methods discussed in \Cref{sec:markovCaseStudies}.

We build our solution on top of our Bayesian Personalized Ranking implementation from \Cref{subsec:BPRTensorflow}. First, when parsing the data, we must be careful to process the timestamp:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
for d in parse('goodreads_reviews_comics_graphic.json.gz'):
    u = d['user_id']
    i = d['book_id']
    t = d['date_added'] # Raw timestamp string
    r = d['rating']
    dt = dateutil.parser.parse(t) # Structured timestamp
    t = int(dt.timestamp()) # Integer timestamp
    if not u in userIDs: userIDs[u] = len(userIDs)
    if not i in itemIDs: itemIDs[i] = len(itemIDs)
    interactions.append((t,u,i,r))
    interactionsPerUser[u].append((t,i,r))
\end{lstlisting}
\end{minipage}

Note the use of the \texttt{dateutil} library to process the timestamp. The original timestamp in this dataset consists of raw strings (e.g.~`Wed Apr 03 10:10:41 -0700 2013'); the operation \texttt{dt = dateutil.parser.parse(t)} converts this to a structured format; this can be used to extract features associated with the timestamp, e.g.~\texttt{dt.weekday()} reveals that this date is a Wednesday, which might be useful for extracting features for temporal models such as those in \Cref{sec:temporalFeatures}.\footnote{While fairly obvious in this instance, determining even such simple properties is difficult for certain date formats.}
To build a sequential recommender, we are mainly interested in determining the \emph{sequence order} of the interactions; to do so we call \texttt{dt.timestamp()}. For the date above this returns 1365009041, which represents the number of seconds since January 1, 1970 (`unix time'). Such a time representation, while seemingly fairly arbitrary, is useful when our goal is simply to sort observations 
%in order of time, 
chronologically,
as we do when building sequential recommenders.
%\done{hastily written. Fine}

Next we sort each user's history by time, and augment our interaction data such that each interaction $(u,i)$ includes the previous item $j$. We also add a `dummy' item which acts as the previous item for the first observation:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
itemIDs['dummy'] = len(itemIDs)
interactionsWithPrevious = []

for u in interactionsPerUser:
    interactionsPerUser[u].sort()
    lastItem = 'dummy'
    for (t,i,r) in interactionsPerUser[u]:
        interactionsWithPrevious.append((t,u,i,lastItem,r))
        lastItem = i
\end{lstlisting}
\end{minipage}

Given these augmented interactions, we can modify the model from \Cref{subsec:BPRTensorflow} to include the additional terms from \Cref{eq:FPMC}. Here we train in a BPR-like setting (i.e.,~including a sampled negative item $k$), though we could similarly adapt the model for rating prediction following code from \Cref{subsec:LFMTensorflow}. Omitting a few boilerplate elements, the model equation (\cref{eq:FPMC_BPR}) becomes:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
gamma_ui = tf.nn.embedding_lookup(self.gammaUI, u)
gamma_iu = tf.nn.embedding_lookup(self.gammaIU, i)
gamma_ij = tf.nn.embedding_lookup(self.gammaIJ, i)
gamma_ji = tf.nn.embedding_lookup(self.gammaJI, j)
# (etc.)
x_uij = beta_i +\
        tf.reduce_sum(tf.multiply(gamma_ui, gamma_iu), 1) +\
        tf.reduce_sum(tf.multiply(gamma_ij, gamma_ji), 1)
x_ukj = beta_k +\
        tf.reduce_sum(tf.multiply(gamma_uk, gamma_ku), 1) +\
        tf.reduce_sum(tf.multiply(gamma_kj, gamma_jk), 1)
return -tf.reduce_mean(tf.math.log(tf.math.sigmoid(x_uij - x_ukj)))
\end{lstlisting}
\end{minipage}
%\done{Fix formatting}

The code above could be straightforwardly adapted to implement other sequential models, such as PRME (\cref{sec:PRME}) or translation-based recommendation (\cref{sec:transRec}).

\section{Recurrent Networks}
\label{sec:rnn}

%https://colah.github.io/posts/2015-08-Understanding-LSTMs/
%https://arxiv.org/pdf/1409.2329.pdf

A fundamental limitation of the Markov-Chain-based models we saw in \Cref{sec:markov} is that they have a very limited notion of `memory,' due to the assumption that the next event is conditionally independent of all historical events, given the most recent observation. This assumption may be sufficient in certain scenarios, such as recommendation settings that are highly dependent on the context of the previously clicked item (for example). However, as we begin to model text data (\cref{chap:text}), or sequence data such as heart-rate logs (\cref{sec:caseStudyHR}), 
%which have 
or more complex recommendation scenarios,
we will need to handle
longer-term semantics (such as grammatical structures in a sentence, or even an individual's level of `fatigue' in a heart-rate trace).

%\done{balance all parens}

\emph{Recurrent Neural Networks} (RNNs) seek to achieve this notion of `memory' by maintaining a `hidden state' during each step.\footnote{Note that `simpler' models attempt to accomplish the same goal within the framework of Markov-Chain-based models; see e.g.~Hidden Markov Models. However Recurrent Neural Networks are more typical of current practice and form the basis of models we develop later.
%Note: Hidden Markov Models also do this...
} The hidden state is a vector of latent variables that somehow capture the `context' that the model needs to know to capture the long-term semantics of the problem. Formally, we can visualize the RNN as taking a sequence of inputs ($x_1 \ldots x_N$), and producing a sequence of outputs ($y_1 \ldots y_N$), and maintaining hidden states that update at each step ($h_1 \ldots h_N$). We might visualize this model as follows:
\begin{center}
% \includegraphics[scale=0.25]{fig/rnn1.pdf}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
%itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]
itemc/.style={item,on chain,join},font= \scriptsize]
%\done{>=LaTeX above controls the arrow style. Is it good? Looks fine this way}
%  \begin{scope}[start chain=going right,nodes=itemc,every
%  join/.style={-latex,very thick},local bounding box=chain]
%  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
%  {$A$};
%  \end{scope}
%  \node[left=1em of chain,scale=2] (eq) {$=$};

 \node(x01) {$x_{t-1}$};
 \node[right=of x01] (x02) {$x_{t}$};
 \node[right=of x02] (x03) {$x_{t+1}$};
 \node[right=of x03] (x04) {$x_{t+2}$};

 \node[item, above=of x01, yshift=-10pt, minimum width=5mm] (x11) {};
 \node[left=of x11, xshift=10pt] (x10) {$h_{t-2}$};
 \node[item, above=of x02, yshift=-10pt, minimum width=5mm] (x12) {};
 \node[item, above=of x03, yshift=-10pt, minimum width=5mm] (x13) {};
 \node[item, above=of x04, yshift=-10pt, minimum width=5mm] (x14) {};
 \node[right=of x14, xshift=-10pt] (x15) {$h_{t+2}$};
 
%  \node[item, above=of x11, minimum width=5mm] (x21) {};
%  \node[left=of x21, xshift=10pt] (x20) {$h^{l}_{t-2}$};
%  \node[item, above=of x12, minimum width=5mm] (x22) {};
%  \node[item, above=of x13, minimum width=5mm] (x23) {};
%  \node[item, above=of x14, minimum width=5mm] (x24) {};
%  \node[right=of x24, xshift=-10pt] (x25) {$h^{l}_{t+2}$};

 \node[above=of x11, yshift=-10pt] (x31) {$y_{t-1}$};
 \node[above=of x12, yshift=-10pt] (x32) {$y_{t}$};
 \node[above=of x13, yshift=-10pt] (x33) {$y_{t+1}$};
 \node[above=of x14, yshift=-10pt] (x34) {$y_{t+2}$};

%  \draw [->] (x11) -- node[right] {$h^{l-1}_{t-1}$} (x21);
%  \draw [->] (x12) -- node[right] {$h^{l-1}_{t}$} (x22);
%  \draw [->] (x13) -- node[right] {$h^{l-1}_{t+1}$} (x23);
%  \draw [->] (x14) -- node[right] {$h^{l-1}_{t+2}$} (x24);
 
 \draw [->] (x11) -- node[above] {$h_{t-1}$} (x12);
 \draw [->] (x12) -- node[above] {$h_{t}$} (x13);
 \draw [->] (x13) -- node[above] {$h_{t+1}$} (x14);
 \draw [->] (x14) -- node[right] {} (x15);
 
%  \draw [->] (x21) -- node[above] {$h^{l}_{t-1}$} (x22);
%  \draw [->] (x22) -- node[above] {$h^{l}_{t}$} (x23);
%  \draw [->] (x23) -- node[above] {$h^{l}_{t+1}$} (x24);
%  \draw [->] (x24) -- node[right] {} (x25);
 
 \draw [->] (x10) -- (x11);
%  \draw [->] (x20) -- (x21);
 
 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);
 
 \draw [->] (x11) -- (x31);
 \draw [->] (x12) -- (x32);
 \draw [->] (x13) -- (x33);
 \draw [->] (x14) -- (x34);
\end{tikzpicture}
\end{center}
The RNN cell
%depicted here as a box, 
%\done{not a box. Removed}
is now responsible for determining how the hidden state should change at each step, and what outputs should be generated.

More complex RNN models repeat this idea across multiple \emph{layers}, so that RNN cells can be stacked, e.g.:
%\done{improve figures, latex renders too small. Replaced}
\begin{center}
%OLD
% \includegraphics[scale=0.25]{fig/rnn2.pdf}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
%itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]
itemc/.style={item,on chain,join},font= \scriptsize]
%\done{>=LaTeX above controls the arrow style. Is it good? Looks fine}
%  \begin{scope}[start chain=going right,nodes=itemc,every
%  join/.style={-latex,very thick},local bounding box=chain]
%  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
%  {$A$};
%  \end{scope}
%  \node[left=1em of chain,scale=2] (eq) {$=$};



 \node(x01) {$h^{l-2}_{t-1}$};
 \node[right=of x01] (x02) {$h^{l-2}_{t}$};
 \node[right=of x02] (x03) {$h^{l-2}_{t+1}$};
 \node[right=of x03] (x04) {$h^{l-2}_{t+2}$};

 \node[item, above=of x01, yshift=-10pt, minimum width=5mm] (x11) {};
 \node[left=of x11, xshift=10pt] (x10) {$h^{l-1}_{t-2}$};
 \node[item, above=of x02, yshift=-10pt, minimum width=5mm] (x12) {};
 \node[item, above=of x03, yshift=-10pt, minimum width=5mm] (x13) {};
 \node[item, above=of x04, yshift=-10pt, minimum width=5mm] (x14) {};
 \node[right=of x14, xshift=-10pt] (x15) {$h^{l-1}_{t+2}$};
 
 \node[item, above=of x11, minimum width=5mm] (x21) {};
 \node[left=of x21, xshift=10pt] (x20) {$h^{l}_{t-2}$};
 \node[item, above=of x12, minimum width=5mm] (x22) {};
 \node[item, above=of x13, minimum width=5mm] (x23) {};
 \node[item, above=of x14, minimum width=5mm] (x24) {};
 \node[right=of x24, xshift=-10pt] (x25) {$h^{l}_{t+2}$};

 \node[above=of x21, yshift=-10pt] (x31) {$h^{l}_{t-1}$};
 \node[above=of x22, yshift=-10pt] (x32) {$h^{l}_{t}$};
 \node[above=of x23, yshift=-10pt] (x33) {$h^{l}_{t+1}$};
 \node[above=of x24, yshift=-10pt] (x34) {$h^{l}_{t+2}$};

 \draw [->] (x11) -- node[right] {$h^{l-1}_{t-1}$} (x21);
 \draw [->] (x12) -- node[right] {$h^{l-1}_{t}$} (x22);
 \draw [->] (x13) -- node[right] {$h^{l-1}_{t+1}$} (x23);
 \draw [->] (x14) -- node[right] {$h^{l-1}_{t+2}$} (x24);
 
 \draw [->] (x11) -- node[above] {$h^{l-1}_{t-1}$} (x12);
 \draw [->] (x12) -- node[above] {$h^{l-1}_{t}$} (x13);
 \draw [->] (x13) -- node[above] {$h^{l-1}_{t+1}$} (x14);
 \draw [->] (x14) -- node[right] {} (x15);
 
 \draw [->] (x21) -- node[above] {$h^{l}_{t-1}$} (x22);
 \draw [->] (x22) -- node[above] {$h^{l}_{t}$} (x23);
 \draw [->] (x23) -- node[above] {$h^{l}_{t+1}$} (x24);
 \draw [->] (x24) -- node[right] {} (x25);
 
 \draw [->] (x10) -- (x11);
 \draw [->] (x20) -- (x21);
 
 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);
 
 \draw [->] (x21) -- (x31);
 \draw [->] (x22) -- (x32);
 \draw [->] (x23) -- (x33);
 \draw [->] (x24) -- (x34);
\end{tikzpicture}
\end{center}
In this depiction (and in many treatments of the topic), the model is described only in terms of hidden states: the model receives an input, updates its hidden state, and passes this state to the next timestep and the next layer. The first layer may receive inputs (i.e.,~observed values $x$) while the last layer generates outputs (i.e., $y$).

When designing this cell, one might consider the types of dynamics required to model state transitions between successive steps:
\begin{itemize}
    \item Based on the current hidden state, what output should be generated?
    \item How should the hidden state change as a function of the input that was just seen?
    \item What part of the hidden state should be preserved, and what part can be discarded?
    \item How can hidden state be preserved across long interaction sequences?
\end{itemize}
 
Below we describe one particular implementation of an RNN cell. Although there is nothing particularly sacred about the particular model we present, it is representative of the overall design approach, in terms of capturing the features described above.
%\done{Needs some text here and below. Somewhat useless paragraph}

\subsection{The Long Short-Term Memory Model}
\label{subsec:LSTM}


The Long Short-Term Memory Model \citep{hochreiter1997long} is a specific implementation of an RNN that has been popularized especially for use in text generation tasks, as we'll discuss in \Cref{chap:text}.
%\done{poorly written}

A challenge in designing RNN cells as described above is how to encourage them to `remember' state across long sequences. To achieve this, the LSTM cell (\cref{fig:LSTMcell}) preserves the cell state ($c$ in the equation below), mostly unmodified across steps.
Other components are responsible for `forgetting' part of the state ($f$), as a function of the current input and previous hidden state; updating the cell state ($i$ and $g$); updating the hidden state ($h$); and determining what to output ($o$).
Although not particularly relevant for the current discussion---the models discussed here could easily be replaced by different architectures---the specific form of these components in an LSTM cell is as follows:
%\done{not great}
\begin{align}
 f_t^l & =  \sigma(W_l^{(f)} \times [h_t^{l-1}; h_{t-1}^l]) \label{eq:lstmFirst}\\ % forget
 i_t^l & =  \sigma(W_l^{(i)} \times [h_t^{l-1}; h_{t-1}^l])\\ % input gate
 o_t^l & =  \sigma(W_l^{(o)} \times [h_t^{l-1}; h_{t-1}^l])\\ % output
 g_t^l & =  \tanh(W_l^{(g)} \times [h_t^{l-1}; h_{t-1}^l])\\ % input gate
 c_t^l & =  g_t^l \times
 %\odot 
 i_t^l + c_{t-1}^l 
 %\odot 
 \times
 f_t^l\\ % cell state
 h_t^l & =  \tanh(c_t^l) 
 %\odot
 \times
 o_t^l. \label{eq:lstmLast} % hidden
 %\done{Called \tilde{c} in the figure, fix}
 %\done{Nothing in the figure has layers}
\end{align}


\begin{figure}
    \centering
\begin{tikzpicture}[
    % GLOBAL CFG
    font= \scriptsize,
    %font=\sf \scriptsize,
    % >=LaTeX,
    % Styles
    cell/.style={% For the main box
        rectangle, 
        rounded corners=5mm, 
        draw,
        thick,
        %very thick,
        },
    operator/.style={%For operators like +  and  x
        circle,
        draw,
        inner sep=-0.5pt,
        minimum height =.2cm,
        },
    function/.style={%For functions
        ellipse,
        draw,
        inner sep=1pt
        },
    ct/.style={% For external inputs and outputs
        circle,
        draw,
        %line width = .75pt,
        %minimum width=0.8cm,
        line width = .6pt,
        minimum width=0.8cm,
        inner sep=1pt,
        },
    gt/.style={% For internal inputs
        rectangle,
        draw,
        minimum width=4mm,
        minimum height=3mm,
        inner sep=1pt
        },
    mylabel/.style={% something new that I have learned
        %font=\scriptsize\sffamily
        },
    ArrowC1/.style={% Arrows with rounded corners
        rounded corners=.25cm,
        %thick,
        },
    ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        %thick,
        },
    ]

%Start drawing the thing...    
    % Draw the cell: 
    \node [cell, minimum height =4cm, minimum width=6cm] at (0,0){} ;

    % Draw inputs named ibox#
    \node [gt] (ibox1) at (-2,-0.75) {$\sigma$};
    \node [gt] (ibox2) at (-1.5,-0.75) {$\sigma$};
    \node [gt, minimum width=1cm] (ibox3) at (-0.5,-0.75) {$\tanh$};
    \node [gt] (ibox4) at (0.5,-0.75) {$\sigma$};

   % Draw opérators   named mux# , add# and func#
    \node [operator] (mux1) at (-2,1.5) {$\times$};
    \node [operator] (add1) at (-0.5,1.5) {+};
    \node [operator] (mux2) at (-0.5,0) {$\times$};
    \node [operator] (mux3) at (1.5,0) {$\times$};
    \node [function] (func1) at (1.5,0.75) {$\tanh$};

    % Draw External inputs? named as basis c,h,x
    \node[ct, label={[mylabel]\emph{cell}}] (c) at (-4,1.5) {\empt{c}{t-1}};
    \node[ct, label={[mylabel]\emph{hidden}}] (h) at (-4,-1.5) {\empt{h}{t-1}};
    %\node[ct, label={[mylabel]left:\emph{input}}] (x) at (-2.5,-3) {\empt{x}{t}};
    % \node[ct, label={[mylabel]left:\emph{input}}] (x) at (-2.5,-3) {\empt{h}{t-1}};
    \node[ct, label={[mylabel]left:\emph{input}}] (x) at (-2.5,-3) {$h_{t}^{l-1}$};

    % Draw External outputs? named as basis c2,h2,x2
    \node[ct, label={[mylabel]\emph{cell}}] (c2) at (4,1.5) {\empt{c}{t}};
    \node[ct, label={[mylabel]\emph{hidden}}] (h2) at (4,-1.5) {\empt{h}{t}};
    \node[ct, label={[mylabel]left:\emph{output}}] (x2) at (2.5,3) {\empt{h}{t}};

% Start connecting all.
    %Intersections and displacements are used. 
    % Drawing arrows    
    \draw [ArrowC1] (c) -- (mux1) -- (add1) -- (c2);

    % Inputs
    \draw [ArrowC2] (h) -| (ibox4);
    \draw [ArrowC1] (h -| ibox1)++(-0.5,0) -| (ibox1); 
    \draw [ArrowC1] (h -| ibox2)++(-0.5,0) -| (ibox2);
    \draw [ArrowC1] (h -| ibox3)++(-0.5,0) -| (ibox3);
    \draw [ArrowC1] (x) -- (x |- h)-| (ibox3);

    % Internal
    \draw [->, ArrowC2] (ibox1) -- node[left] {$f_t^l$} (mux1);
    \draw [->, ArrowC2] (ibox2) |- node[left] {\ \hspace{2mm}$i_t^l$\hspace{-2mm}\ } (mux2);
    %\done{Move less left. Hacky though}
    \draw [->, ArrowC2] (ibox3) -- node[left] {$g_t^l$} (mux2);
    \draw [->, ArrowC2] (ibox4) |- node[left] {\ \hspace{2mm}$o_t^l$\hspace{-2mm}\ } (mux3);
    %\done{Move less left}
    \draw [->, ArrowC2] (mux2) -- (add1);
    \draw [->, ArrowC1] (add1 -| func1)++(-0.5,0) -| (func1);
    \draw [->, ArrowC2] (func1) --  (mux3);
    %Outputs
    \draw [-, ArrowC2] (mux3) |- (h2);
    \draw (c2 -| x2) ++(0,-0.1) coordinate (i1);
    \draw [-, ArrowC2] (h2 -| x2)++(-0.5,0) -| (i1);
    \draw [-, ArrowC2] (i1)++(0,0.2) -- (x2);

\end{tikzpicture}
    \caption[Visualization of an LSTM cell.]{Visualization of an LSTM cell (\protect\Crefrange{eq:lstmFirst}{eq:lstmLast}).}
    \label{fig:LSTMcell}
\end{figure}
%\done{rewrite to avoid h^{<t>} to be consistent with other notation}
% Explain the cell state
%https://colah.github.io/posts/2015-08-Understanding-LSTMs/




Several variants of the above have been proposed to incorporate additional components, mostly consisting of specific differences to 
%the specific activations above 
how the state is preserved and transformed among the equations above. It should be remarked though that the overall view of a `cell,' in terms of inputs, outputs, and hidden states, is largely interchangeable across models in spite of differences in specific details.






\section{Neural Network-Based Sequential Recommenders}
\label{sec:rnnRec}

Having developed a basic understanding of the structure of neural network-based models (including some `basic' architectures in \Cref{subsec:MLPRec} and recurrent networks in \Cref{sec:rnn}), we now have the basic building blocks to understand how recurrent networks can be used to build personalized models of sequences.
%
%A variety of modern approaches to modeling temporal and sequential dynamics of user behavior are built on top of neural network models. 
%All of the models we'll explore in this section are based on neural networks.
Although we won't present a full treatment of such approaches here (since some depend on techniques and models quite different from those covered in this book), here we outline the general directions explored by a few representative examples.

%Neural network-based approaches have emerged as the state-of-the-art for a variety of problems in natural language processing (as well as in computer vision, as we discuss in \Cref{chap:visual}, and elsewhere).
%\done{funny wording. It's okay}
%Techniques range from recurrent neural networks (similar to those we introduced in \Cref{sec_?}), to recent approaches based on self-attention and Transformer architectures. 

%\done{discuss relationship to text}

\subsubsection{Relationship to natural language processing}

Many of the models we'll study in this section are closely related to approaches used to model natural language. As we'll discuss,
%below, 
the development of modern sequential recommenders 
%very closely follows 
has closely followed
the development of state-of-the-art natural language models, ranging from recurrent networks \citep{hidasi2015session}, attention mechanisms \citep{li2017neural}, self-attention \citep{kang2018self}, and BERT-based models \citep{devlin2018bert}.

%The appeal of applying models from natural language
The basic mechanism of applying natural language models to sequential data is fairly straightforward. 
%just as models of text model sequences of words in documents, in a recommendation setting
In a natural language setting,
documents are represented as sequences of discrete tokens (words); a model (such as that of \Cref{subsec:LSTM}) might be trained to estimate the next word in a sequence.
%might be trained across many documents to predict the next word in sequence. 
Likewise, in a recommendation setting, a user's sequence of actions can be represented as a sequence of discrete tokens (items). Via this analogy, state-of-the-art models from natural language can be fairly straightforwardly adapted to describe the dynamics of user interaction sequences. We'll discuss this relationship more when designing models specifically for text in \Cref{chap:text}.

%Broadly speaking,
Conceptually,
the appeal of neural network approaches to sequential modeling (and more broadly, the appeal of borrowing models from natural language) is that they allow models to capture complex \emph{syntax} or \emph{semantics} in sequential data.
%\done{Organization not so good, since this comes after LSTMs have been described. Probably adjust. Okay}
So far, when developing temporal and sequential recommenders we have mostly focused on either of (a) developing simple parametric functions to capture long-term dynamics, as we did in \Cref{sec:caseStudyNetflixTemporal}; or (b) modeling sequential dynamics using the context of previous interactions, following a Markov chain-like setup, as in \Cref{sec:markov}. Both approaches have limitations: the former tends to require carefully hand-crafting models around specific dynamics, while the latter potentially captures only limited context (e.g.~a single previous interaction). In principle, the techniques in this section aim to address these issues, by automatically uncovering both long- and short-term dynamics simultaneously, or potentially capturing complex interrelationships between items in a sequence.
%\done{Check all instances of (a) (1) etc., maybe drop. Always (a) (b)}


% Below we describe a few specific instances of models that use neural networks to model user interaction sequences. However ultimately the specific models and architectures are not particularly import: our intention here is to highlight the general approach by which state-of-the-art models, from sequence mining or natural language processing, can be adapted to model user interaction sequences.
%\done{too similar to opening paragraph. Dropped}
%\done{not great paragraph. Fine}


\subsubsection{`User-free' sequential recommendation}
%\done{rename section. Seems okay}

When introducing sequential models based on Markov Chains in \Cref{sec:markov}, we argued that it was valuable to model \emph{both} the compatibility between the user and the item, as well as the compatibility between adjacent items in a sequence. In short, both sources provide useful and complementary signals that each explain variation in users' activities.

However it is unclear whether this argument applies in the limit. Clearly, a single previous interaction does not provide the full context of a user's preferences; however, a model that includes \emph{enough} historical interactions can possibly capture all of the necessary context without explicitly including any user term (we explored this idea a little when introducing FISM in \Cref{subsec:fism}).

%Below we describe several attempts to model users' historical interaction sequences without explicitly including a user term. 
It is interesting to note that \emph{none of the models we describe below learn explicit user representations}, relying instead on item representations plus a sufficient window of \emph{context} to capture the dynamics of individual users.
Doing so is conceptually appealing as an alternative to (e.g.)~the cold-start models we discussed in \Cref{sec:coldStart}: by eliminating the need for an explicit user term, we can quickly make effective recommendations by observing a few user actions, without needing to retrain the model (to fit user terms) or rely on side-information.

%As we see below (\cref{sec:RNNSeqRec}), 
%This type of `user-free' sequential model is a growing trend in deep learning-based recommendation. Borrowing ideas from natural language processing, recent sequential models are able to model dynamics across long user histories without needing an explicit user term. 
Such models fall somewhere in between the memory-based approaches we studied in \Cref{chap:recsysIntro} and model-based approaches we saw in \Cref{chap:modelbasedRecsys}: although item representations are learned from data, user representations must be captured implicitly by the model `remembering' the context among their sequential actions.
%\done{could drop. Fine for now}


The models covered in this section are summarized in \Cref{tab:userFreeSequence}.


%summarizes the models we cover in this section.

% \section{Neural-Network Based Sequential Models}

% \subsection{Neural Network-Based Approaches}


%\done{elevate to section. Adjusted}


%\done{Table formatting}
\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Summary of deep-learning based sequential models.]{Summary of deep-learning based sequential models. References: \protect\cite{hidasi2015session,li2017neural,kang2018self,sun2019bert4rec}.}
%\done{could be two tables? Is}
\label{tab:userFreeSequence}
%\done{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{|XXX|}
% Method & Reference & Description\\
% %\done{column ordering across tables}
% ? & \cite{hidasi2015session} & Item sequences are passed to a recurrent network; the network's hidden state is used to predict the next interaction.\\
% Neural Attentive Recommendation (NARM) & \cite{li2017neural} & Combines an RNN \cite{hidasi2015session}, but with the addition of an attention mechanism that operates on the sequence of network latent states in order to `focus' on relevant interactions.\\
% Self-Attentive Sequential Recommendation & \cite{kang2018self} & Similar to NARM, but uses the principle of \emph{self-attention} (i.e.,~a Transformer model) rather than a recurrent network.\\
% BERT4Rec & \cite{sun2019bert4rec} & Also uses the principle of self-attention, though with a different architecture (based on BERT \cite{devlin2018bert}).\\
% \end{tabularx}
% \begin{tabularx}{\linewidth}{|XXX|}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.22\textwidth}X}
\toprule
\textbf{Ref.} & \textbf{Method} & \textbf{Description}\\
\midrule
%\done{column ordering across tables}
% \cite{hidasi2015session}
H15 &
Session-based Recommendation with RNNs &  Item sequences are passed to a recurrent network; the network's hidden state is used to predict the next interaction (\cref{subsec:rnnRec2}).\\
%\cite{li2017neural}
L17 &
Neural Attentive Recommendation (NARM) & Combines an RNN (similar to that of \cite{hidasi2015session}) 
%\done{garbled sentence. Fixed}
%but with the addition of 
with an attention mechanism that operates on the sequence of network latent states in order to `focus' on relevant interactions (\cref{subsec:attention}).\\
% \cite{kang2018self}
KM18 &
Self-Attentive Sequential Recommendation & Similar to NARM, but uses the principle of \emph{self-attention} (i.e.,~a Transformer model) rather than a recurrent network (\cref{subsec:attention}).\\
% \cite{sun2019bert4rec}
S19 &
BERT4Rec & Also uses the principle of self-attention, though with a different architecture based on BERT \citep{devlin2018bert} (\cref{subsec:attention}).\\
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{not referenced}
%\done{this whole table is too far right for some reason?}




\subsection{Recurrent Network-Based Recommendation}
\label{subsec:rnnRec2}
%\done{paragraph title. Approaches to what?}
%\done{paragraph title? Is fine}

%\done{Probably move this to text section, so that it's along side item2vec and similar -- maybe whole section about adapting NLP models. No}

An early paper to explore the use of recurrent networks for recommendation did so using the types of model we developed in \Cref{sec:rnn}.\footnote{Specifically they used a Gated Recurrent Unit, or GRU, though methodologically the approach is similar to the LSTM-based techniques we explored in \protect\Cref{subsec:LSTM}.}.
%
\cite{hidasi2015session} explored the problem of \emph{session-based} recommendation, where user interactions are divided into distinct `sessions' (typically using some heuristic based on interaction timestamps). This setting is typical for recommenders based on recurrent networks (or more broadly, for recommenders that borrow techniques from natural language), since it allows sessions to be treated analogously to \emph{sentences,} i.e.,~short sequences of discrete tokens (items).

%\done{Needs to be after item2vec. Changed}

% Conceptually, the method is fairly similar to \emph{item2vec} (\cref{sec:item2vec}). Recall
%\done{only makes sense if after item2vec. Changed}
%that item2vec learns representations of items such that adjacent items in a sequence (e.g.~items that were both clicked on during a short period) have compatible representations. Likewise, 
The method from \cite{hidasi2015session} seeks to pass sequences of items into a recurrent network (similar to that of \Cref{subsec:LSTM}) such that the hidden state of the network is capable of predicting the next interaction.\footnote{The specific architecture used to achieve this passes the final hidden state into a feed-forward network (similar to those we studied in \protect\Cref{subsec:MLPRec}), whose final layer estimates scores associated with all items; while computationally expensive, this is feasible for datasets with relatively small item vocabularies, e.g.~in the low tens-of-thousands for data in \protect\cite{hidasi2015session}.}
%\done{clunky sentence in footnote. Seems okay}
%
%Note that this method (along with most of the related methods below) does not learn a user representation: rather the user `context' is captured via the hidden state of the recurrent network. As such, these methods are a form of `contextual personalization' in the sense that there is no user model as such.
As stated previously, this model (along with most of the related methods below) does not learn a user representation: rather the user `context' is captured via the hidden state of the recurrent network. 
%As such, these methods are a form of `contextual personalization' in the sense that there is no user model as such.
%\done{argument might go above. In text section, could go before item2vec too. Fine}
\cite{hidasi2015session} argue that a benefit of this type of approach are that it will work well in situations where long user histories are unavailable (e.g.~on niche platforms or long-tailed datasets). In such cases, techniques like those we developed for the Netflix prize (\cref{sec:caseStudyNetflixTemporal}) are unreliable as effective user representations ($\gamma_u$) cannot be learned from only a few interactions.
%; \cite{hidasi2015session} suggests the scenario of recommendation on niche online platforms.
%\done{e-Commerce or e-commerce? lowercase. Hate the word though. Gone}
%This possible benefit of `user-free' sequence modeling approaches is put into more context in our broader discussion of the benefits of deep learning-based recommendation in \Cref{sec_?}.
%\done{maybe rewrite the above depending on where this discussion is. Removed}
%\done{Make sure this point about user-free models is actually in that discussion. Removed}

\subsection{Attention Mechanisms}
\label{subsec:attention}

The main argument in favor of recurrent network-based approaches is that they can potentially capture longer-term sequential semantics compared to (e.g.)~the Markov chain-based approaches we studied in \Cref{sec:markov}. On the other and, the fact that simple Markov chain-based models are so effective in the first place suggests that even the context of a single recent item might be enough to capture a user's `context' in many cases.

The basic 
%goal of 
intuition behind
incorporating \emph{attention mechanisms} into sequential recommender systems is to help the model to `focus' on a small set of interactions among a relatively longer interaction sequence. Intuitively, this ought to allow the model to capture context from long interaction sequences, while still leveraging the fact that the \emph{relevant} part of the context may only consist of a few interactions
\citep{kang2018self}. Attention mechanisms have been used in other settings such as image captioning
\citep{xu2015show}
and machine translation
\citep{bahdanau2014neural}
(among others), where a small component of the input is `attended on' when generating part of the output.



%\done{Promote deep learning methods to a section, put in text chapter. No}
\subsubsection{Neural Attentive Recommendation}
Neural Attentive Recommendation (NARM) \citep{li2017neural} essentially seeks to extend and combine ideas from recurrent recommendation with the use of an attention mechanism. Their suggested model has two main components: a \emph{global} encoder, 
%https://arxiv.org/pdf/1711.04725.pdf
which models sequential data from sessions in much the same way as a traditional RNN-based model (as described above); here the final hidden state of the RNN is responsible for (generating features capable of) predicting the user's next action. A second component, the \emph{local} encoder, uses an attention mechanism over \emph{all} of the hidden states in the sequence to build a feature. %Conceptually, the global encoder's representation describes the semantics of the entire action history, while the local encoder
Roughly speaking, the local encoder's representation captures those specific actions in the user's recent history that best capture the user's intent.
%\done{Wrote long after previous paragraph, check for consistency. Looks okay}
%\done{Figure from narm paper is nice. Probably enough figures}

\subsubsection{Self-Attentive Sequential Recommendation}
%SASRec \cite{kang2018self}
%
The approaches above essentially treat attention mechanisms as an \emph{additional} component to extend existing recurrent models (i.e.,~NARM uses an attention mechanism to learn features from the latent representations from an RNN). 
A recent trend in natural language processing has been to rely more heavily on attention modules
to capture complex structures in sentences,
%and to retrieve
%relevant words (in the source language) for generating the next
%word (in the target language)
again with the attention mechanism retrieving relevant words in a source sentence to generate words in a target sentence (e.g.~for machine translation). 
Recently, attention-based sequence-to-sequence methods, especially those based on the \emph{Transfomer} model \citep{vaswani2017attention}, have emerged as the state-of-the-art for various general-purpose language modeling tasks.
%achieved state-of-the-art performance and
%This reliance on attention, self-attention 
The overall goal of such models is much the same as we saw when developing recurrent networks in \Cref{sec:rnn}: to estimate an output sequence on the basis of an input sequence (or in the case of modeling interaction sequences, estimating the next item in the sequence based on the previous items). Whereas the recurrent networks we developed in \Cref{sec:rnn} did so by successively passing and modifying a latent state across across network cells, the Transformer
%\done{style. Caps}
architecture 
%eschews this, relying 
instead relies
purely on attention modules, 
%whereby 
so that
the estimation of the next item in a sequence is a function of all previous items,\footnote{Up to some fixed maximum length.} with attention determining which of the previous items is \emph{relevant} to the next prediction.\footnote{This description of the Transformer architecture is admittedly very rough, though a full description is quite involved; see the original paper \protect\citep{vaswani2017attention} or one of many excellent tutorials that attempt to distill the core idea.}

%\done{Make figure of different attention-based models. Maybe}

\cite{kang2018self} sought to apply the principle of self-attention to sequential recommendation problems; the method is a fairly straightforward adaptation of the Transformer architecture, whereby the next item in a sequence of user interactions is predicted on the basis of the previous items, with an attention module being responsible for determining which of the previous interactions are relevant to predicting the next one. 

Other than the performance benefits of Transformer-based models (compared to recurrent networks and Markov chain models), \cite{kang2018self} attempt to visualize the attention weights of their model in order to understand which previous interactions the model `attends on' when 
%making each prediction. 
predicting the next item in a sequence.
Not surprisingly they find that the most relevant item 
%when making the next prediction 
tends to be the previous item in the sequence, though interestingly, significant weight is given to more distant interactions. On some datasets these weights decay rapidly, while on others (such as \emph{MovieLens}), relevance weights are more widely distributed across several historical interactions; collectively this suggests that while recommendations are subject to sequential context, this `context' requires more than a single previous observation to capture.

%\paragraph{Attention Mechanisms}

%
\subsubsection{BERT4Rec}
%Following the trend in 
%\cite{kang2018self} of applying state-of-the-art language models to build sequential recommenders, 
%BERT4Rec \cite{sun2019bert4rec} adapts BERT \cite{devlin2018bert}, which (like the model above) 
While \cite{kang2018self} 
%is based 
base their model
on a `standard' Transformer implementation, various extensions have been made to this architecture. For example, BERT4Rec \citep{sun2019bert4rec} adapts BERT \citep{devlin2018bert}, another Transformer-based model architecture. Without going into detail, this model is largely an architectural modification to the approach above; again their superior performance in language modeling tasks appears to translate well to sequential recommendation problems.
Overall this (and other related approaches) further highlight the general trend of applying state-of-the-art language models to build sequential recommenders.
%\done{not great. It's fine}

\subsubsection{Attentional Factorization Machines}
%\done{Reference this in a footnote or something from deep learning section in recsys chapter. no}
We also note that attention mechanisms have been applied in the context of traditional recommender systems.
%, such as factorization machines (which are not necessarily sequential).
%\done{style. Fine}
Attentional Factorization Machines
\citep{xiao2017attentional} apply attention mechanisms to factorization machines, as introduced in \Cref{chap:contentStructure} (recall that such models are not necessarily sequential, though can encode sequential features). The application of attention mechanisms in this context is fairly straightforward. Recall from \Cref{chap:contentStructure} that factorization machines aggregate pairwise interactions across (latent representations of) all features, as in \Cref{eq:FMequation}.  \cite{xiao2017attentional} use attention to determine which pairwise interactions are most important in a particular context, which arguably helps the model not to focus on irrelevant interactions. Experiments show improvements over traditional factorization machines, and some of the deep learning-based recommendation approaches studied in \Cref{sec:deepRec}.



\subsection{Summary}
%\done{not great paragraph title. It's fine}
Above we have only given a limited sample of recommendation methods based on recurrent networks and attention mechanisms. Although we have provided little detail about the specifics of each approach, we have 
%hopefully 
highlighted the general trend of borrowing 
%state-of-the-art 
models from natural language and repurposing them for recommendation.
The use of general-purpose language and sequence models for recommendation is increasingly representative of the current state-of-the-art;
we'll further visit this idea as we explore additional language modeling approaches in \Cref{chap:text}.
%\done{Delete sentence if move to text chapter. Didn't move}
%\done{repetitive compared to above paragraph. Fine}

Note also that the above models in general have no explicit user representation: the input to the model is a sequence of items, based on which the next item is predicted. As such they can be thought of as contextual or memory-based recommendation (though they do have \emph{item} representations which are analogous to $\gamma_i$ in traditional models). 
%Extending the analogy to natural language, a sequence of items is essentially a `sentence' made up of discrete tokens. 
In principle, 
%so long as 
if
long enough 
%sufficiently long
sequences are modeled, we can still capture the broad historical interests of the user, without a need for an explicit user term.
%Sequential Recommendation Via Personalized
%Transformer
In many cases the lack of a user term is a virtue of this type of model: when dealing with new users, we can make reasonable predictions based on a few sequential actions, without needing to resort to complex cold-start approaches, or retraining the model (though of course, attempts have been made to explicitly incorporate user terms into such sequential models, see e.g.~\cite{wu2020sse}).





%\done{Anything about session-based models? I think one of the user-free models is session based, so maybe discuss the idea there. They're all "ssesion -based " but maybe mention that somewhere? Organization is fine}

%Note that
We will further revisit neural network based approaches at various points throughout the book. In \Cref{chap:text} we consider neural network approaches to modeling text (including for text generation), following the same types of approaches we developed in \Cref{sec:rnn}. We also consider text representation approaches in \Cref{sec:word2vec}, which can in turn be used to develop \emph{item} representations for use in sequential recommendation (\cref{sec:item2Vec}). Finally in \Cref{chap:visual} we explore convolutional neural network approaches for image representation, recommendation, and generation.


\section{Case Study: Personalized Heart-Rate Modeling}
\label{sec:caseStudyHR}

Other than building personalized sequence models 
%of user interactions 
%as in 
for the purpose of recommendation as in
\Cref{sec:rnnRec},
%personalized sequence 
such
models can also be used to 
%model 
capture dynamics in
various other types of sequence data. In \Cref{chap:text} we'll explore using sequence models for personalized text generation; here we briefly examine how such models can be used to model heart-rate sequences.

\cite{ni2019modeling}
explored using personalized sequence models to estimate users' heart-rate profiles as they exercised using data from \emph{EndoMondo}.
%\done{How do I cite url? Footnote? I don't}
That is, given the GPS route (latitude, longitude, and altitude) that a user intends to run (or walk, cycle, etc.), and a historical record of the user's previous activities, the goal of the model is to forecast the user's heart-rate profile as accurately as possible. 

Modeling the dynamics of heart-rate sequences is particularly challenging for a few reasons, including:
\begin{itemize}
  \item The dynamics are noisy, complex, and are highly dependent on external factors (e.g.~the user's running speed, the altitude, etc.). 
  %A large amount of training data is required to capture these dynamics.
  \item These factors are a combination of both long- and short-term dynamics, e.g.~the user gradually becoming fatigued, versus the user encountering a hill.
  \item There is significant variation among individuals, to the point that a \emph{non-}personalized model would be ineffective for the task.
  \item A large amount of training data is likely required to capture the complex factors involved in heart-rate dynamics. On the other hand, very few observations are likely to be available for each individual.
\end{itemize}

The model used by \cite{ni2019modeling} is fairly similar to 
%the personalized language 
recurrent network
models we studied in %\Cref{sec:personalTextGeneration}, 
\Cref{sec:rnn},
where a recurrent neural network (an LSTM in \cite{ni2019modeling}) is augmented with low-dimensional embeddings capturing user characteristics, and contextual features describing the current activity: these embeddings are essentially analogous to user/item embeddings $\gamma_u$ and $\gamma_i$. The argued benefit of low-dimensional user representations in this context is that one can learn a `global' model to capture complex heart-rate dynamics from a large amount of data, along with a small number of user-specific parameters that allow the model to adapt to specific user characteristics from a limited number of observations. Again this is similar to personalized language models, where a high-dimensional language model is adapted via low-dimensional user and item terms.

In addition to user and contextual features, the recurrent network in \cite{ni2019modeling} takes as input the GPS trace (latitude, longitude, altitude) of the route the user intends to complete. This sequence of variables is passed as an additional input to the model during each network step, so that the goal is to forecast heart-rate profiles as a function of the user, contextual variables (weather, time, etc.)~and the intended route.
%\done{a little repetetive compared to above para. Okay}


\cite{ni2019modeling} show the benefit of such a model, over non-personalized alternatives and various alternative model architectures. Ultimately they 
argue that
a system that can accurately make such forecasts can be used in various ways, e.g.
\begin{itemize}
    \item To recommend routes that will help a user to achieve a desired heart-rate profile.
    \item To recommend alternative routes that are semantically similar (in terms of heart-rate dynamics) to those they normally take, e.g.~if they want to maintain their routine while visiting a new city.
    \item To detect anomalies in the event that a user's heart-rate should significantly exceed the projected rate at any point.
\end{itemize}

Ultimately, we present the approach from \cite{ni2019modeling} to highlight that personalized models of sequences have applications beyond `traditional' recommendation settings that operate on (e.g.)~click or purchase data. 
%Many of the approaches presented throughout this book have potential application to settings involving personalized health
This case study highlights the potential application to settings such as personalized health.

%\done{mention why we're talking about this model}

\section{History of Personalized Temporal Models}
%\done{History sections should have similar title styles. Is okay}

Although the \emph{Netflix Prize} was one of the early drivers for the use of temporal models in recommendation scenarios, the basic idea of 
%\done{italicization. It's okay}
\emph{concept drift}, in which the distribution of labels changes over time, dates back significantly further. For example, the \emph{Dynamic Weighted Majority} algorithm \citep{kolter2007dynamic} considers using an ensemble of classifiers (`experts'); the weighting of these classifiers' decisions changes over time as a function of the empirical performance of the classifiers. \cite{kolter2007dynamic} also give a historical perspective on the problem of concept drift, dating back to early papers on the topic \citep{schlimmer1986incremental}.

While \cite{koren2009collaborative} showed the effectiveness of temporal factors on the Netflix data,
%\done{repetitive writing. Fine}
a few earlier efforts to use temporal dynamics in recommendation scenarios are notable.
%Adaptive web search based on user profile constructed without any effort from users
For example,
\cite{sugiyama2004adaptive} considered the problem of \emph{personalized search}, though their solution to this problem is actually a form of recommender system. Theirs is an interesting case study in the use of simple similarity-based methods (like those of \Cref{eq:pearsonSim} and \Cref{eq:simPredict3}) for a setting (personalized search) that doesn't immediately appear to be a recommender system. In terms of temporal dynamics, the main idea is to distinguish between `persistent preferences' and `ephemeral prefernces' when building user profiles (from interaction histories), though this is mostly achieved by considering interactions within differently sized recency windows.

Following the success of modeling temporal dynamics on Netflix, there has been a proliferation of models that attempt to capture temporal dynamics in a variety of settings. As we described in \Cref{sec:otherTemporal}, temporal dynamics may occur for a variety of reasons, ranging from long- and short-term dynamics \citep{xiang2010temporal}, community effects
community \citep{godes2012sequential}, or user evolution
%
\citep{mcauley2013amateurs}.

%moved up
%Time weight collaborative filtering
%\cite{ding2005time}

%Techniques based on recurrent networks, self-attention, and Transformer-like architectures have dominated recent research on temporally-aware recommendation, and arguably represent the current state-of-the-art. We revisit this kind of approach 
%\done{correct ordering visit/revisit/visited}
%in \Cref{chap:?} when exploring recommendation techniques based on models from natural language processing.
%\done{redundant paragraph. Boring. Likely drop. Dropped}



%https://colah.github.io/posts/2015-08-Understanding-LSTMs/



% \section{Exercises}

\begin{exercises}
 \item 
 Just as the autoregressive and sliding window techniques we introduced in \Cref{sec:introTime} can be used for prediction, they can also be used for \emph{anomaly detection}, that is, to determine which events in a sequence substantially deviate from our expectations. \emph{Using any time-series dataset} (e.g.~the bike rental data from \Cref{sec:introTime} or otherwise), train an autoregressive model and plot the model's residuals (i.e.,~the difference between the label and the prediction, as in \Cref{eq:qq}) over time to find where the autoregressive prediction is least accurate. Creating such visualizations can be a good strategy to design additional model features: do the anomalies you find appear to occur at random, or can you design additional features to account for them?
 %\done{style: auto-regres or autoregre, thoughout. No hyphen}
 %Something with auto-regression (if it's in this chapter). E.g.~compare sliding windows vs auto-regression in terms of the MSE.
 %\done{finish or drop. Not great but okay}
%  \item Our approaches to similarity-based recommendation (write something that uses a similarity-based recommender but considers both user/item and item-to-item similarity)
 
%  \item In Exercise~\ref{ex:developLatentFactorModelBPR}, we developed a model that included user and item latent factors $\gamma_u$ and $\gamma_i$.
%  %\done{This exercise should be a BPR-like model, not a rating prediction model. Requires that there is a BPR exercise in the previous section. Reference that exercise. Dropped}
%  Adapt the model so that instead of modeling the user, it models the previous item ($\gamma_j$)
 %\done{dropped. seems redundant compared to next exercise}
 \item 
 \label{ex:seq1}
 In \Cref{chap:contentStructure} we introduced factorization machines as a technique to incorporate various types of features into recommendation approaches (and discussed libraries to do so in \Cref{sec:fastFM}). When introducing Factorized Personalized Markov Chains in \Cref{sec:FPMC} we argued that this type of model can be represented via a factorization machine.
 %\done{no we didn't! Added}
 Essentially, this can be done by concatenating representations for the item, the user (as in \cref{sec:factorization}), and the previous item. Following this approach, using any interaction dataset, compare models that use (a) only the sequential term $\gamma_i \cdot \gamma_j$; (b) only the preference term $\gamma_u \cdot \gamma_i$; and (c) both (i.e.,~FPMC).
 You could use either a regression (e.g.~rating prediction) or implicit feedback setting, though if extending the code from \Cref{sec:fastFM} it is more straightforward to model the problem as one of rating prediction.
 %\done{Specify whether each problem is rating prediction or implicit feedback}
 \item 
 \label{ex:fism}
 Likewise, FISM (\cref{subsec:fism}) can be implemented via a factorization machine. Here, the user term is replaced by a feature representing items the user has consumed (divided by the number of interactions), as in \Cref{eq:fismEq}. Incorporate such a feature into a factorization machine, i.e.,%
 \begin{equation}
     f_i = \left\lbrace \begin{tabular}{ll} $1/ | I_u |$ & if $i \in I_u$\\
     $0$ & \text{otherwise}
     \end{tabular}
     \right..
 \end{equation}
 When extracting a feature based on the interaction history, be sure to exclude the current interaction. Compare this model to the variants you trained in Exercise~\ref{ex:seq1}.
 %\done{Ugly, replace with an equation to be clearer}
 \item \emph{Herding} effects occur when users' decisions (e.g.~their ratings) are biased by those they've already seen: e.g.~a user may enter a high rating for an item simply because they see that ratings are already high. Investigate two types of models to determine the role of herding effects in recommendation data:
 \begin{itemize}
     \item An \emph{autoregressive}
     %-based 
     model, in which previous item ratings are used to predict the next (with no personalization or item-specific terms).
     \item A recommendation-based approach, e.g.~by incorporating the most recent ratings as features in a factorization machine.\footnote{Consider how to deal with missing features (i.e.,~for observations with insufficient rating history), 
     %as well as 
     and whether
     any derived features (such as the historical average) may be useful.}
\end{itemize}
%\done{footnote spread over two pages}
Use these models to study under what conditions herding plays a role. For example, does it have a larger effect on high- or low-rated items, items with few ratings, etc.
\end{exercises}

\refstepcounter{projectCounter}
\label{proj:tempBusiness}
\section*{Project \theprojectCounter: Temporal and Sequential Dynamics in Business Reviews}
\addcontentsline{toc}{section}{Project \theprojectCounter: Temporal and Sequential Dynamics in Business Reviews}

In this project we'll explore the various notions of temporal dynamics we covered in this chapter, and conduct a comparative study to determine which notions work well in a particular setting. Specifically, we'll consider consumption patterns of business reviews on \emph{Google Local}, which has been used in various studies on temporal dynamics for recommendation (see e.g.~\cite{he2017translation}). Business reviews are useful for the study of temporal dynamics, as activities have a combination of short-term, long-term, and sequential dynamics, though in principle this project could be conducted using any dataset that includes temporal information.

For most components in this project, you can base your implementation on the \emph{factorization machine} framework we covered in \Cref{sec:factorization}  (e.g.~using the \emph{fastFM} library as in \Cref{sec:fastFM}).
%\done{Check formatting and references. Fine}
Others which don't follow this framework, such as models based on metric embeddings, are significantly more challenging to implement, and may require solutions based on gradient descent.

We'll explore this problem via the following steps:

\begin{enumerate}
    \item Start by training a (non-temporal) latent factor model for the task. Note that this problem could be cast either as one of rating prediction (as in \cref{subsec:matrixFactor}) or as visit prediction (as in \cref{sec:implicit}).
    %\done{repeated text from cold-start project. Is fine}
    Use this initial model to set up your learning pipeline and to find good initial values in terms of the number of latent factors, regularizers, etc. Also consider how you might pre-process the dataset, e.g.~is it better to consier all businesses, or to consider only businesses within a specific category (such as restaurants)?
    %\done{shoild I include rhetorical questions? It's fine}
    If your setting requires negative samples (e.g.~you are modeling visit prediction), consider how you will sample timestamps for the negative items.\footnote{For instance, you could duplicate a timestamp from a positive instance from the same user.}
    \item Next, we'll incorporate temporal dynamics into our model via simple features. 
    Consider how temporal dynamics might be at play in each of these settings, e.g.~ratings might vary on certain days of the week, or be more positive for new businesses, etc.
    Process the timestamps associated with each activity 
    to extract the day of the week, the month of the year, and the `absolute' timestamp (which you may want to scale e.g.~for the range $\lbrack 0, 1 \rbrack$ to represent the lifetime of the dataset).
    Consider any other temporal features that might be useful in this dataset, e.g.~following ideas from \Cref{sec:caseStudyNetflixTemporal}.
    \item Alternately, try including features based on sequential dynamics, e.g.~following the techniques from Exercises~\ref{ex:seq1} and \ref{ex:fism}. Again you may use simple features (e.g.~a one-hot encoding of the previous item, as in Exercise~\ref{ex:seq1}), or more complex ones (such as including several recent items, weighting in terms of recency or geographical distance, etc.).
    \item 
    %(Hard)
    %\done{style (HARD). Gone}
    %\done{maybe put some hard tags back. No}
    Try to implement an alternate sequential model, such as one based on recurrent neural networks from \Cref{sec:rnnRec}. Although challenging to implement `from scratch,' several of the cited papers have public code repositories on which you may base your implementation.
\end{enumerate}

Although the main goal of the project is to build familiarity with and compare various temporal modeling techniques, consider also whether the temporal models you develop give you \emph{insight} into the underlying dynamics of the data. For example, under what circumstances are users guided by temporal dynamics? Or what types of businesses are most subject to seasonal trends? 

\part{Emerging Directions in Personalized Machine Learning}
%Not a great name for part...

\chapter{Personalized Models of Text}
\label{chap:text}

Throughout this book we have already worked with a wide variety of datasets involving text. Although we have made little use of this data so far, text can be useful both as a feature to improve the predictive performance of the types of method we've seen so far, but can also be used to explore a variety of new applications.

As a feature to improve prediction, effectively making use of text is not straightforward. Text is noisy, varies in length, has complex syntax, and only a small fraction of words may be important to prediction. As such, we start this chapter with a primer on the types of feature engineering techniques that can be used to extract meaningful information from text (\cref{sec:textBasics}).

Following this, we explore how text can be used to improve the types of personalized models we've developed in previous chapters. In the case of recommender systems, text ought to be helpful, as there is abundant text (e.g.~product reviews) that can help us to `explain' the underlying dimensions of users' preferences and items' properties; however effectively extracting these signals is not straightforward (\cref{sec:personalText}).

% As a form of sequential data, models from natural language processing can also guide us to better ways to model sequences, following the material we developed in \Cref{chap:temporalSequential}. After introducing a selection of models of natural language, we explore techniques that borrow the same ideas for sequential recommendation (\cref{sec:item2Vec}).
%\done{repetitive. Gone}

Having explored methods that use text as a feature to improve prediction, we also explore recent trends in natural language \emph{generation}. 
%Methods that generate text 
The proliferation of language models based on recurrent networks, along with a slew of recent architectures for general-purpose language modeling and generation, have opened up a range of applications ranging from goal-oriented dialog \citep{bordes2016learning}
to story \citep{roemmele2016writing} or poetry \citep{zhang2014chinese} generation.
Naturally, such methods benefit from personalization, to better capture the context, preferences, or characteristics of individual users 
\citep{joshi2017personalization,majumder2020like}. We'll explore such settings in the context of recommendation approaches that generate text to `explain' recommendations to users (\cref{subsec:textExplain}), and systems that generate recommendation via natural language conversations with users (\cref{subsec:convRec}).

Outside of recommendation, we also examine the use of text in other personalized or contextual settings, from simple forms of personalized retrieval (Project~\ref{proj:text}), to complex systems such as Google's \emph{Smart Reply} (\cref{sec:caseStudySmart}).
%\done{style. Fine}

%Make the points that (1) many personalization tasks involve text somehow; (2) models from natural language are often used when modeling sequence data -- item interactions etc. can benefit from similar sequence models.

\section{Basics of Text Modeling: The Bag-of-Words Model}
\label{sec:textBasics}

%Many of the datasets we've already been working with contain natural language data. Several tasks can potentially benefit from from this data when making predictions, and as we'll see later, text can drive a variety of unique applications.
%\done{maybe cover this in intro to chapter, otherwise this paragraph is too isolated anyway}

In the following, we'll explore the challenges involved in developing fixed-length feature vectors that describe text, as we develop the so-called \emph{bag-of-words} representation. We'll start by trying to develop text representations as na\"ively as possible, and in doing so explore the various pitfalls and ambiguities involved.

\subsection{Sentiment Analysis}
\label{sec:sentiment}

To understand why modeling textual data is difficult, let's consider the seemingly simple task of predicting a rating based on the text of a review:
\begin{equation}
    \text{rating} = x^{(\text{review})} \cdot \theta
    \label{eq:sentiment0}
\end{equation}
%\done{Capital X isn't really consistent notation. Changed to lower case}
This is the same as the type of problem we set up in \Cref{chap:basicsReg}, except that our features $X$ are derived from review text. Intuitively it makes sense that reviews should help us predict ratings, as they are specifically intended to explain a user's rating.

\begin{figure}
\caption{What's the point of sentiment analysis?
\label{box:sentiment}
} 
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
%\done{Need citations for this box! No}
Sentiment analysis, viewed superficially, may seem like an odd (or a `toy') task: why would we ever want to predict ratings from reviews, given that in practice we never observe a review \emph{without} a rating? Nevertheless, sentiment analysis is one of the core topics in natural language processing whose importance transcends the immediate task of predicting ratings. Sentiment analysis research is generally focused on:
\begin{itemize}
    \item Understanding the socio-linguistic
    %\done{Right word?}
    dimensions of sentiment, rather than the immediate task of predicting the rating.
    \item Building sentiment models that are \emph{general purpose}, i.e., models that can be trained on one corpus (e.g.~of reviews), but can be used to predict sentiment in settings where ratings aren't available.
    \item As a benchmarking task to test scalability, and the ability of NLP systems to understand detailed nuances in language.
\end{itemize}
\end{minipage}
}
\end{figure}


The task described in \Cref{eq:sentiment0} captures the basic setting of \emph{sentiment analysis}, i.e.,~learning what types of features are associated with `positive' (i.e.,~high ratings) and `negative' language. We discuss the importance of this task a little in \Cref{box:sentiment}. The main challenge is how to appropriately extract meaningful features from text.


The first problem that we must deal with is that the features in \Cref{eq:sentiment0} are \emph{fixed length} (i.e.,~$X$ is a matrix), whereas text data is sequential. 
%is sequence data.
%Later, we'll see how to model sequences more directly in \Cref{sec:RNN},
%\done{will we? No}
Later we'll see how to establish more complex representations of text (\cref{sec:word2vec})
but for now let's see how much progress we can make just by extracting a pre-defined set of features from each review.

\emph{Bag-of-words} models attempt to solve this by composing $X$ out of features that encode the presence or absence of certain words in a model. Thus it ignores key details such as the order in which words appear (see \cref{fig:bowExample}).

\begin{figure}
\begin{center}
% \fbox{
\parbox{0.47\textwidth}{ \small Loved every minute. So sad there isn't another! I thought JK really made Harry an even stronger archetypal hero - almost in a Paul Maud'Dib from Dune kind of way. He's fighting the ultimate evil, he's brave and takes risks, and believes in himself and doesn't give up despite many hardships.}\hspace{0.04\textwidth}\parbox{0.47\textwidth}{ \small risks, Paul and hardships. believes the almost sad ultimate kind up every - an there in brave hero I fighting Dune another! way. himself made really he's despite He's Loved from archetypal minute. and a Maud'Dib isn't even evil, of in many stronger So takes JK thought Harry give and doesn't}
% } 
\end{center}
\caption[Bag-of-Words models.]{Bag-of-Words models. The two reviews above have identical \emph{bag-of-words} representations (the second randomly shuffles the words of the first). The review at right misses details that depend on syntax. 
Consider whether
%Is 
there is still enough information in the review at right to tell whether the overall sentiment is positive. \label{fig:bowExample}}
\end{figure}

%\done{placement of figures could be improved. Fine} 

A key component in the bag-of-words model is the \emph{dictionary} that is used to build our feature vector. Our first attempt to build this dictionary might merely consist of compiling \emph{every} word in a given corpus, e.g.:
%\done{Find better way to include code. This is fine}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
wordCount = defaultdict(int)
for d in data:
    for w in d['review/text'].split():
        wordCount[w] += 1

print(len(wordCount))
\end{lstlisting}
\end{minipage}
%\done{Do I want print statements in these? Seems harmless}

Doing so on just the first 5,000 of our beer reviews
%\done{Use this or something else. Fine}
%\done{clean up writing, colloquial. Fine}
reveals a total of \emph{36,225} unique words. In other words, each review (on average) contains around seven previously unseen words. This number sounds large,
but 
we might think that by the time we have seen 5,000 reviews that we would have `saturated' the vocabulary of English words, and will not see too many more. However, if we repeat the same experiment for (e.g.)~10,000 reviews, we obtain 55,699 unique words---not quite double, but still a substantial increase, revealing that saturating the vocabulary happens slowly, if at all.
%that we do not quickly saturate the vocabulary.

Looking at a few of the actual words in our dictionary, we see words like:
\begin{equation*}
    \text{...the}; \quad\! \text{09:26-T04.}; \quad\! \text{Hopsicle}; \quad\! \text{beery}; \quad\! \text{\#42}; \quad\! \text{\$10.65}; \quad\! \text{(maybe}; \quad\! \text{(etc.)}
\end{equation*}
That is, we see words including proper nouns, unusual spelling, prices, punctuation, etc. From this, we quickly see that we will not quickly run out of unique `words,' given any reasonable number of reviews.

To use a bag-of-words representation, we will need to reduce this dictionary to a manageable size. Some potential steps to do so include:

\paragraph{Removing capitalization and punctuation}
Removing punctuation will reduce our dictionary size substantially, as word variants like `(maybe' (i.e.,~a word following a parenthesis) will be resolved to a common word. Likewise, we might ignore different capitalization patterns simply by converting all documents to lower-case.

\paragraph{Stemming}
Stemming, i.e.,~resolving similar words to their common word stems. Words like `drink', `drinking', and `drinks' would all map to `drink'.\footnote{Or words like `argue', `arguing', and `argus' would map to `argu'---the stem need not be an actual word.} Such techniques are mostly motivated by search and retrieval settings (i.e.,~to make sure results are retrieved even if a query uses a different word inflection than the result), though they could also be used to reduce our dictionary size. See e.g.~\cite{lovins1968development,porter1980algorithm} for examples of stemming algorithms.

\paragraph{Stopwords}
Stopwords are common (English) words that likely carry (relatively) little predictive power compared to their frequency in a document. Standard stopword lists\footnote{See e.g.~\texttt{nltk.corpus.stopwords} in Python.} include words such as `am,' 'is', `the', `them', etc. Removing such words can reduce our dictionary size a little, or otherwise prevent our feature representations from being overwhelmed by common words (though we will see other ways to address this in \Cref{sec:tfidf}).

Decisions like whether to remove punctuation, whether to stem, or whether to remove stopwords, are largely dataset and application dependent. Characters like exclamation points could be predictive of sentiment;\footnote{Often, important punctuation characters are preserved by treating them like separate words, e.g.~the string `great!'~would be replaced by `great !'.} or different word inflections (such as `drinks' or `drinker' in a corpus of beer reviews)
%\done{not the best example. It's fine}
may have different meanings; or stopwords like `i' and `her' could change the meaning of a sentence.

As such, the procedures above essentially amount to feature engineering choices: we should ultimately accept or reject the above procedures based on whether they improve performance for our given application (again, these are the kind of choices we'd make with our validation set as in \Cref{sec:regularization}).

For the moment, let's consider removing punctuation and capitalization, e.g.:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
for d in data:
    r = ''.join([c for c in d['review/text'].lower() if not c in string.punctuation])
    for w in r.split():
        wordCount[w] += 1
\end{lstlisting}
\end{minipage}
%\done{Indentation (probably four spaces always)}

After removing them, we are left with 19,426 unique words. This is a reduction of nearly half compared to what we had before removing them, but is still a fairly large dictionary size.

A more straightforward way to reduce our dictionary to a manageable size is simply to include only the most commonly occurring words:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
counts = [(wordCount[w], w) for w in wordCount]
counts.sort()
counts.reverse()

words = [x[1] for x in counts[:1000]]
\end{lstlisting}
\end{minipage}

Although perhaps not a completely satisfactory solution, this representation at least allows us to build a feature representation. 
%Put later
%In this case the 1,001$^\text{st}$ most popular word in this example appears in around one percent of all reviews. 
A simple bag-of-words-based sentiment analysis model would now consist of predicting:
\begin{equation}
    \mathrm{rating} = \theta_0 + \sum_{w \in \mathcal D} \mathrm{count}(w) \cdot \theta_w.
    \label{eq:sentiment1}
\end{equation}
Here $\mathcal D$ is our \emph{dictionary} (i.e.,~our set of words); $\theta$ is indexed by a word $w$, but in practice we would replace each word by an index (here from 1 to 1,000) for the sake of building a feature matrix.

Fitting such a model on our beer review dataset\footnote{Here with an $\ell_2$ regularizer with coefficient $\lambda=1$, 4,000 data points for training and 1,000 for testing.} yields a training MSE of 0.27 and a testing MSE of 0.51; this attests to both the high accuracy of such models, as well as their ability to overfit.

Examining the coefficients $\theta$, we find the five words associated with the most positive and most negative coeficients are:
\begin{align*}
\theta_\mathrm{exceptional} & = 0.320; & \theta_\mathrm{skunk} & = -0.364;\\
\theta_\mathrm{always} & = 0.256; & \theta_\mathrm{oh} & = -0.312;\\
\theta_\mathrm{keeps} & = 0.234; & \theta_\mathrm{skunky} & = -0.292;\\
\theta_\mathrm{impressed} & = 0.224; & \theta_\mathrm{bland} & = -0.284;\\
\theta_\mathrm{raisins} & = 0.204; & \theta_\mathrm{recommend} & = -0.267.
%  \theta_\mathrm{skunk} & = -0.364;\\
%  \theta_\mathrm{oh} & = -0.312;\\
%  \theta_\mathrm{skunky} & = -0.292;\\
%  \theta_\mathrm{bland} & = -0.284;\\
%  \theta_\mathrm{recommend} & = -0.267,
%(-0.3635808995815085, 'skunk'),
%(-0.31193813774769147, 'oh'),
%(-0.29234181779758006, 'skunky'),
%(-0.2839149980664344, 'bland'),
%(-0.26701385744199396, 'recommend')
\end{align*}
% and the five most positive words are:
% \begin{align*}
% \theta_\mathrm{raisins} & = 0.204;\\
% \theta_\mathrm{impressed} & = 0.224;\\
% \theta_\mathrm{keeps} & = 0.234;\\
% \theta_\mathrm{always} & = 0.256;\\
% \theta_\mathrm{exceptional} & = 0.320
%(0.2043839147952575, 'raisins'),
%(0.22364690773701465, 'impressed'),
%(0.23419283704758836, 'keeps'),
%(0.2564967471468373, 'always'),
%(0.3197211175655953, 'exceptional')
% \end{align*}
%\done{could combine equations in one line}
For example, every instance of the word `skunk' (a negative association with beer) decreases our prediction of the rating by around one third of a star; likewise every instance of the word `exceptional' increases our prediction by around the same amount. A few more observations:
\begin{itemize}
    \item Words like `skunk' and `skunky' presumably convey the same information, indicating some redundancy in our representation; possibly this could be addressed by stemming.
    \item The word `oh' is extremely negative, despite conveying little meaning by itself. Presumably, it occurs in phrases like `oh no' (whereas `no' itself appears in a wide variety of phrases so is not as negative); possibly we could account for such confounds by using N-grams (\cref{sec:ngrams}).
    \item Likewise words like `always' or `keeps' are highly positive despite conveying little sentiment in isolation; the word `raisins' possibly appears in the context of a specific popular item.
    \item The word `recommend' is highly \emph{negative}, in spite of seeming to convey positive sentiment. Presumably, this word frequently appears in negative phrases (`would not recommend,' etc.); we could account for such confounds by better handling of negation.
\end{itemize}
Finally, it is notable that the words `skunk' and `exceptional' are the 962$^\text{nd}$ and 991$^\text{st}$ most popular words in our corpus---that is, they are words we nearly discarded when we selected a 1,000 word dictionary. On the one hand, this might be an argument that our dictionary size was too small, since we nearly missed the most `important' words. On the other hand, it is almost true by definition that the most predictive words will be rare ones: it is, after all, quite exceptional for an item to be described as `exceptional.'

\subsection{N-grams}
\label{sec:ngrams}

Some of the oddities with 
%our 
the
sentiment model 
we've developed so far
%from the previous section 
possibly arose due to the simplifying assumptions made by the bag-of-words model. Critically, a bag-of-words model has no notion of \emph{syntax} in a document, and as such cannot handle even simple concepts such as negation (e.g.~`not bad') or compound expressions that carry different meaning together than when alone (e.g.~`oh no').

\emph{N-gram} models attempt to address some of these issues by considering words that frequently co-occur in sequence. That is, \emph{bigrams} consist of pairs of words that are adjacent in a document; \emph{trigrams} consist of groups of three words that appear consecutively in a document (etc.). %\done{not great writing}

For example, the N-grams associated with the following sentence would be:%
\begin{equation*}
\begin{tabular}{ll}
Sentence: & `Dark red color, light beige foam'\\
Unigrams: & [`Dark', `red', `color,', `light', `beige', `foam']\\
Bigrams: & [`Dark red', `red color,', `color, light', `light beige', $\ldots$]\\
Trigrams: & [`Dark red color,', `red color, light', `color, light beige', $\ldots$]\\
(etc.)
\end{tabular}
\end{equation*}
%\done{doesn't fit}
From the above example we can already see some potential benefits of using an N-gram representation, e.g.~several of the words in the above sentence are adjectives that modify the words `color' and `foam;' without an N-gram representation we might fail to correctly understand those nouns in context.\footnote{Though one could argue the opposite: if an adjective like `beige' rarely occurs in any other context, then a regular bag-of-words representation may be sufficient to capture the relevant information.} Similarly, negated terms (e.g.~`not bad' etc.)~are readily handled by an N-gram representation.

N-grams are straightforward to extract in Python, e.g.:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[firstnumber=last]
sentence = 'Dark red color, light beige foam'
unigrams = sentence.split()
bigrams = list(zip(unigrams[:-1], unigrams[1:]))
trigrams = list(zip(unigrams[:-2], unigrams[1:-1], unigrams[2:]))
\end{lstlisting}
\end{minipage}

Having extracted N-grams, our approach to modeling the data is much the same as that of our bag-of-words model, i.e.,~we extract counts associated with each N-gram, and use those counts as features to predict some outcome.

Note that generally we would not use (e.g.)~bigrams exclusively, but rather would use a representation that included both unigrams and bigrams simultaneously. Much like our previous approach from \Cref{sec:sentiment}, which simply found the most popular words (i.e.,~unigrams), we might 
%simply 
count the popularity of unigrams and bigrams 
%simultaneously.
together.

On the same review corpus as in \Cref{sec:sentiment}, the majority of popular terms are still unigrams---N-grams comprise 452 of the 1,000 most popular terms.

A few of the most popular N-grams include terms like `with a', `in the', `of the', etc. At first glance, these don't look particularly useful, and are mostly combinations of stopwords. Likewise, few of the top N-grams appear to include negation---for example among the top 1,000 terms, only eleven include the word `not', including e.g.~`but not', `not a', `is not', `not much', `not too', etc. Again these seem unlikely to be informative given that they do not modify meaningful terms.

The above example highlights that N-grams are not quite a simple panacea for the issues mentioned above about handling adjectives and negation (etc.). Indeed, one must carefully trade off the fact that N-grams introduce substantial redundancy into our feature vector with the possibility that they introduce some useful compound terms. Note that it is not straightforward to get the `best of both worlds' in this scenario: assuming a dictionary of fixed size, we are potentially discarding informative unigrams (like `exceptional' from \Cref{sec:sentiment}) in favor of less-informative N-grams. Again, these issues are model- and dataset-specific, and likely could be addressed by better accounting for stopwords (or possibly by using a larger dictionary). Mostly this example simply highlights that we must make extra considerations when incorporating N-grams into a model, and that they will not confer benefits unless carefully handled.

Some of these potential advantages and disadvantages of N-gram representations are summarized in \Cref{box:Ngrams}. Many of the disadvantages center around the redundancy of the representation, which can partly be mitigated by carefully selecting \emph{important} features. We will revisit the topic of word importance in \Cref{sec:tfidf}

\begin{figure}
\caption{Arguments for and against N-grams. \label{box:Ngrams}}
% \fbox{
% \begin{minipage}{0.95\linewidth}
\fcolorbox{black}{gray!15}{
\begin{minipage}{0.94\linewidth}
\setlength{\parskip}{4pt}
To summarize our discussion in \Cref{sec:ngrams}, N-grams are not always beneficial in language modeling tasks, partly because one has to trade off the predictive value of N-grams against the redundancy in encoding them. Some positive and negative points are summarized below:
\begin{itemize}
    \item N-grams straightforwardly allow us to handle negation, and various forms of compound expressions, allowing us to handle relationships among words without having to resort to more complex models that explicitly handle syntax.
    \item Our dictionary size quickly multiplies when using N-grams. Assuming that we can handle a fixed dictionary size, in practice this sometimes means some informative unigrams will be replaced by uninformative N-grams.
    \item Stopwords, which made up a small fraction of our unigram dictionary, quickly multiply when building N-gram representations; thus there is additional redundancy in the N-gram representation.
    \item An N-gram representation may add substantial redundancy (or co-linearity) between features, e.g.~as informative unigrams are now duplicated among several N-gram features.
    %\done{I think all the points are in there}
    %\item Colinearity / redundancy (maybe two points
    %\item Not useful words?
\end{itemize}
\end{minipage}
}
\end{figure}

Finally, let's evaluate an N-gram representation on the same task from \Cref{sec:sentiment}. Again we'll build a `bag-of-ngrams' model by taking the 1,000 most popular N-grams (for any value of N). Once extracted our model is again the same as that from \Cref{eq:sentiment1}, the only difference being that our dictionary consists of a combination of N-grams of different lengths.

After fitting the model, performance in fact slightly degrades compared to the model from \Cref{sec:sentiment} (a test MSE of 0.54 compared to 0.51 in \Cref{sec:sentiment}). Examining some of the most predictive N-grams (i.e.,~largest and smallest values of $\theta_w$) we find terms such as
\begin{equation*}
    \theta_{\text{pitch black}} = -0.397; \quad \theta_{\text{ptich}} = 0.354.
\end{equation*}
Upon further inspection, the word `pitch' \emph{always} appears in the expression `pitch black', to the point that the two terms will mostly `cancel out'; again this highlights an issue of redundancy 
%(and co-linearity, as in \Cref{sec:colinear}) 
in our representation.
%\done{rushed prose. Looks fine}

Possibly we can address this simply by further regularizing our model. Increasing the regularization coefficient to $\lambda = 10$ improves the performance somewhat (test MSE of 0.506), and results in more reasonable coefficients, such as:
\begin{equation}
    %\theta_{\text{water}} = -0.232; \quad
    \theta_{\text{wonderful}} = 0.177; \quad
    \theta_{\text{not bad}} = 0.174; \quad
    \theta_{\text{low carbonation}} = 0.137,
\end{equation}
which appear to correctly handle negation (`not bad') and compound words (`low carbonation').

%\subsection{Personalized Sentiment Analysis}

\subsection{Word Relevance and Document Similarity}
\label{sec:tfidf}
%\done{Is this the right section label? Who cares}
%\done{Worth having this section at all? Yes}

%A question that arose in the previous section was how to prevent our feature representations from being weighed down by uninformative or `irrelevant' words. To investigate this idea further, we need to specify what it means for a word to be `relevant' in a document.
%\done{mediocre motivation, since we don't really use tf-idf to build dictionaries. Removed}

%When building a dictionary in \Cref{sec:sentiment} we simply counted the number of times each word appeared across all documents, i.e.,~we took the most popular words. However, this left us with many stopwords (or otherwise irrelevant words), which (probably not surpringly) are the most common words in most documents.

Suppose we wanted to build a system that recommends articles that are similar in content to ones a user had recently interacted with. As in \Cref{sec:neighborhood}, we might do so by defining an appropriate similarity metric between articles, and recommend those articles that are most similar:
\begin{equation}
    f(i) = \argmax_j \mathrm{Sim}(i,j).
\end{equation}
Given that our goal is to define similarity based on article \emph{content} (rather than interaction histories as in \Cref{sec:neighborhood}), we might consider defining similarity based on the feature representations we developed in \Cref{sec:textBasics}. For example, we could compare the cosine similarity of two bag-of-words representations $x_i$ and $x_j$:
\begin{equation}
    \mathrm{Sim}(i,j) = \text{Cosine Similarity}(x_i, x_j).
\end{equation}
%\done{sim / Sim is not consistently capitalized (in various sections). Made capital}
%\done{Jaccard is italicized (in other section), but should it be? No}
%\done{Never refered to BoW vectors as x_i. Maybe call x^Bow_i or something. Fine}
However as we discussed in \Cref{sec:textBasics}, the vectors $x_i$ and $x_j$ will be dominated by the most common words in the corpus (i.e.,~the largest magnitude words will likely be stopwords).

In practice a user would probably not regard two documents as `similar' just because they used words like `the', `of', or `and' in similar proportions. 
%\done{,' not used consisently elsewhere, check. Looks okay}
As such, we presumably want a feature representation $x_i$ that focuses on \emph{relevant} terms.

In \Cref{fig:tfidfexamp} we see an example of a book review (of \emph{Watership Down}) with the most frequently occurring words in boldface. Naturally, we would not say that the topic of this review was `i' or `a', even though those words are the most frequently occurring.

\begin{figure}
\parbox{0.48\textwidth}{ \small \textbf{I} read \textbf{this} after hearing from \textbf{a} few people that \textbf{it} \textbf{was} among their all-time favorites. \textbf{I} \textbf{was} almost put off when  \textbf{I} \ul{saw} \textbf{it} \textbf{was} \textbf{a} \textbf{story} \ul{\textbf{about}} rabbits, originally written as \textbf{a} tale by \textbf{a} \ul{father} \textbf{to} his \ul{children}---but I'm \ul{glad} \textbf{I} wasn't. \textbf{I} found \textbf{the} folk \ul{tales} \ul{\textbf{about}} El-ahrairah \textbf{to} be very impressive. \textbf{The} author clearly had \textbf{a} vivid imagination \textbf{to} create so much of \textbf{the} rabbits culture \textbf{and} \ul{history}. But \textbf{I} think \textbf{this} book \textbf{was} \ul{worth} reading as it's really \textbf{a} \textbf{story} \ul{\textbf{about}} survival, leadership, \textbf{and} \ul{human} \ul{nature}. Oh \textbf{and} Fiver rocks. \textbf{And} BigWig is \textbf{the} man.}\hspace{0.04\textwidth}\parbox{0.48\textwidth}{I was delighted by this book... the only fault is that it was too short! What a fantastic idea; a refuge for the children who have had adventures \& now cannot fit back into the identity assigned to them. How many of us are not comfortable in the families we were born to? I loved the way the different doorways were sorted; one would think that adventures shared would be a bonding moment. Rivalries will be ever present; guess that is human nature. I don't want to describe too much \& ruin the magic[...]
%this author is the heir apparent to Neil Gaiman. Im so glad she has more books to discover !
}
\caption[Term frequency and \emph{tf-idf} comparison.]{Term frequency and \emph{tf-idf} comparison. At left, the top 10 words by term frequency are bolded (i.e.,~the most common words in the review), and top 10 \emph{tf-idf} words (based on a sample of 50,000 reviews) are underlined. A highly-similar review (based on cosine similarity of \emph{tf-idf} vectors is shown at right. \label{fig:tfidfexamp}}
\end{figure}

Rather, we might argue that words like `nature' or `children' are more characteristic of the document, presumably because they do \emph{not} occur in most documents. As such, we might consider words characteristic of a particular document to be those that occur frequently in that document but not in others.

To capture such a definition we should separately consider word frequency in a particular document, and frequency across a corpus at large. To do so we define two terms. First, the \emph{term frequency} of a word $w$ in a document $d$ is simply the number of times that word appears in the document:
\begin{equation}
    \text{Term Frequency}(w,d) = \mathit{tf}(w,d) = |\lbrace t \in d \mid t = w \rbrace|.
\end{equation}
Note that this is essentially the same as the Bag-of-Words representation we developed in \Cref{sec:textBasics} (although the latter includes a fixed dictionary size).

%Next, we define the \emph{document frequency}
Next, the \emph{document frequency} measures how many documents in a corpus contain a particular word. In terms
%of a word across a set of documents in terms 
of a word $w$ and a corpus $\mathcal D$:
\begin{equation}
    \text{Document Frequency}(w,\mathcal D) = \mathit{df}(w,\mathcal D) = |\lbrace d \in D \mid w \in D \rbrace|.
\end{equation}

Now, for a word to be `relevant' in a particular document, we want the term frequency $\mathit{tf}(w,d)$ to be high, and the frequency of the word across the entire corpus $\mathit{df}(w,\mathcal D)$ to be relatively low. The \emph{tf-idf} measure (term frequency-inverse document frequency) is a heuristic which achieves this goal via the following function:
\begin{equation}
\mathchardef\mhyphen="2D
    \mathit{tf\mhyphen idf}(w,d,\mathcal D) = \mathit{tf}(w,d) \times \log_2 \left( \frac{|\mathcal D|}{\mathit{df}(w,\mathcal D)} \right).
    \label{eq:tfidf}
\end{equation}
%\done{How to get a small hyphen in tf-idf in math mode? \mhyphen}
%\done{Should change all of the above (and below) to be (t,d) rather than (w,d). Why? w is fine.}
While the expression above captures our intuition that the term frequency should be high while the document frequency is relatively low, the specific expression may seem somewhat arbitrary (e.g.~the inclusion of the $\log_2$ term). Indeed, this expression is merely a heuristic, as described in the original implementation \citep{jones1972statistical}. Later work has attempted to justify this choice, e.g.~by interpreting $\log_2\frac{|\mathcal D|}{\mathit{df}(w,\mathcal D)}$ as a log-probability of a word appearing in a document \citep{robertson2004understanding}, though these are post-hoc justifications for what was ultimately a heuristic choice.

Likewise, the term frequency is also a heuristic and is often modified for use in specific contexts. For instance, two alternate definitions of the term frequency include:
\begin{align}
\mathit{tf}'(w,d) & = \delta(w \in d) \label{eq:maxDL0}\\
\mathit{tf}''(w,d) & = \frac{\mathit{tf}(w,d)}{\max_{w' \in d} \mathit{tf}(w',d)}.
%\done{change w to t, maybe w' to w. No see above}
\label{eq:maxDL}
\end{align}
Both of the above are essentially normalization schemes, intended to prevent \emph{tf-idf} scores being higher for longer documents.

\subsection{Using TF-IDF for Search and Retrieval}
\label{subsec:tfidfRetrieve}

Although our interest in developing \emph{tf-idf}
%TF-IDF
%\done{style - \emph{tf-idf}}
is mostly to develop an effective, general-purpose feature representation of bag-of-words models of text, we briefly describe the general strategy when using this type of representation for document retrieval.

While \emph{tf-idf} can be used relatively straightforwardly to retrieve \emph{similar} documents, e.g.~by combining \emph{tf-idf} representations with the cosine similarity (Exercise~\ref{item:qTFIDF}). However in a search or retrieval setting, the `query' is not typically a document but rather a few user-specified keywords.

\emph{Okapi BM-25} \citep{robertson2009probabilistic} adapts \emph{tf-idf}-based similarity measures to retrieval settings, essentially by treating terms in a query $q$ and document $d$ differently. While document words are represented using a \emph{tf-idf} representation, all query words are regarded as being equally important. The specific scoring function between a query $q$ and document $d$ is defined as:
\begin{equation}
    \text{score}(d,q) = \sum_{i = 1}^{|q|} \mathit{idf}(q_i) \cdot \left(
    \frac{\mathit{tf}(q_i, d) \cdot (k_1 + 1)}{\mathit{tf}(q_i, d) + k_1 \cdot \left(1 - b + b \cdot \frac{|d|}{\text{avgdl}}\right)}
    %+ \delta
    \right).
    \label{eq:bm25}
\end{equation}
Most terms are similar to those in \Cref{eq:tfidf}; 
$k_1$ and $b$ are tunable parameters (e.g.~$k_1 \in \lbrack 1.2, 2.0 \rbrack$ and $b = 0.75$, as in \cite{schutze2008introduction}). $\text{avgdl}$ normalizes by the average document length (much like the normalization strategies in \Cref{eq:maxDL0,eq:maxDL}). The inverse document frequency score in \Cref{eq:bm25} also uses a custom normalization:
%
% \Cref{eq:maxDL}
%
\begin{equation}
    \text{idf}(q_i) = \log \left( \frac{|D| - \mathit{df}(q_i, \mathcal D) + 0.5}{\mathit{df}(q_i, \mathcal D) + 0.5} + 1 \right)
\end{equation}

Although we avoid an in depth treatment of the topic, the above is merely to note the general difference in strategy between retrieval based on a query versus similar-document retrieval. We refer to \cite{robertson2009probabilistic} or \cite{schutze2008introduction} for further details.

% Cover BM-25?


% \section{Recurrent Models of Text}


\section{Distributed Word and Item
%, and Document 
Representations}
\label{sec:word2vec}
%\done{Don't actually have document representations, add them or change the title. Dropped}

\emph{Word2Vec} is a popular approach to developing \emph{semantic representations} of words \citep{mikolov2013distributed}. Such representations are somewhat analogous to the user and item representations ($\gamma_u$ and $\gamma_i$) we have been studying throughout this book. 
%That is, just as our user and item representations help us to predict which items `go with' which user (based on some latent representation of user preferences and item properties), word representations ought to tell us which words are likely to `go with,' or appear in the 
That is, just as our latent item representations give us a sense of which items are `similar to' which others (likewise for users), we would like to find latent word representations $\gamma_w$ that tell us which words are similar, or are likely to appear in the same context as each other.

These types of `distributed' word representations are potentially useful for a variety of reasons:
\begin{itemize}
    \item Unlike bag-of-words models (\cref{sec:textBasics}), distributed representations offer a natural mechanism to handle synonyms. That is, words $w$ and $v$ that are synonyms of each other ought to have nearby representations $\gamma_w$ and $\gamma_v$, since they will tend to appear in 
    %the same 
    related
    contexts.
    \item Beyond synonyms, distributed representations might allow us better understand \emph{relationships} among words.\footnote{See for example in \protect\cite{mikolov2013distributed}, where the word representation $\gamma_{\mathit{king}} - \gamma_{\mathit{man}} + \gamma_{\mathit{woman}}$ is close to that of $\gamma_{\mathit{queen}}$.}
    %\done{Say something more about this (king vs. queen etc.)
    \item In certain settings, distributed representations allow us to avoid dimensionality issues associated with bag-of-words models. For instance, when developing generative models of text (which we'll touch on briefly in \Cref{sec:personalTextGeneration}), documents are typically represented as sequences of low-dimensional word vectors $\gamma_w$, rather than as vectors of (e.g.)~word IDs via a bag-of-words model.
\end{itemize}

Below we briefly outline \emph{word2vec} as described in \cite{mikolov2013distributed}; in \Cref{sec:item2Vec} we describe how this idea applies to learning item representations $\gamma_i$ for recommendation.
Although the latter and the former are essentially equivalent, the latter may feel more familiar, as it is similar to the way we learned item representations $\gamma_i$ 
%using sequence 
in user-free models
models in \Cref{sec:userFree1}.
%\done{Is above useful? And does it reflect the final structure? Is fine}

Methodologically, \emph{word2vec} seeks to model the probability that a word in a sequence $w_t$ appears near words $w_{t+j}$, i.e.,~$p(w_{t+j} | w_t)$. So, for a sequence of words $w_1 \ldots w_T$, we would like to learn word representations that maximize the (log) probability
\begin{equation}
    \frac{1}{T} \sum_{t=1}^T \sum_{-c \leq j \leq c, j \neq 0} \log p(w_{t+j} | w_t).
    \label{eq:word2vec0}
\end{equation}
%https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf
Here $c$ is the size of a \emph{context window}, which determines how many neighboring words we consider; this is a hyperparameter
%\done{do I define this word somewhere?}
that may be chosen to balance accuracy and training time, though potentially can vary depending on the word $w_t$.
A simple way to define the probability $p(w_{t+j} | w_t)$ is to say that words $w_{t+j}$ are likely to appear near words $w_t$ with similar representations. In  \cite{mikolov2013distributed} this is defined in terms of the inner product between representations:
\begin{equation}
    p(w_o | w_i) = \frac{e^{\gamma'_{w_o} \cdot \gamma_{w_i}}}{\sum_{w \in \mathcal W} e^{\gamma'_w \cdot \gamma_{w_i}}}
    \label{eq:word2vec1}
\end{equation}
%\done{Do I normally use \mathcal W for a dictionary or word set? Is fine}
where $\mathcal W$ is the set of words in the dictionary.
The numerator in the above encodes the compatibility between the `input' and `output' words $w_i$ and $w_o$; the denominator simply normalizes the value so that it corresponds to a probability over class labels.
%(much as we saw in \Cref{sec:multiclassClassification}).
%\done{Do I discuss multiclass anywhere? no}

Note also that we learn \emph{two} representations, $\gamma_w$, and $\gamma'_w$ for each word (referred to as the `input' and `output' representation, respectively). 
Although doing so doubles the number of parameters, this type of representation avoids symmetries, for example a word is not likely to appear near itself. This is similar to the idea we saw when developing item-to-item recommender systems, and
sequential recommender systems in \Cref{sec:markovCaseStudies},
%\done{Do I ever describe such an example? Maybe in clothing?}
%\done{not sure, item2vec may come before or after. After}
again the intuition being that an item is not likely to be co-purchased with itself (or appear nearby in a sequence).

Since the denominator in \Cref{eq:word2vec1} requires normalizing across all words in the dictionary $\mathcal W$, it is not efficient to compute. \cite{mikolov2013distributed} suggest a few schemes to overcome this issue, though the most straightforward is simply to sample a small number of `negative' words, rather than normalizing over the whole dictionary. As such each computation of $p(w_o | w_i)$ is replaced by an approximation:
\begin{equation}
    \log p(w_o | w_i) \simeq \sigma(\gamma'_{w_o} \cdot \gamma_{w_i}) + \sum_{w \in \mathcal N} \log \sigma(-\gamma'_{w} \cdot \gamma_{w_i})
    \label{eq:word2vec2}
\end{equation}
where $\mathcal N$ is a sampled set of negative words. \cite{mikolov2013distributed} propose various schemes for choosing the sample $\mathcal N$, though most critically argue that the sampling probability should be proportional to the overall frequency of each word.

%\subsection{Doc2Vec}
%Probably skip, check e.g. https://medium.com/wisio/a-gentle-introduction-to-doc2vec-db3e8c0cce5e

\subsection{Item2Vec}
\label{sec:item2Vec}

\emph{Item2Vec}
\citep{barkan2016item2vec}
%ITEM2VEC: NEURAL ITEM EMBEDDING FOR COLLABORATIVE FILTERING (is this the actual item2vec paper?)
adapts the basic idea from \emph{word2vec} as a means of learning item representations $\gamma_i$ for recommendation settings. The main difference between item2vec and word2vec is simply that sequences of words in sentences/documents are replaced by ordered sequences of items that each user has consumed. In practice this simply means that the probability in \Cref{eq:word2vec2} is replaced by
\begin{equation}
    \log p(i | j) \simeq \sigma(\gamma'_i \cdot \gamma_j) + \sum_{i' \in \mathcal N} \log \sigma(-\gamma'_{i'} \cdot \gamma_j),
\end{equation}
where $\mathcal N$ is a set of negative items, again sampled proportionally to overall item frequency.

\cite{barkan2016item2vec} discuss the effectiveness of this type of item representation in the setting of item-to-item recommendation on a corpus of song listens from \emph{Microsoft Xbox Music}. 
%It was shown 
They show
that the method naturally identifies latent dimensions that are associated with song genres; and 
%it was argued 
they argue
qualitatively that related items are semantically more meaningful than those produced by alternate item-to-item recommendation techniques.

\subsection{Word2Vec and Item2Vec with Gensim}

Finally, we show how \emph{word2vec} and \emph{item2vec} work in practice, using interaction and review data from beer reviews.
%, much as we did in \Cref{sec_?}.
%
To learn word representations, the input to the model is a list of documents (in this case reviews), each of which is a list of tokens (words). For this example we first remove capitalization and punctuation before tokenizing, as in \Cref{sec:sentiment}.

Here we use the \emph{Gensim} implementation of \emph{word2vec}.\footnote{\url{https://radimrehurek.com/gensim/}}
%\done{Don't know if I want urls floating around. There are a few others}
%\done{Do I have one for that other library? If so reference consistently. Yes though some are papers and others are urls}
The model takes as input our tokenized reviews (in this case, we use a corpus of 50,000), a minimum word frequency, a dimensionality (i.e.,~$|\gamma_i|$), and a window size (i.e.,~$c$ in \Cref{eq:word2vec0}). The final argument specifies which specific version of the model is used, which corresponds to the model presented above:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}
from gensim.models import Word2Vec

model = Word2Vec(reviewTokens, # Tokenized documents
                 min_count=5,  # Minimum frequency before words are discarded
                 size=10,      # Model dimensionality
                 window=3,     # Window size
                 sg=1)         # Skip-gram model

model.wv.similar_by_word('grassy')
\end{lstlisting}
\end{minipage}

In the final line, we retrieve the most similar words for a particular query; in \emph{Gensim} this is based on the cosine similarity (\cref{eq:cosineSim}) between the two word vectors:
\begin{equation}
    \max_{w} \frac{\gamma_w \cdot \gamma_{\mathit{grassy}}}{\| \gamma_w \| \| \gamma_{\mathit{grassy}} \|} = \text{`citrus'},
\end{equation}
followed by `citric', `floral', `flowery', `piney', `herbal', etc.
%, i.e.,~the words that maximize $\gamma'{w_o} \cdot \gamma_{w_i}$

Similarly, we can use the same code to run \emph{item2vec}, where our tokenized reviews are replaced by lists of items (i.e.,~product IDs) that each user has consumed (ordered by time).

After training a model on review histories, we find that the most similar beers to \emph{Molson Canadian Light} are other light beers such as \emph{Miller Light}, \emph{Molson Golden}, \emph{Piels}, \emph{Coors Extra Gold}, \emph{Labatt Canadian Ale}, (etc.) In \Cref{fig:item2vec} we train a two-dimensional \emph{item2vec} model
%\done{italics}
for the sake of visualizing the data, which reveals that beers belonging to different categories tend to occupy different parts of the item space.\footnote{A more effective visualization might be produced by using higher-dimensional embeddings, followed by a distance-preserving visualization technique like \emph{t-SNE} \protect\citep{maaten2008visualizing}, though for here we plot the embedding dimensions directly for simplicity. We explore \emph{t-SNE} a little further in \Cref{chap:visual}.}

%\done{style on t-SNE}

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/item2vec.pdf}
\caption[Item representations under the \emph{item2vec} model.]{Item representations ($\gamma_i$) using a two-dimensional \emph{item2vec} model. Representations for items from three distinct categories are shown. \label{fig:item2vec}}
\end{SCfigure}



\section{Personalized Sentiment and Recommendation}
\label{sec:personalText}

The models for text we have explored so far, although they can be applied to data like reviews, and can be used to recommend related documents, are ultimately not personalized.

Several attempts have been made to combine models of text with models of users and preferences, and in particular with recommendation approaches. For example, just as we saw techniques in \Cref{chap:contentStructure} that improve recommender systems through the use of side-information, \emph{text} can be useful to efficiently understand the dimensions of user's opinions.

Often there is a significant amount of textual data available in addition to interactions that might be leveraged to fit better models. Other than helping understand sentiment, text can help to understand the dimensions of items and preferences, e.g.~the different properties of products and the different aspects that users care about.

Text can also be helpful for model \emph{interpretability}. So far, the recommender systems we've developed are essentially `black boxes,' whose predictions (as in \cref{eq:MFscore}) are defined purely in term of latent factors. Models of text can be used to understand what aspects these latent dimensions correspond to (\cref{sec:reviewAware}), to 
%summarize reviews, 
synthesize reviews (\cref{sec:personalTextGeneration}),
or to explain recommendations (\cref{subsec:textExplain}).
%\done{Add references if I eventually have sections on these things}

However, extracting \emph{meaningful} information from text is challenging. For example, most of the simple text representations we've seen so far (\cref{sec:textBasics}) are high-dimensional and not particularly sparse; simply incorporating such features into general-purpose feature-aware models is possibly not effective, and one must instead design methods to work specifically with text. Below we cover a few representative approaches.

%\done{Anything about personalized sentiment analysis specifically? Could rename section. It's fine}

\subsection{Case Studies: Review-Aware Recommendation}
\label{sec:reviewAware}
%\done{Probably make this section heading above. No}

\emph{Product reviews} are often used as a source of information to improve recommendation performance. Conceptually, reviews ought to tell us much more about preferences and opinions than (e.g.)~a single rating can. This could be especially true for latent dimensions ($\gamma_u$ and $\gamma_i$ for users and items) since product reviews are intended to `explain' 
%opinions 
the underlying dimensions behind users' decisions.

Roughly speaking, there are two schools of thought as to how reviews should be incorporated to improve recommendation performance. One option is to treat review text as a form of \emph{regularization}, essentially to encourage the low-dimensional representations of users or items (via $\gamma_u$ and $\gamma_i$) to be similar to low-dimensional representations extracted from text. Others seek to extract representations from text that can be used to improve feature-based matrix factorization methods. We give examples of both below.

%\paragraph{Hidden Factors as Topics}
\subsubsection{Hidden Factors as Topics}
%\done{probably rearrange so that this is a subsection. No}
%
An early attempt to incorporate text into recommender systems attempted to do so by making use of \emph{topic models} applied to product reviews \citep{mcauley2013hidden}.
%
Topic models \citep{blei2003latent} learn low-dimensional representations of text (essentially finding structure in the types of bag-of-words representations we covered in \Cref{sec:textBasics}). `Topics' then correspond to sets or clusters of words that tend to co-occur together. For example, a topic model trained on movie reviews might discover that groups of words associated with `action,' `comedy,' or `romance,' might tend to co-occur together, and therefore that these are distinct topics among movie reviews (we give examples of actual topics in \Cref{fig:hftTopics}).
%\done{Separate topic on dimensionality reduction in text? No}

\begin{table}
    \centering
    {
    \small
    \begin{tabular}{cccccc}
% \hrule\\
%     theaters & spas & mexican & vietnamese & snacks & italian & medical & donuts & coffee & seafood\\
%     % \midrule
% theater & massage & mexican & pho & cupcakes & pizza & dr & donuts & coffee & sushi\\
% movie & spa & salsa & vietnamese & cupcake & crust & stadium & donut & starbucks & dish\\
% harkins & yoga & tacos & yogurt & hotel & pizzas & dentist & museum & books & restaurant\\
% theaters & classes & chicken & brisket & resort & italian & doctor & target & latte & rolls\\
% theatre & pedicure & burrito & beer & rooms & bianco & insurance & subs & bowling & server\\
% movies & trail & beans & peaks & dog & pizzeria & doctors & sub & lux & shrimp\\
% dance & studio & taco & mojo & dogs & wings & dental & dunkin & library & dishes\\
% popcorn & gym & burger & shoes & frosting & pasta & appointment & frys & espresso & menu\\
% tickets & hike & carne & froyo & bagel & mozzarella & exam & tour & stores & waiter\\
% flight & nails & food & zoo & bagels & pepperoni & prescription & bike & gelato & crab
\toprule
    \textbf{theaters} & \textbf{mexican} & \textbf{italian} & \textbf{medical} & \textbf{coffee} & \textbf{seafood}\\
    \midrule
theater & mexican & pizza & dr & coffee & sushi\\
movie & salsa & crust & stadium & starbucks & dish\\
harkins & tacos & pizzas & dentist & books & restaurant\\
theaters & chicken & italian & doctor & latte & rolls\\
theatre & burrito & bianco & insurance & bowling & server\\
movies & beans & pizzeria & doctors & lux & shrimp\\
dance & taco & wings & dental & library & dishes\\
popcorn & burger & pasta & appointment & espresso & menu\\
tickets & carne & mozzarella & exam & stores & waiter\\
flight & food & pepperoni & prescription & gelato & crab\\
\bottomrule
    \end{tabular}
    }
    %\includegraphics{}
    \caption[Example of topics that explain variance in rating dimensions on \emph{Yelp}.]
    {
    Example of topics that explain variance in rating dimensions on \emph{Yelp} \protect\citep{mcauley2013hidden}.
    }
    \label{fig:hftTopics}
\end{table}
%\done{Should be a figure or a table? Depends if I have enough tables for it to be worth having an index for them. Table is fine}
%\done{Fix table formatting}
%\done{Move figure to where it is referenced}


The basic idea in \cite{mcauley2013hidden} is that the low dimensional structure among \emph{reviews} should be related to the low-dimensional structure in \emph{ratings}---after all, reviews are intended to explain why a user rated a product a certain way. Furthermore, even though a single rating can tell us very little about the underlying dimensions that explain a user's ratings, a single review potentially contains enough information to understand which dimensions are important to a user, or the characteristics of an item.

The method is somewhat reminiscent of the social recommendation approach we covered in \Cref{subsec:soRec} \citep{ma2008sorec}, in which a shared parameter $\gamma_u$ was tasked with simultaneously explaining rating dimensions as well as social connections. The argument we made in \Cref{subsec:soRec} was that in the absence of sufficient rating data, our estimate of $\gamma_u$ will then be estimated from $u$'s friends.

Likewise, \cite{mcauley2013hidden} suggest that a shared parameter $\gamma_u$ could simultaneously explain rating dimensions via a latent factor model, as well as the topics in reviews:
\begin{equation}
    \underbrace{\sum_{(u,i) \in \mathcal T}\vphantom{\sum_{w \in d_{u,i}}} (\alpha + \beta_u + \beta_i + \gamma_u \cdot \gamma_i - r_{u,i})^2}_{\text{rating error}} + \lambda \underbrace{\sum_{(u,i) \in \mathcal T} \sum_{w \in d_{u,i}} \log p(w | \gamma_u, \psi)}_{\text{topic likelihood}},
\end{equation}
%\done{(u,i) \in or u,i \in ? Former}
%\done{equation renders poorly. Looks fine}
where $\psi$ is a set of additional (non-shared) parameters specific to the topic model (much like the approach in \Cref{subsec:soRec} had shared and non-shared social parameters). This idea is depicted in \Cref{fig:hftIdea}.


\begin{figure}
    \centering
\begin{tikzpicture}
 %\draw[dashed] (0,0) rectangle (5,4) node[pos=.5] (A) {Dataset $\lbrace X; y \rbrace$};
% \node [below=of A,yshift=0em] {Estimate $\theta$};
\node [style=circle,draw] (gamma1) {$\gamma_i$};
\node [right=of gamma1, circle, fill, inner sep=0.1em] (pt1) {};
\node [style=circle,draw,right=of pt1] (gamma2) {$\gamma_u$};
\node [right=of gamma2, circle, fill, inner sep=0.1em] (pt2) {};
\node [style=circle,draw,right=of pt2] (psi) {$\psi$};
\node [below=of pt1] (out1) {rating};
\node [below=of pt2] (out2) {word frequency};
\draw[->] (pt1) -- (out1);
\draw[->] (pt2) -- (out2);
\draw (gamma1) -- (pt1);
\draw (pt1) -- (gamma2);
\draw (gamma2) -- (pt2);
\draw (pt2) -- (psi);

\node[style=dashed,draw, inner sep=3mm, yshift=0.2em, fit=(gamma1) (gamma2)] (Fit1) {};
\node[style=dashed,draw, inner sep=3mm, yshift=-0.2em, fit=(gamma2) (psi)] (Fit2) {};

\node[above=of Fit1,yshift=-3em] (label1) {ratings};
\node[above=of Fit2,yshift=-2.6em] (label2) {document topics};

\end{tikzpicture}
    \caption[Shared parameters in personalized text models.]{Similar to the social recommendation models from \protect\Cref{subsec:soRec}, personalized models of text often make use of a \emph{shared} parameter that must simultaneously explain structure in interactions and documents.}
    \label{fig:hftIdea}
    %\done{Reference this somewhere!}
\end{figure}



Critically, the above model assumes an alignment between latent rating dimensions and review topics. In practice, there may be dimensions in reviews (i.e.,~topics) which are not related to rating dimensions (e.g.~if a user discusses the plot in a book review, it may have little connection to their rating); likewise there may be `intangible' latent dimensions that don't correspond to topics expressed in reviews. By assuming a one-to-one relationship between topics and latent dimensions, the model is useful in cold- (or cool-) start settings by forcing the topic model to discover \emph{those topics that are capable of explaining the variation in ratings}. These topics in particular will be the ones that help us to quickly understand the dimensions that explain user ratings from a few interactions.
%\done{wordy. Is fine}

Examples of the types of topics discovered by this model are shown in \Cref{fig:hftTopics}; mostly, the discovered factors correspond to fine-grained product categories, which mostly reflect users' tendency to favor certain types of item over others.


% \paragraph{Other topic-modeling approaches}
\subsubsection{Other topic-modeling approaches}

The above is a simple approach to combine low-dimensional representations of text (via a topic model) with low-dimensional representations of interactions (via a latent factor model).
%\done{Do I have the above sentence already somewhere?}
Several others have adopted a similar approach, typically by modifying how user factors $\gamma_u$ and topic dimensions are related to each other.

\cite{ling2014ratings} and \cite{diao2014jointly} both consider the same setting as above, in which reviews are used to improve the performance of rating prediction. Both note the limitations of assuming a simple one-to-one mapping between review topics and user preferences as in \cite{mcauley2013hidden}, and suggest more flexible ways to align topic and preference dimensions.

\cite{wang2011collaborative} proposed a similar approach to the problem of recommending scientific articles, where document representations are extracted from article text, and user preferences are modeled to predict which articles they will include in their libraries. This differs from the above formulation in that text is associated with items (documents) rather than users, and the setting is essentially an instance of `one-class' recommendation (as in \cref{sec:implicit}), since one generally doesn't have explicit negative feedback about the articles a user didn't read.



% \paragraph{Neural-Network Approaches}
\subsubsection{Neural-network approaches}

Although our discussion above centered around `traditional' models of text (such as topic models), more recent approaches learn representations of text using neural networks. 
%
\cite{zheng2017joint} adopt a CNN based approach (based on \emph{TextCNN} \citep{kim-2014-convolutional}),
in which they
treat user and item reviews as two separate `documents,' based on which user and item representations (essentially $\gamma_u$ and $\gamma_i$) are estimated. Later works
extend this idea using attention mechanisms (which we discussed in \Cref{subsec:attention}), to infer which reviews are more relevant in a particular context
\citep{tay2018multi,chen2018neural}.






% \paragraph{How Useful are Reviews in Practice?}

% \begin{figure}
% \caption{How useful are reviews for recommendation?}
% Although user reviews have been used in several academic studies of text-based recommendation, it is worth thinking about how useful such data would be in practice. After all, in real-world settings the vast majority of interactions would not be associated with reviews. \cite{sachdeva2020useful} studied the settings
% \label{box:usefulReviews}
% %\done{referenced anywhere?}
% %\done{Dropped}
% \end{figure}


%\section{Question Answering Systems}

%\done{drop}
%\done{remove other refs to question answering}

\section{Personalized Text Generation}
\label{sec:personalTextGeneration}

In \Cref{sec:rnn} we presented Recurrent Neural Networks as general-purpose models that can be used to estimate the next value in a sequence, or to generate sequences, based on some context and the sequence of tokens seen so far.

Such models are routinely used to sample (or generate) realistic-looking text. Recurrent networks for text generation (see e.g.~\cite{graves2013generating})
%\done{good reference? see others from jianmo paper. It's fine}
follow essentially the same setup we saw in \Cref{sec:rnn}. 
%where characters (or words) are generated by the model, following h
At each step $t$, the network takes 
an input $x_t$ (either a character, or a word representation),
%for each step t and 
and
updates its hidden states $h_t$
based on the current input $x_t$
and the previous step's hidden state
$h_{t-1}$ (as in \cref{sec:rnn}, multiple network layers can be stacked). 
The sequence of target outputs $y$ is identical to $x$, but shifted by a single token, i.e.,~the model is responsible for generating the \emph{next} token in a sequence based on all the tokens seen so far.
%The model is trained autoregressively, i.e.,~each network outputs a token $y_t$, which is then 
This type of setup is depicted in \Cref{fig:rnnText}.
%\done{Is this same notation from RNN section? Yes}
%\done{Is text distinct. Yes, think so }
%t−1
%The network predicts the input $x_{t+1}$ at the
%next step, given all inputs $x_{\leq t}$.
%The output layer is connected to a softmax laye
%\done{looks okay, based on what Jianmo wrote}
%This is the essentially the same setting as we described when introducing recurrent networks as general-purpose sequence models in \Cref{chap:temporalSequential} (\cref{sec_?}).



\begin{figure}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
% itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]
itemc/.style={item,on chain,join},font= \scriptsize]
%\done{>=LaTeX above controls the arrow style. Is it good?}
%  \begin{scope}[start chain=going right,nodes=itemc,every
%  join/.style={-latex,very thick},local bounding box=chain]
%  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
%  {$A$};
%  \end{scope}
%  \node[left=1em of chain,scale=2] (eq) {$=$};


 \node[item, minimum width=5mm] (x11) {};
 \node[item, right=of x11, xshift=-10pt, minimum width=5mm] (x12) {};
 \node[item, right=of x12, xshift=-10pt, minimum width=5mm] (x13) {};
 \node[item, right=of x13, xshift=-10pt, minimum width=5mm] (x14) {};
 \node[item, right=of x14, xshift=-10pt, minimum width=5mm] (x15) {};
 \node[item, right=of x15, xshift=-10pt, minimum width=5mm] (x16) {};
 \node[item, right=of x16, xshift=-10pt, minimum width=5mm] (x17) {};
 \node[item, right=of x17, xshift=-10pt, minimum width=5mm] (x18) {};
 \node[right=of x18, xshift=-10pt, minimum width=5mm] (x19) {$\ldots$};

 
 \node[item, below=of x11, yshift=8pt,minimum width=5mm,rectangle,fill=gray!10] (x01) {\texttt{\textless{}s\textgreater{}}};
 \node[item, below=of x12, yshift=10pt, minimum width=5mm,fill=gray!10] (x02) {I};

 \node[item, below=of x13, yshift=10pt, minimum width=5mm,fill=gray!10] (x03) {\_};
 \node[item, below=of x14, yshift=10pt, minimum width=5mm,fill=gray!10] (x04) {w};
 \node[item, below=of x15, yshift=10pt, minimum width=5mm,fill=gray!10] (x05) {a};
 \node[item, below=of x16, yshift=10pt, minimum width=5mm,fill=gray!10] (x06) {s};
 \node[item, below=of x17, yshift=10pt, minimum width=5mm,fill=gray!10] (x07) {\_};
 \node[item, below=of x18, yshift=10pt, minimum width=5mm,fill=gray!10] (x08) {d};

 
%  \node[item, above=of x11, minimum width=5mm] (x21) {};
%  \node[left=of x21, xshift=10pt] (x20) {$h^{l}_{t-2}$};
%  \node[item, above=of x12, minimum width=5mm] (x22) {};
%  \node[item, above=of x13, minimum width=5mm] (x23) {};
%  \node[item, above=of x14, minimum width=5mm] (x24) {};
%  \node[right=of x24, xshift=-10pt] (x25) {$h^{l}_{t+2}$};

 \node[item, above=of x11, yshift=-10pt, minimum width=5mm,fill=gray!10] (x31) {I};
 \node[item, above=of x12, yshift=-10pt, minimum width=5mm,fill=gray!10] (x32) {\_};
 \node[item, above=of x13, yshift=-10pt, minimum width=5mm,fill=gray!10] (x33) {w};
 \node[item, above=of x14, yshift=-10pt, minimum width=5mm,fill=gray!10] (x34) {a};
 \node[item, above=of x15, yshift=-10pt, minimum width=5mm,fill=gray!10] (x35) {s};
 \node[item, above=of x16, yshift=-10pt, minimum width=5mm,fill=gray!10] (x36) {\_};
 \node[item, above=of x17, yshift=-10pt, minimum width=5mm,fill=gray!10] (x37) {d};
 \node[item, above=of x18, yshift=-10pt, minimum width=5mm,fill=gray!10] (x38) {e};
 

%  \draw [->] (x11) -- node[right] {$h^{l-1}_{t-1}$} (x21);
%  \draw [->] (x12) -- node[right] {$h^{l-1}_{t}$} (x22);
%  \draw [->] (x13) -- node[right] {$h^{l-1}_{t+1}$} (x23);
%  \draw [->] (x14) -- node[right] {$h^{l-1}_{t+2}$} (x24);
 
 \draw [->] (x11) -- (x12);
 \draw [->] (x12) -- (x13);
 \draw [->] (x13) -- (x14);
 \draw [->] (x14) -- (x15);
 \draw [->] (x15) -- (x16);
 \draw [->] (x16) -- (x17);
 \draw [->] (x17) -- (x18);
 \draw [->] (x18) -- (x19);
 
%  \draw [->] (x21) -- node[above] {$h^{l}_{t-1}$} (x22);
%  \draw [->] (x22) -- node[above] {$h^{l}_{t}$} (x23);
%  \draw [->] (x23) -- node[above] {$h^{l}_{t+1}$} (x24);
%  \draw [->] (x24) -- node[right] {} (x25);
 
%  \draw [->] (x10) -- (x11);
%  \draw [->] (x20) -- (x21);
 
 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);
 \draw [->] (x05) -- (x15);
 \draw [->] (x06) -- (x16);
 \draw [->] (x07) -- (x17);
 \draw [->] (x08) -- (x18);
 
 \draw [->] (x11) -- (x31);
 \draw [->] (x12) -- (x32);
 \draw [->] (x13) -- (x33);
 \draw [->] (x14) -- (x34);
 \draw [->] (x15) -- (x35);
 \draw [->] (x16) -- (x36);
 \draw [->] (x17) -- (x37);
 \draw [->] (x18) -- (x38);
\end{tikzpicture}
%OLD
% \begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
% itemc/.style={item,on chain,join}]
% %  \begin{scope}[start chain=going right,nodes=itemc,every
% %  join/.style={-latex,very thick},local bounding box=chain]
% %  \path node (A0) {$A$} node (A1) {$A$} node (A2) {$A$} node[xshift=2em] (At)
% %  {$A$};
% %  \end{scope}
% %  \node[left=1em of chain,scale=2] (eq) {$=$};
%  \node[item] (AL1) {$A$};
%  \node[item, right=of AL1, xshift=-1em] (AL2) {$A$};
 
% %  \path (AL.west) ++ (-1em,2em) coordinate (aux);
% %  \draw[very thick,-latex,rounded corners] (AL.east) -| ++ (1em,2em) -- (aux) 
% %  |- (AL.west);
% %  \foreach \X in {0,1,2,t} 
% %  {\draw[very thick,-latex] (A\X.north) -- ++ (0,2em)
% %  node[above,item,fill=gray!10] (h\X) {$h_\X$};
% %  \draw[very thick,latex-] (A\X.south) -- ++ (0,-2em)
% %  node[below,item,fill=gray!10] (x\X) {$x_\X$};}
% %  \draw[white,line width=0.8ex] (AL.north) -- ++ (0,1.9em);
%  \draw[thick,-latex] (AL1.north) -- ++ (0,2em) node[above,item,fill=gray!10] {$h_t$};
%  \draw[thick,latex-] (AL1.south) -- ++ (0,-2em) node[below,item,fill=gray!10] {$x_t$};
% %  \path (x2) -- (xt) node[midway,scale=2,font=\bfseries] {\dots};
% \end{tikzpicture}
\caption[Recurrent neural network for text generation.]{Recurrent neural network for text generation. At each step the network is responsible for generating the next token (in this case a character) on the basis of the tokens seen so far, and the network's current hidden state. `\textless{}s\textgreater{}' represents a `start token' to begin generation (and generation would be terminated once the network generates an end token `\textless{}\textbackslash{}s\textgreater{}'). \label{fig:rnnText}}
%\done{figure not referenced?} 
\end{figure}



While the above model will be capable of generating realistic-looking samples (i.e.,~documents that mimic those in the training set), the samples will not be context-dependent, and they will not be \emph{personalized}.

Several papers have sought to adapt RNNs to generate \emph{personalized} text, that is, text that mimics the context, preferences, or writing style of an individual user.
We describe several such approaches below.
%(and also further discuss the value of this type of approach). 
Most are models of product reviews: partly because such data exhibits variation due to the user, item, and interaction between them, but also simply because such data is widely available. 
%We discuss the broader value of this type of technique 
%after presenting several models below.
%below.
%\done{clunky text, could move explanation text to here or give it a subsection heading to reference it. Dropped}

%Several papers have focused on learning language models to generate coherent reviews.
%\done{More introducory text}

%\done{repetitive (above and below para. Fine}

% \paragraph{Why generate reviews?}
\subsubsection{Why generate reviews?}

%\done{Moved to top}
Personalized text generation, especially when applied to datasets of product reviews, may seem an unusual task with no obvious application (beyond generating review spam): existing platforms are unlikely to surface generated reviews (such as the one in \Cref{fig:genConcat}) to users. However this task has more value when considering the broader context of personalized language generation. In practice, as we've seen elsewhere, reviews are often simply a convenient test-bed for training due to the wide availability of data. Other applications where personalized language generation could include dialog systems,
%(\cref{sec_?}), 
assistive language tools, or natural language processing in other datasets with significant personal variability (e.g.~clinical NLP).
%\done{is test-bed a word? is it used elsewhere? Who cares}
%

Even within the context of reviews, a high-fidelity personalized language model can be used for other functions besides generation. For example the model could be `reversed' for personalized retrieval or search, i.e.,~to retrieve items \emph{that a user might describe in a particular way} (e.g.~a query such as `good dress for summer wedding').
%\done{writing not totally clear}
Personalized text generation can also be used within systems that explain or justify predictions, as we see in 
%the following section.
\Cref{subsec:textExplain}.
%\done{make sure it's still the following section. Changed}

% Other applications besides reviews, health, personalized writing
% Justification generation

The models covered in this section are summarized in \Cref{tab:personTextGen}.
%summarizes the models we cover in this section.
%\done{boring. Fine}

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Summary of personalized text generation approaches.]{Summary of personalized text generation approaches. References: \protect\cite{radford2017learning,dong2017learning,li2017neural,ni2017estimating}.}
\label{tab:personTextGen}
%\done{Make sure all tables/figures are referenced}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{|XXX|}
% \textbf{Goal} & \textbf{Reference} & \textbf{Description}\\
% Sentiment analysis & \cite{radford2017learning} & Focuses on the task of generating user reviews, though is not personalized; the goal is mainly to learn disentangled representations and discover sentiment.\\
% Attribute-based generation & \cite{dong2017learning} & An encoder-decoder approach to generate reviews based on contextual attributes (such as the user, item, or rating). \\
% Abstractive tip generation & \cite{li2017neural} & Generates short `tips' (similar to review summaries), also based on an encoder-decoder approach; reviews are used during training to learn an intermediate representation. \\
% Personalized review generation & \cite{ni2017estimating} & Generates reviews using a latent factor approach that takes user and item factors as input. \\ 
% \end{tabularx}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.22\textwidth}X}
\toprule
% \textbf{Goal} & \textbf{Reference} & \textbf{Description}\\
\textbf{Ref.} & \textbf{Goal} & \textbf{Description}\\
\midrule
% \cite{radford2017learning}
R17 &
Sentiment analysis &  Focuses on the task of generating user reviews, though is not personalized; the goal is mainly to learn disentangled representations and discover sentiment.\\
% \cite{dong2017learning}
D17 &
Attribute-based generation &  An encoder-decoder approach to generate reviews based on contextual attributes (such as the user, item, or rating). \\
% \cite{li2017neural}
L17 &
Abstractive tip generation &  Generates short `tips' (similar to review summaries), also based on an encoder-decoder approach; reviews are used during training to learn an intermediate representation. \\
% \cite{ni2017estimating}
N17 &
Personalized review generation & Generates reviews using a latent factor approach that takes user and item factors as inputs. \\ 
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{Could add justification approaches. Nah}







%\done{Organization (heading levels) a bit funny in this section. Fixed}

%\paragraph{RNN-based review generation}
\subsection{RNN-based Review Generation}
\label{subsec:rnnRevGen}
%
\cite{radford2017learning} were among the first 
%papers 
to explore the use of recurrent neural networks (and in particular, LSTM neural networks, as in \Cref{subsec:LSTM}) to generate 
%realistic-looking 
reviews.

Although the model from \cite{radford2017learning} is not personalized, the approach shows the effectiveness of recurrent networks to sample realistic-looking reviews. Below we explore several approaches that attempt to personalize reviews to individual users. `Personalization' in this setting includes understanding an individual user's writing style, the context associated with a particular item, and the \emph{interaction} between the two (which determines an individual user's reaction and their sentiment toward an item).
%; in particular, the model operates at the \emph{character} level (i.e.,~the sequences as in \Cref{fig:LSTMcell} are made up of characters, rather than words or word fragments)

%\paragraph{Conditional review generation}
\subsubsection{Conditional review generation}

%
Given the promise of using recurrent networks to sample realistic-looking reviews from a background distribution, several papers have followed a similar approach to \cite{radford2017learning} to generate reviews that are relevant to a specific context.

Conceptually, generating reviews to match a specific context follows ideas from \emph{encoder-decoder} architectures which have proved useful in (for example) image captioning settings \citep{vinyals2015show}. Here, rather than passing a start token to the generator (as in \cref{fig:rnnText}) one passes an encoding (i.e.,~a low dimensional representation) of the context, such as an embedding representing an image; following this, decoding follows the same approach as in \Cref{fig:rnnText},
%\done{is this the right figure? Yes}
where the model's hidden states ought to retain the essential components of the context necessary to generate text conditionally.
%\done{clunky text. Fine}
%\done{maybe make a conditional generation diagram (encoder decoder) and refer to it above}
This type of encoder-decoder approach is depicted in \Cref{fig:rnnText2} (left).


\begin{figure}
\begin{tikzpicture}[item/.style={circle,draw,thick,align=center},
%itemc/.style={item,on chain,join},font= \scriptsize,>=LaTeX]
itemc/.style={item,on chain,join},font= \scriptsize]


 \node[item, minimum width=5mm] (x11_) {};
 \node[item, right=of x11, xshift=-15pt, minimum width=5mm] (x12_) {};
 \node[item, right=of x12, xshift=-20pt, minimum width=5mm] (x13_) {};
 \node[item, right=of x13, xshift=-25pt, minimum width=5mm] (x14_) {};
 \node[right=of x14, xshift=-30pt, minimum width=5mm] (x15_) {$\ldots$};
 
 \node[item, below=of x11, yshift=8pt,minimum width=5mm,rectangle] (x01_) {$\gamma_u;\gamma_i$};
 \node[item, below=of x12, xshift=-5pt,yshift=10pt, minimum width=5mm,fill=gray!10] (x02_) {I};
 \node[item, below=of x13, xshift=-10pt, yshift=10pt, minimum width=5mm,fill=gray!10] (x03_) {\_};
 \node[item, below=of x14, xshift=-15pt, yshift=10pt, minimum width=5mm,fill=gray!10] (x04_) {w};

 \node[item, above=of x11, yshift=-10pt, minimum width=5mm,fill=gray!10] (x31_) {I};
 \node[item, above=of x12, xshift=-5pt, yshift=-10pt, minimum width=5mm,fill=gray!10] (x32_) {\_};
 \node[item, above=of x13, xshift=-10pt, yshift=-10pt, minimum width=5mm,fill=gray!10] (x33_) {w};
 \node[item, above=of x14, xshift=-15pt, yshift=-10pt, minimum width=5mm,fill=gray!10] (x34_) {a};

 \draw [->] (x11_) -- (x12_);
 \draw [->] (x12_) -- (x13_);
 \draw [->] (x13_) -- (x14_);
 \draw [->] (x14_) -- (x15_);

 \draw [->] (x01_) -- (x11_);
 \draw [->] (x02_) -- (x12_);
 \draw [->] (x03_) -- (x13_);
 \draw [->] (x04_) -- (x14_);

 \draw [->] (x11_) -- (x31_);
 \draw [->] (x12_) -- (x32_);
 \draw [->] (x13_) -- (x33_);
 \draw [->] (x14_) -- (x34_);


 \node[item, minimum width=5mm, right=of x11_, xshift=100pt] (x11) {};
 \node[item, right=of x11, xshift=-15pt, minimum width=5mm] (x12) {};
 \node[item, right=of x12, xshift=-15pt, minimum width=5mm] (x13) {};
 \node[item, right=of x13, xshift=-15pt, minimum width=5mm] (x14) {};
 \node[right=of x14, xshift=-15pt, minimum width=5mm] (x15) {$\ldots$};
 
 \node[item, below=of x11, yshift=8pt,minimum width=5mm,rectangle,fill=gray!10] (x01) {\texttt{\textless{}s\textgreater{}}};
 \node[item, below=of x12, yshift=10pt, minimum width=5mm,fill=gray!10] (x02) {I};
 \node[item, below=of x13, yshift=10pt, minimum width=5mm,fill=gray!10] (x03) {\_};
 \node[item, below=of x14, yshift=10pt, minimum width=5mm,fill=gray!10] (x04) {w};

 \node[below=of x01, yshift=8pt,minimum width=5mm,rectangle] (g01) {$\gamma_u;\gamma_i$};
 \node[below=of x02, yshift=10pt, minimum width=5mm] (g02) {$\gamma_u;\gamma_i$};
 \node[below=of x03, yshift=10pt, minimum width=5mm] (g03) {$\gamma_u;\gamma_i$};
 \node[below=of x04, yshift=10pt, minimum width=5mm] (g04) {$\gamma_u;\gamma_i$};

 \node[item, above=of x11, yshift=-10pt, minimum width=5mm,fill=gray!10] (x31) {I};
 \node[item, above=of x12, yshift=-10pt, minimum width=5mm,fill=gray!10] (x32) {\_};
 \node[item, above=of x13, yshift=-10pt, minimum width=5mm,fill=gray!10] (x33) {w};
 \node[item, above=of x14, yshift=-10pt, minimum width=5mm,fill=gray!10] (x34) {a};

 \draw [->] (x11) -- (x12);
 \draw [->] (x12) -- (x13);
 \draw [->] (x13) -- (x14);
 \draw [->] (x14) -- (x15);

 \draw [->] (x01) -- (x11);
 \draw [->] (x02) -- (x12);
 \draw [->] (x03) -- (x13);
 \draw [->] (x04) -- (x14);

 \draw [->] (g01) to [out=45,in=315] (x11);
 \draw [->] (g02) to [out=45,in=315] (x12);
 \draw [->] (g03) to [out=45,in=315] (x13);
 \draw [->] (g04) to [out=45,in=315] (x14);

 \draw [->] (x11) -- (x31);
 \draw [->] (x12) -- (x32);
 \draw [->] (x13) -- (x33);
 \draw [->] (x14) -- (x34);

\end{tikzpicture}
\caption[Personalized recurrent network architectures.]{Personalized (or contextual) recurrent network architectures. Left: an encoder-decoder architecture; here the start token from \protect\Cref{fig:rnnText} is replaced by an input signal produced by (or jointly trained with) a previous model (in this case encoding user and item information). Right: the generative-concatenative network from \protect\cite{ni2017estimating}; here the contextual information is input during every step to help the model `remember' the context over many steps.
\label{fig:rnnText2}}
%\done{figure not referenced, also put somewhere}
%\done{Put in right place. Seems fine}
\end{figure}




% \paragraph{Personalized review generation}
\subsubsection{Personalized review generation}

%\cite{dong2017learning} (attribute based generation)
%\done{This method belongs in personalized review generation, since it is personalized. move below}
\cite{dong2017learning} follow this type of encoder-decoder approach to accomplish `attribute-based' conditional review generation. The approach follows the setting described above, whereby attributes are encoded and passed to an LSTM text generation model. The attributes used in their model include a user ID, an item ID, and the score associated with the review.\footnote{Though this dependency on having the rating as an input could presumably be overcome by estimating it separately.}
%\done{Say something about results. No, not interesting}

\cite{li2017neural} follow a similar approach to generate short `tips,' i.e.,~short summaries of reviews. The setting also follows an encoder-decoder approach, though is trained on data from \emph{Yelp} and \emph{Amazon} that includes both reviews and summaries (or `tips' on \emph{Yelp}).
%\done{italics}
While summaries are used as the model output, reviews are used during training to learn an effective intermediate representation that explains interactions between users and items.
%\done{maybe move below as a segue to explanation generation. Is okay }
%(abstractive tip generation)

% \paragraph{Personalized Review Generation}
%
The above methods generate reviews based on specific features or attributes, and as such can essentially be thought of as forms of `contextual' personalization.
%as we discussed in \Cref{sec_?}.
\cite{ni2017estimating} designed text generation methods that directly model users (and items) in order to estimate reviews given only the context of a user and item ID.

The basic setup follows a latent factor approach, i.e.,~the `input' to the model is a representation $\gamma_u$ for the user and $\gamma_i$ for the item. These latent user and item representations are trained jointly with the language model (in this case an LSTM, as in \Cref{subsec:LSTM}); in practice these representations are concatenated onto the input tokens 
%$x$ 
as in \Cref{fig:rnnText2} (right).
%\done{check notation}
Conceptually, rather than the latent factors $\gamma_u$ and $\gamma_i$ explaining user preferences and item properties that predict ratings (as in \cref{chap:modelbasedRecsys}), user factors must now account for patterns of variation in user writing styles (e.g.~the structure used in their reviews), and item factors must account for the overall characteristics of items (e.g.~their objective properties) that users are likely to write about. At the same time both the user and item factors must \emph{jointly} explain the user's sentiment toward the item, e.g.~the positive or negative language that they will use.

An example of a review generated via this technique is shown in \Cref{fig:genConcat}. The review
%(in this case of a beer)
%\done{italics, capitals. Capitals}
is surprisingly coherent and seems to capture (a) the user's writing style (e.g.~they tend to write their reviews across separate paragraphs describing each aspect); (b) the item's overall characteristics (e.g.~the category and flavor profile); and (c) the essential features of the user's preference (i.e.,~a lukewarm review). Recall that as with a traditional recommender system although both the user and item have been seen during training, 
%though 
this specific user and item \emph{combination} have not.

%\done{figure style for this and BoW figure. Make consistent}
\begin{figure}
\parbox{0.53\textwidth}{12 oz.~bottle, excited to see a new Victory product around, \textbf{A:}
\emph{Pours a dark brown, much darker than I thought it would be, rich
creamy head, with light lace.} \textbf{S:} Dark cedar/pine nose with some
dark bread/pumpernickel. \textbf{T:} This ale certainly has \emph{a lot of malt,}
bordering on Barleywine. \emph{Molasses, sweet maple} with a clear
\emph{bitter} melon/white grapefruit hop flavour. Not a lot of complexity
in the hops here for me. Booze is noticable. \textbf{M:} Full-bodied,
creamy, resinous, nicely done. \textbf{D:} A good beer, it isn't exactly
what I was expecting. \emph{In the end above average,} though I found
it monotonous at times, hence the 3. \emph{A sipper for sure.}}\hspace{0.04\textwidth}\parbox{0.41\textwidth}{\textbf{A:} \emph{Pours a very dark brown} with a \emph{nice finger of tan
head that produces a small bubble and leaves decent
lacing} on the glass. \textbf{S:} Smells like a nut brown ale.
It has \emph{a slight sweetness} and a bit of a woody note
and a little cocoa. The nose is \emph{rather malty} with some
chocolate and coffee. The taste is strong but not overwhelmingly sweet. The sweetness is overpowering,
but not overwhelming and is a pretty strong \emph{bitter} finish. \textbf{M:} Medium bodied with a slightly thin feel. \textbf{D:}
\emph{A good tasting beer. Not bad.}}
\caption[Real and synthetically generated user reviews.]{A real (left) and a synthetically generated review for the same user and item (right). Bold/italics added for emphasis. \label{fig:genConcat}}
\end{figure}
%\done{add highlights, and maybe select a different review}
%\done{write caption}

Extensions of this work combined latent user and item representations with observed attributes \citep{ni2018personalized}. Doing so can help the model to use language that better captures specific item details (such as technical features of an electronics product). Further applications are proposed that use this type of technique for \emph{assistive} review generation, e.g.~rather than generating reviews from scratch, the system could be used to help users write reviews based on a template or specific points or attributes, while still following their personalized writing style.

%\section{Case Study: Personalized Recipes}
\subsection{Case Study: Personalized Recipe Generation}
\label{sec:recipeCaseStudy}
% \label{subsec:recipes}
%\done{Could move this to same section as review generation section, and not call it a "case study"}

So far, our study of personalized text generation has focused on product reviews. This is largely an opportunistic choice, simply as review corpora are a widely available source of user-generated text. Here we focus on an alternative source of text data with user-level interactions, namely \emph{recipes}.

%%Given the significant variation in people's tastes and dietary needs, \emph{recipes} 
Recipes have recently emerged as an interesting source of textual data, both for personalized retrieval, and more recently, generation %\citep{li2020recipes}.
\citep{majumder2019generating}.
%\cite{li2020recipes}
%
Early systems to facilitate personalized interactions with recipes did so by helping users to search for recipes whose ingredients target specific health conditions \citep{ueta2011recipe}. Later systems approached the same task with explicit rules, and constraints of ingredients to avoid specific dietary restrictions
\citep{inagawa2013support}. Other retrieval-oriented systems have sought to help users find recipes `in the wild,' e.g.~by searching for recipes that correspond to a photo \citep{marin2019recipe1m}.

More recently, \cite{majumder2019generating} considered whether personalized recipes can be synthesized using text generation frameworks. Here, the idea is to generate novel recipes that are consistent with a particular user's preferences (or specifically, their previous interactions). The setup is somewhat similar to that of \Cref{sec:personalTextGeneration}, where we sought to generate personalized text conditioned on a (representation of a) user and an item. In \cite{majumder2019generating}, the training objective consists of a set of recipes a particular user has consumed, based on which the system should output (i.e.,~generate the text of) another recipe the user would consume.\footnote{In practice, \protect\cite{majumder2019generating} input other metadata (such as a recipe title) to the method to overcome the difficulty of generating recipes `from scratch.'} In this way, the method can generate recipes that are consistent (in terms of e.g.~ingredients, cooking techniques, etc.)~with those that the user would normally consume. An example of a recipe generated with this system is included in \Cref{fig:recipeGenerated}.

%\done{fix figure formatting}
\begin{SCfigure}
\parbox{0.62\textwidth}{
\begin{center}
\textbf{Name:} Pomberrytini;\\[2pt]
\textbf{Ingredients:} pomegranate-blueberry juice, cranberry juice, vodka;
\end{center}
%; Calorie: Low
%
\vspace{-3pt}
Combine all ingredients except for the ice in a blender or food processor. Process to make a smooth
paste and then add the remaining vodka and blend until smooth. Pour into a chilled glass and
garnish with a little lemon and fresh mint.
}
\caption[Example of a personalized recipe.]{Example of a personalized recipe, from \protect\cite{majumder2019generating}. \label{fig:recipeGenerated}}
\end{SCfigure}

%

\subsection{Text-Based Explanations and Justifications}
\label{subsec:textExplain}

So far, we have examined (a) textual data as a means of improving the \emph{predictive accuracy} of personalized models (\cref{sec:personalText}); and (b) textual data as the \emph{output} of a predictive model (\cref{sec:personalTextGeneration}). Beyond these applications, text is also appealing as a means of \emph{explaining} model predictions.

Text-based explanation connects to the more broad topics of \emph{interpretability}, \emph{explainability}, and \emph{justification} of machine learning models. In the case of text-based explanations, the goal is to retrieve or generate a short text fragment that explains 
a model prediction.
%Conceptually, 
%\done{explain how this is related to "human" notions of explanation}
Such models bring us a little closer to the conceptual goal of producing `human-like' explanations, i.e.,~mimicking the way one person would justify a decision or recommendation to another.

As with the personalized text generation models above, much of the work in this space is focused on recommendations and review datasets, as review data serves as a convenient testbed to train models for personalized explanation. The more broad topic of (non-personalized) text-based explanation has been considered in other settings, such as text-based classification
\citep{liu2018towards}.
%\done{check. Looks fine}

\paragraph{Extractive vs.~abstractive approaches}
%(2) justifications (extractive, e.g.~Diego's paper)
Justification (and summarization) approaches can broadly be categorized in \emph{extractive} or \emph{abstractive}. `Extractive' approaches use retrieval-like approaches to select text fragments (e.g.~from a training corpus) that are relevant to a given context or query; `abstractive' approaches generate novel text, either by paraphrasing the corpus or via a generative approach (as in \cref{sec:personalTextGeneration}).
%\done{Could drop extractive vs abstractive, not relevant. Is fine}
%Or Maybe make this an explanation box thing and reference somewhere
%}


% \paragraph{Crowd-sourced explanations}
\subsubsection{Crowd-sourced explanations}

%Crowd-Based Personalized Natural Language
%Explanations for Recommendations
Prior to the use of 
%complex 
generative models for text-based explanation, \cite{chang2016crowd}
%https://dl.acm.org/doi/pdf/10.1145/2959100.2959153
sought to use \emph{crowd-sourcing} to generate personalized explanations for movie recommendations. Since crowd workers are unlikely to be available to generate recommendations in real-time, \cite{chang2016crowd} adopt a template-based approach, where workers generate justifications based on users' interest in a particular aspect or tag
(e.g.~why should \emph{Goodfellas} be recommended to a user who likes \emph{drama}?). To help crowd-workers write explanations, workers are shown extracted review segments relevant to a particular tag, and are tasked with abstracting that text into a coherent explanation.\footnote{The complete pipeline from \protect\cite{chang2016crowd} includes a few additional details, such as a step to vote on the best among several crowd-based explanations, among others.}

Regardless of the practicality of crowd-sourcing such explanations, a main goal in \cite{chang2016crowd} is to confirm the overall value of text-based explanations to users (where human-generated explanations can be regarded as something of a gold-standard). They evaluate text-based explanations in terms of \emph{efficiency} (acquainting users with the relevant properties of an item), \emph{effectiveness} (helping users decide whether they want to watch a movie), \emph{trust}, and \emph{satisfaction}.
They find that text-based explanations are preferred over more trivial tag-based explanations in terms of efficiency, trust, and satisfaction (though with negligible change in terms of effectiveness).




% \paragraph{?}
% %\done{paragraph heading. Dropped}
% Let Me Explain: Impact of Personal and Impersonal
% Explanations on Trust in Recommender Systems \citep{kunkel2019let}

%\paragraph{Abstractive Approaches}
%\paragraph{Generating explanations from reviews}
\subsubsection{Generating explanations from reviews}

%(1) Text-based justifications (abstractive)
\cite{ni2019justifying} explore abstractive approaches to justification generation. Overall, the model is similar to the 
%one
%described in \Cref{sec_?}, 
methods we explored for review generation,
in which a model is trained to generate a review given a particular user/item pair.
%\done{style (user/item)}
\cite{ni2019justifying} use a similar training setup, except that the target text used for training is text that has been identified as being suitable to surface as a recommendation justification. Ultimately this text is harvested from reviews, with a main challenge being how to identify suitable justification sentences among review text. \cite{ni2019justifying} argue that training on this type of harvested text yields more effective justifications than models trained on reviews, tips, or retrieval-based techniques.

Examples of justifications generated using different techniques and training setups (from \cite{ni2019justifying}) are shown in \Cref{fig:justification}.

\begin{figure}
{
\setlength{\tabcolsep}{5pt}
\begin{tabularx}{\linewidth}{lll}
\toprule
\textbf{Method} & \textbf{Type} & \textbf{Output}\\
\midrule
Personalized Retrieval & Extractive & \emph{A great burger and fries}\\
Generated `tip' & Abstractive & \emph{This place is awesome}\\
Generated explanation & Abstractive & \emph{breakfast sandwiches are overall very filling}\\
\bottomrule
\end{tabularx}
}
\caption[Examples of generated justifications for a recommendation.]{Examples of generated justifications for a recommendation of \emph{Shake Shack} to a particular user. From \emph{Yelp} data \protect\citep{ni2019justifying}.
\label{fig:justification}
}
%\done{table formatting}
%\done{Could add other examples, show where dataset comes from}
\end{figure}

\subsection{Conversational Recommendation}
\label{subsec:convRec}
%\done{Dunno if this goes here (probably not). Seems fine?}

%Survey on conversational recommendation:
%https://arxiv.org/pdf/2004.00646.pdf
%\done{write about, copy some refs from}


Arguably, our implicit goal when developing systems for text-based explanation or justification above is to mimic the ways humans explain or justify their decisions.
%Beyond our goal above
%\done{only if actually above}
%of generating text-based explanations or justifications of recommendations, 
Following this, perhaps an ideal system 
%various attempts have been made to develop systems 
for interactive recommendations might mimic the paradigm of \emph{conversation}.

\emph{Conversational recommender systems} combine ideas from dialog generation, explainability, and interactive recommendation. The precise paradigm of `conversation' varies widely: early methods facilitate simple iterative feedback from users \citep{thompson2004personalized}, while more recent methods more closely represent free-form conversation \citep{kang2019recommendation}.

Below we survey a few representative approaches; we refer to 
\cite{jannach2020survey} for a more thorough survey.
%done{More paragraph}
The models covered in this section are summarized in \Cref{tab:conversational}.
%summarizes the models we cover in this section.
%\done{boring}


\begin{table}
\caption[Summary of approaches for conversational recommendation.]{Summary of approaches for conversational recommendation. References: \protect\cite{thompson2004personalized,mahmood2009improving,christakopoulou2016towards,li2018towards,kang2019recommendation}.
%T04 \cite{thompson2004personalized}; MR09 \cite{mahmood2009improving}; C16 \cite{christakopoulou2016towards}; L18 \cite{li2018towards}; K19 \cite{kang2019recommendation}.
\label{tab:conversational}
}
% \begin{tabularx}{\linewidth}{XXX}
% Reference & Type & Description\\
% \cite{thompson2004personalized} & Query refinement & Elicits users' preferences and constraints with regard to item attributes.\\
% \cite{mahmood2009improving} & Reinforcement learning & Queries users about recommendations attributes during each round; learns a policy to choose queries to efficiently yield a desirable recommendation. \\
% \cite{christakopoulou2016towards} & Iterative recommendation & Collects feedback about recommended items in order to iteratively learn user preferences; explores various query strategies to elicit preferences quickly. \\
% \cite{li2018towards} & Free-form conversation & Collects conversational data in which a `recommender' suggests movies and a `seeker' provides feedback. Trains a dialog model to mimic the role of the recommender. \\
% \cite{kang2019recommendation} & Free-form conversation and reinforcement learning & Similar to the above, though trained using reinforcement learning so that the `recommender' and `seeker' exchange information to arrive at a target recommendation.
% \end{tabularx}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.22\textwidth}X}
\toprule
\textbf{Ref.} & \textbf{Type} & \textbf{Description}\\
\midrule
%\cite{thompson2004personalized} 
T04
& Query refinement & Elicits users' preferences and constraints with regard to item attributes.\\
%\cite{mahmood2009improving} 
MR09
& Reinforcement learning & Queries users about recommendations attributes during each round; learns a policy to choose queries to efficiently yield a desirable recommendation. \\
%\cite{christakopoulou2016towards} 
C16
& Iterative recommendation & Collects feedback about recommended items in order to iteratively learn user preferences; explores various query strategies to elicit preferences quickly. \\
%\cite{li2018towards} 
L18
& Free-form conversation & Collects conversational data in which a `recommender' suggests movies and a `seeker' provides feedback. Trains a dialog model to mimic the role of the recommender. \\
%\cite{kang2019recommendation} 
K19
& Free-form conversation and reinforcement learning & Similar to the above, though trained using reinforcement learning so that the `recommender' and `seeker' exchange information to arrive at a target recommendation.\\
\bottomrule
\end{tabularx}
%\done{Don't know if RL is ever defined. Removed}
%\done{make sure all tables are tables and figures are figures}
%\done{not referenced}
%\done{table formatting}
\end{table}
%\done{Put table in right place}



%\paragraph{Query refinement}
\subsubsection{Query refinement}

%A personalized system for conversational recommendations
Early approaches for conversational recommendations essentially use conversation as a form of 
iterative query refinement. In
\cite{thompson2004personalized}
users are asked questions that attempt to determine their preferences or constraints toward a fixed set of attributes (e.g.~cuisine type, price range, etc.); as such a user model is simply a weighting over potential attribute values. Retrieval then consists of selecting items whose attributes most closely match user preferences.
%https://arxiv.org/pdf/1107.0029.pdf

Other early approaches to `conversational' recommendation 
%treat conversation
are essentially forms of interactive recommendation, in which a system can query users or gather feedback from users during each round
%Improving recommender
%systems with adaptive conversational strategies
\citep{mahmood2007learning,mahmood2009improving}.
\cite{mahmood2009improving} adopt a reinforcement learning approach to interactive recommendation. During each step, the system may perform a number of actions, including asking users more detail about specific attributes, or asking them for feedback on a panel of recommendations. Reinforcement learning techniques are used to select an optimal policy in terms of what actions the system should perform under a given state.
%Early paper
%\done{pretty useless description of this paper. Is okay}

% Enhancing the diversity of conversational collaborative recommendations: a comparison \cite{kelly2006enhancing}
%Don't know if relevant

%Towards Conversational Recommender Systems 

% \paragraph{Interactive recommendation}
\subsubsection{Interactive recommendation}

\cite{christakopoulou2016towards}
follow a similar strategy to those above, but combines an interactive recommendation framework with a latent factor model, similar to those we studied in \Cref{chap:modelbasedRecsys}.
Here, the goal is to use a conversational strategy to quickly infer users' preferences $\gamma_u$ toward item properties $\gamma_i$.
Interactions consist of
a one-sided form of `conversation' in which the system asks 
simple questions to the user about their preferences;
%questions to the user, which are drawn from a set of templates.
%Templates
a main goal of \cite{christakopoulou2016towards}
is to understand the ideal characteristics of questions that should be asked to users in order to elicit their preferences efficiently.
Questions can be \emph{absolute} (ask a user whether they like or dislike an item), or \emph{pairwise} (ask a user which of two items they prefer). Question selection strategies consist of determining which items should be compared during each step.
Strategies include selecting random items, items with the highest estimated compatibility, or items whose compatibility has the most uncertainty.
After each question, compatibility scores are recomputed based on question responses, resulting in a model that gradually becomes more accurate during successive rounds.
%https://www.kdd.org/kdd2016/papers/files/rfp0063-christakopoulouA.pdf
%Is this the paper I'm thinking of that introduces the dataset? No.
%\done{could say something about results, but not easy to summarize. best technique is thompson sampling. No}

%\done{Table for this section?}

% \paragraph{Free-form conversation}
\subsubsection{Free-form conversation}

More recent approaches try to follow the conversational paradigm more literally. Ideally, conversations should be \emph{free-form}, in which both the system's question and the user's response take the form of free text.
%Toward Deep Conversational Recommendation 
\cite{li2018towards} attempted to formulate conversational recommendation in this form, for the specific task of movie recommendation.
%
A major challenge which they overcome is to build appropriate ground-truth data for this task.
Their approach builds on previous attempts to build dialog datasets, including dialogs specifically focused around movies \citep{dodge2015evaluating}.
%
Dialogs
%\done{Why is dialog a word but dialogs isn't? I think it's fine}
are constructed by crowd workers, who assume roles of a \emph{recommender} or \emph{seeker}; conversations between the recommender and the seeker are tagged in terms of the movies mentioned, as well as explicit feedback (has the seeker seen the movies mentioned and did they like them). Around ten thousand such conversations are collected.

Having collected such data, \cite{li2018towards} then seek to train a dialog generation model that can fulfil the role of the recommender. Their solution combines ideas from dialog generation with a recommender system, so that users' preferences can be estimated and the output controlled to reference specific movies.
%\done{rough. Could add detail. Fine}


%\done{Check all arXiv references in the bib for correct venues}


A potential limitation of \cite{li2018towards} is that it relies on explicit movie mentions and feedback to learn user preferences; as such their method could not straightforwardly recommend a movie based on a simple goal, such as a user requesting `a good sci-fi movie.'
%
\cite{kang2019recommendation} seek to build conversational recommenders following this `goal oriented' view of recommendation.
To collect data, a conversation game is conducted, where both the seeker and recommender (`expert') are given prompt movie sets: the seeker's set represents their ostensible interests, while the expert's set is a collection of movies, one of which matches the seeker's preferences (determined offline based on a similarity metric). The expert's goal is to determine \emph{which} movie in their set matches the user's preferences via repeated conversation turns. \cite{kang2019recommendation} note that while in principle the expert could simply enumerate movies in their set until they reach the `correct' one, 
%though 
in practice this rarely happens, and players tend to engage in free-form conversation, asking about general attributes and qualities.

Like \cite{li2018towards}, \cite{kang2019recommendation} then train dialog agents to play the game. During play, the expert can either engage in a dialog turn or select one of the movies from their set; the goal is for the expert to identify the target movie in as few moves as possible.
%\done{check against paper. Looks right}

%Recommendation as a Communication Game: Self-Supervised Bot-Play for Goal-oriented Dialogue
%\cite{kang2019recommendation}

% \paragraph{Critiquing}
%\done{Probably move up to explanation/justification/interpret section. Dropped}
% A Ranking Optimization Approach to Latent Linear Critiquing in Conversational Recommender System \cite{li2020ranking}
%Don't know if relevant


%\subsection{Personalized Dialog}
%\done{Maybe. No}



\section{Case Study: Google's \emph{Smart Reply}}
\label{sec:caseStudySmart}

Google's \emph{Smart Reply} is a system developed for \emph{GMail} to automatically recommend short responses for e-mail.
%(\cref{fig:smartreply}). 
%\done{shrink figure, convert to black and white. Dropped}
Given an e-mail thread as input, the system is tasked with surfacing (three) likely responses; although the goal is ultimately to maximize engagement with the feature, the system is trained by taking a large corpus of thread/response pairs, and learning to predict (or maximize the likelihood of) users' historical responses to a given e-mail thread.
%\done{somewhat clunky. Fine}

% \begin{SCfigure}
% \fbox{\includegraphics[scale=0.125]{fig/smartreply.png}}
% \caption{Examples of short responses suggested by Google's \emph{Smart Reply}. \label{fig:smartreply}}
% \end{SCfigure}

As a case study in personalized text generation, this system is interesting for several reasons:
\begin{itemize}
    \item As a form of personalized machine learning, Google's solution does not have explicit user parameters;
    %(i.e.,~it is similar to the non-parametric systems described in
    %\done{where? Dropped}).
    rather, `personalization' is done implicitly by learning from the context already present in the e-mail thread.
    \item They describe two successive---and quite different---solutions to this problem in a sequence of papers. The first is based on a sequence-to-sequence language model (i.e.,~a text generation framework); the second is a seemingly more trivial 
    %nearest-neighbor 
    retrieval-based solution.
    \item Other interesting facets of the studies include how they deal with scalability, diversification, appropriateness of suggesting a reply for a given context, etc.
\end{itemize}

The models we discuss are described in \cite{kannan2016smart} and \cite{henderson2017efficient}. 
%We will mainly discuss the latter, more recent paper, as the solution mostly supersedes the former one, though we will make comparisons between both.
The first solution \citep{kannan2016smart} uses an LSTM-based sequence-to-sequence model, similar to those we described in \Cref{subsec:LSTM}. The model reads a sequence of tokens from the original message, which are used to generate a hidden state, which is then used to begin generation of the target response (the model can be used to rank predetermined response candidates as well as for generation). Several issues must be dealt with (in both of the solutions from \cite{kannan2016smart} and \cite{henderson2017efficient}):
\begin{itemize}
    \item The system must be trained in such a way that private/sensitive data is not used during training (and likewise decoding must not leak sensitive data).
    \item 
    %Clustering, diversity, and redundancy
    When selecting among response candidates, the generated responses must be semantically diverse. Even responses with quite different syntax may be redundant in terms of intent; this is achieved using a \emph{semantic clustering} approach.
    \item 
    %Positives and negatives
    Even for messages that tend to receive positive responses, candidates should include a combination of positive and negative responses.
    \item 
    %Triggering
    Finally, the system should only be used in situations where an automatic response is likely to be appropriate.
\end{itemize}

In a follow-up paper, \cite{henderson2017efficient} suggested alternative methods based on multilayer perceptrons. The overall problem setting is similar: to select a small set of responses given an input e-mail. However given that a multilayer perceptron cannot \emph{generate} responses, a large set of response candidates is collected, such that the system is merely responsible for scoring or ranking candidates.

The main thesis in \cite{henderson2017efficient} is simply to argue that given a large enough set of candidate responses, a \emph{retrieval}-based approach has performance on-par with a \emph{generation}-based approach---while being significantly more straightforward, and facilitating faster inference (retrieval). Several details are incorporated in order to achieve the desired result, that mimic several of the strategies we've explored in this book, for instance:
\begin{itemize}
    \item How to effectively represent text (e-mails) using fixed length features. \cite{henderson2017efficient} use a bag-of-words representation based on N-grams (\cref{sec:ngrams}).
    %\done{style. bag-of-words, n-grams. Looks right}
    %\done{style of n-grams not consistent}
    \item How to select negative instances (i.e.,~non-replies); as in \Cref{subsec:BPR} one cannot compare \emph{all} negatives to a given positive instance, and must instead rely on sampling.
    \item How to perform efficient inference (retrieval). After query and response pairs are embedded via a multilayer perceptron, they are scored via an inner product operation. Thus finding the most compatible reply resembles \emph{maximum inner product search}, as in \Cref{sec:retrieval}.
\end{itemize}

Ultimately, this case study reveals (or reinforces) the notion that simple solutions can often be as effective as more complex ones, and that complex, real-world systems can be developed from exactly the kind of `standard' techniques we've studied.
%\done{repetitive, not great. Okay}

%\done{put this somewhere when discussing sequence to sequence models. No} 
%\cite{sutskever2014sequence}






% \cite{shuyang?}
%\done{only cite follow-up paper if it is accepted. Removed}




% \begin{figure}
% \parbox{0.48\textwidth}{
% \textbf{Chicken Dijon}

% \textbf{Ingredients:} Butter, Chicken, Chicken broth, Dijon mustard, Flour,Light cream, Onion, Pepper,Salt, Wine

% 1) Sprinkle the chicken breasts with onion powder, lemon pepper and salt. 2) Saute the chicken in the butter for 20 min, or cooked through and tender. 3) Remove chicken to a platter and keep warm. 4) Measure the pan juices and enough chicken broth to make 1 cup liquid. 5) Return the stock mixture to the pan and add the wine. 6) Stir together the light cream and the flour till smooth. 7) Add to the broth, cook and stir till smooth, thickened and bubbly. 8) Stir in the dijon mustard, stir till smooth. 9) Return the chicken to the pan, or serve the sauce separately.

% }\hspace{0.04\textwidth}\parbox{0.48\textwidth}{
% \textbf{Low-Calorie Chicken Dijon}

% \textbf{Ingredients:} Chicken, Dijon mustard, Flour, Pepper, Salt, Carrot, Garlic, Mushroom, Soy sauce

% 1) Preheat oven to 350 degrees f. 2) Place chicken breasts, breast side up, in a shallow roasting pan. 3) Combine soy sauce, mustard,garlic and salt. 4) Brush over both sides of the chicken breasts. 5) Bake for 40 minutes. 6) Meanwhile, heat a small sauce-pan over medium heat. 7) Stir in carrots, mushrooms and pepper. 8) Cook and stir for 5 minutes. 9) Stir in flour. 10) Pour over chicken breasts, and bake for an additional 10 minutes until chicken is cooked through. 11) Serve chicken with carrots and sauce.
% }
% \caption{Example of a recipe (left) and a modified version (right) targeting a specific dietary goal \cite{?}. \label{fig:recipes}}
% %\done{not referenced. Removed}
% \end{figure}
% %\done{Ugly formatting in figure. Removed}


% \section{Exercises}

%\done{say more, also formatting is ugly. Dropped}

\begin{exercises}
\item \emph{The exercises below may be conducted using any dataset that includes reviews associated with numerical ratings.} Implement a sentiment analysis pipeline based on a bag-of-words model, as in \Cref{sec:textBasics}, \Cref{eq:sentiment1}. You may follow the code provided as a starting point to build a model based on the 1,000 most common unigrams.
%; this exercise may be conducted using any dataset that includes reviews associated with numerical ratings.
%\done{moved above}
\label{ex:text1}
\item Several choices must be made when building feature representations from text. Extending your model from Exercise~\ref{ex:text1}, and using a validation set, experiment with various modeling choices (in addition to the regularization hyperparameter $\lambda$). Possible modeling choices include:
\begin{itemize}
    \item The dictionary size;
    \item Whether to use unigrams, bigrams, or a combination of both.\footnote{That is, all unigrams \emph{and} bigrams can be sorted by popularity; certain common bigrams (such as common negations) will then have higher frequency than some unigrams and will be included in the model.
    }
    \item Whether to remove capitalization, punctuation, or to treat punctuation characters as separate words;
    \item Whether to use \emph{tf-idf} (\cref{eq:tfidf});
    %\done{correct reference?}
    \item How to handle stopwords, stemming, etc.
    %\done{style of tf-idf}
    %\done{style of these lists: ending each entry with a semicolon}
\end{itemize}
%we should be careful to tune the model to avoid overfitting. Split the data into train, validation, and test sets, and use the validation set to find a regularization hyperparameter $\lambda$ as well as a dictionary size that works better than the solution you found above.
\label{ex:text2}
\item Item2vec (\cref{sec:item2Vec}) gave us an alternate means of item-to-item recommendation. Compare item2vec to simple item-to-item heuristics (such as those in \Cref{chap:recsysIntro}), or to model-based item-to-item approaches (such as those we compared in \Cref{tab:userFree1}). Methods can be compared in terms of each model's ability to predict (i.e.,~assign a high ranking to) a held-out item.
\item 
\label{item:qTFIDF}
%Using TF-IDF for item-to-item similarity (could be project)
As discussed in \Cref{subsec:tfidfRetrieve}, \emph{tf-idf} representations of documents
%\done{tf-idf or TF-IDF (throughout)?}
%\done{Is this actually discussed anywhere? should be. Yes}
can be used to retrieve documents that are semantically similar to each other. For this exercise, we'll use all reviews of a single item as a single `document.' First, compute the \emph{tf-idf} representations of all items, and compute the most similar item (in terms of cosine similarity between \emph{tf-idf} vectors) given a particular query (e.g.~the first item in the dataset). Compare this to similarity computed on bag-of-words representations.
\item In \Cref{chap:recsysIntro}, Exercise~\ref{ex:ratingFromSim}, we considered using similarity functions to predict ratings for user/item pairs. Adapt one of those predictors (e.g.~the predictor from \Cref{eq:simPredict3}) to estimate ratings using the item-to-item similarity function you developed in Exercise~\ref{item:qTFIDF}.\footnote{Given that repeatedly computing item-to-item similarities for high-dimensional text features is likely quite computationally intensive, it is most likely feasible to evaluate your method only on a small sample of user/item pairs.}
\end{exercises}

\refstepcounter{projectCounter}
\label{proj:text}
\section*{Project \theprojectCounter: Personalized Document Retrieval}
\addcontentsline{toc}{section}{Project \theprojectCounter: Personalized Document Retrieval}
%\done{Rename project. Don't see the problem}

%In the above exercises, 
%\done{check that the above is true}
%we have focused on building predictive models using features derived from text. 
The models we developed in the above exercises use features derived from text, but most are not \emph{personalized}.
%In fact, most of the models we developed were not \emph{personalized}. 
Here, we'll explore how to personalize these models following the approaches we developed in this chapter. This project could be conducted using any dataset involving ratings and user reviews, as in the above exercises.

%\done{say what dataset we're using!}

As a starting point, we'll build a personalized model of sentiment analysis; your model can extend the one you developed in Exercises~\ref{ex:text1} and \ref{ex:text2}.
Perhaps the simplest form of personalization consists of fitting bias terms for each user, much like the bias terms we included when developing recommender systems in \Cref{chap:modelbasedRecsys}. Such a term can account for the fact that one user may regard (e.g.)~a three-star rating as positive (and therefore use positive language)
%\done{therefor or therefore?}
whereas anther user may regard it as negative. This term can be incorporated by extending a model like that in \Cref{eq:sentiment1}:
\begin{equation}
    \alpha + \beta_u + \sum_{w \in \mathcal D} \text{count}(w; \mathcal D) \cdot \theta_w.
    \label{eq:projBoW1}
\end{equation}
This model could be fit either by (a) treating the user identity as a one-hot vector, and treating the problem as an ordinary linear regression problem, or (b) by gradient descent, much as we fit bias terms when developing recommender systems in \Cref{chap:modelbasedRecsys}.

Having fit this model, investigate the extent to which the addition of the bias term improves performance (e.g.~in terms of the MSE), and the extent to which it alters the weights associated with the most positive and negative words.

Following this, we would like to develop a more complex model that estimates personalized compatibility with (the words in) a particular document. Fitting a model $\theta_w$ per-user would be impractical (we likely wouldn't have enough interactions per user).
%develop a more complex personalized 
Instead, develop a
model that estimates a user's compatibility with a document in terms of latent factors. Rather than associating latent factors with individual documents, we'll associate them with \emph{words} in documents, augmenting our bag-of-words model from \Cref{eq:projBoW1}:
\begin{equation}
    \alpha + \beta_u + \sum_{k = 1}^K \sum_{w \in \mathcal D} \text{count}(w; \mathcal D) \cdot \gamma_{u,k} \theta_{k,w}.
    \label{eq:textProj}
\end{equation}
Here $K$ is essentially a set of \emph{topics}: $\gamma_{u,k}$ measures the extent to which user $u$ is interested in topic $k$; $\theta_{k,w}$ measures the extent to which the word $w$ is relevant to topic $k$; and $\text{count}(w; \mathcal D)$ measures how many times a word appears in a particular document.

Again, the above model could be implemented via gradient descent, though alternately consider how you might fit a similar model using a factorization machine (\cref{sec:factorization}).
\Cref{eq:textProj} essentially captures latent interactions between word embeddings (via a bag-of-words model), and users.\footnote{Roughly speaking, we're replacing the item in the design matrix from \protect\Cref{eq:FMinteractionMatrix} with a bag-of-words representation of a document.}

Compare the performance of your model from \Cref{eq:textProj} to a bias-only (\cref{eq:projBoW1}) and non-personalized (\cref{eq:sentiment1}) model. The model could easily be extended to incorporate N-grams, or additional features from text.
%: essentially, this type of model is analogous to 



\chapter{Personalized Models of Visual Data}
\label{chap:visual}

%\done{not connected to rest of paragraph, maybe move below}
Traditional models of visual data deal with problems like classification, detection, or (more recently) image generation, though the bulk of approaches are not personalized:
discriminative models (classifiers, detectors) are usually concerned with identifying some objective label in an image,
or generative models are concerned with learning a background distribution governing the overall dynamics of a large corpus of data.
On the other hand,
many of our decisions and interactions can be guided by visual factors, and preferences toward visual attributes can be highly subjective.

The situation above is much as we saw when introducing models for text in \Cref{chap:text}. 
%Likewise, just as we saw in \Cref{chap:text}, many problems involving visual data depend on subjective factors or user context.
%\done{not connected. Removed}
Just as we saw in \Cref{sec:personalText} that text can be used to improve both the fidelity and interpretability of recommendations, visual data can likewise be included to improve the accuracy of models in settings where personal preferences are significantly guided by visual signals.

Visual data are critical in domains like fashion, 
where preferences are largely guided by visual factors.
%Furthermore
%. For one, fashion recommendation problems are 
Problems like recommendation in such settings are
highly personalized, and problems like compatibility among items depend on complex factors that are hard to precisely define. 
%Presumably, visual attributes play a key role in users' decisions. 
Furthermore, recommendation in scenarios frequently suffers from \emph{cold-start} (or `cool start') problems, given the long tail of new and rarely consumed items.

We'll begin this chapter by exploring 
personalization in `traditional' settings like image search and retrieval (\cref{sec:imageSearch}). Following this we'll explore
how to incorporate visual data into recommendation approaches (\cref{sec:visualRec}).
Much of our discussion will be centered around domains like fashion recommendation, where visual features naturally play a key role, though we'll also look at other visually-guided scenarios ranging from art to home decor.

Following this we'll explore new recommendation modalities involving visual data. Item-to-item, or set-based recommendation are particularly important in settings involving visual data, again including settings like outfit generation in fashion (\cref{sec:fashionCaseStudies}). 
%Like the text-based models 
%\done{No section refs above?}

Finally we'll explore personalized \emph{generative} models of images (\cref{sec:personalGans}). Just as we saw models that generate personalized text in \Cref{chap:text}, there are a few settings where one might wish to generate images that are personalized to a user's preference or context, such as systems for personalized design.

\section{Personalized Image Search and Retrieval}
\label{sec:imageSearch}
%\done{Put this section before or after recommendation? Before.}

%\done{fix intro to reflect this section coming first}

%Before visiting visual data in the context of recommendation, it is worth briefly considering how visual data is handled in `traditional' settings like image search.
%Having studied the use of 
Before studying the use of
visual data in the context of recommendation, it is worth briefly considering how visual data is handled in `traditional' settings like image search and retrieval, and how those settings can be personalized. We'll explore two representative approaches that have common elements with methods we've seen in previous chapters, namely the use of latent factor representations to describe users and queries, and the use of joint embeddings. These same themes will reappear as we develop more complex personalized models in later sections.

% Paper: Image ranking based on user browsing behavior
% http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.707.502&rep=rep1&type=pdf

% \cite{trevisiol2012image}

%\done{organization and writing. Could have subheadings for both}

\paragraph{Latent factors}
\cite{wu2014learning}
%Paper: Learning to personalize trending image search suggestion
personalize image retrieval by identifying trending searches that are relevant to a particular user. After finding trending queries (based on an approach from \cite{al2012finding}), 
%the approach follows closely from latent factor models such as those we devel
they estimate compatibility between a user and a query using a latent factor approach. The setting is one of implicit feedback (we only observe positive instances, i.e.,~historical queries), and \cite{wu2014learning} adopt an instance reweighting scheme similar to those we saw in \Cref{subsec:instanceReweight}. Here they fit latent factors associated with a user ($\gamma_u$) and a query ($\gamma_q$) as follows:
\begin{equation}
    \sum_{u,q} c_{u,q} (R_{u,q} - \gamma_u \cdot \gamma_q)^2 + \Omega(\gamma).
\end{equation}
Here $R_{u,q}$ is a binary interaction matrix measuring whether the user has ever issued query $q$ during training. The weight matrix $c$ controls our instance reweighting strategy (see \cref{eq:weightedMF}); the basic idea is that higher importances should be associated with trending instances.


%Paper: Click-through-based cross-view learning for image search

\paragraph{Joint embeddings}
%A basic concept in image retrieval settings, as we saw in
In \Cref{chap:text} we saw the use of joint embeddings to capture hidden factors that are shared between interaction and review data (see e.g.~\Cref{fig:hftIdea}).
%\Cref{sec:SameThingButWithText}, 
Similar ideas are 
%commonly 
used in image retrieval settings, in this case 
%is that of learning 
to learn
a
shared embedding 
%space 
between a query $q$ and an image $i$:
\begin{equation}
%Sec 3.2 of that paper
    d(q,i) = \| g(q) - g(i) \|_2^2
    \label{eq:imageRetrieval1}
\end{equation}
For example, in \cite{pan2014click}, $g(q)$ and $g(i)$ were based on simple linear embeddings of (textual) query features $f_q$ and (visual) image features $f_i$:
\begin{equation}
    g(q) = f_q W^{(\text{query})}; \quad g(i) = f_i W^{(\text{image})}.
    \label{eq:queryBased}
\end{equation}
%Note that this setting is very similar to that of 


%Finally, 
$W^{(\text{query})}$ and $W^{(\text{image})}$ are trained so that distances in \Cref{eq:imageRetrieval1} are minimized based on click-through data.
%, again following a learning approach similar to that from \Cref{sec:hengelPaper}. 
That is, distances should be small between query/image pairs associated with a large number of clicks.\footnote{Note that this approach assumes that query and click data is available at training time, presumably from a method \emph{not} based on visual embeddings.}

We'll see a similar setting in
\Cref{sec:hengelPaper}, 
%\done{reference points forward? should move?}
in which a query image is 
%linearly 
projected into a low-dimensional `style space' (\cref{eq:hengelPaper1}) so that neighboring images can be retrieved. 
%In query-based retrieval This type of image retrieval system 
The query-based retrieval approach of \Cref{eq:queryBased}
operates on a similar principle, except that query features are extracted from text (as we studied in \Cref{sec:reviewAware}).


%OLD
% Note that this setting is very similar to that of \Cref{sec:hengelPaper}, 
% in which a query image is linearly projected into `style space' (\cref{eq:hengelPaper1}) so that neighboring images can be retrieved. This type of image retrieval system operates on the same principle, except that the query features are extracted from text (as we studied in \Cref{sec:textQueryFeatures}).

% Finally, $W^{(\text{query})}$ and $W^{(\text{image})}$ are trained so that distances in \Cref{eq:imageRetrieval1} are minimized based on click-through data, again following a learning approach similar to that from \Cref{sec:hengelPaper}. That is, distances should be small between query/image pairs associated with a large number of clicks.\footnote{Note that this approach assumes that query and click data is available at training time, presumably from a method \emph{not} based on visual embeddings.}





% Paper: Prism: Concept-preserving social image search results
% summarization (maybe not...)

% \subsection{Using Visual Data for Explanation}
% %\done{maybe make subsection}

% %\done{dropped}
% Eliciting User Preferences for Personalized Explanations for Video Summaries



\section{Visually-Aware Recommendation and Personalized Ranking}
\label{sec:visualRec}


%Both of the above issues suggest 
%The issues we introduced above suggest the possibility of 
%systems for recommendation and ranking that make use of visual features. 
Much as we saw with text in \Cref{chap:text} (\cref{sec:personalText}), visual data is difficult to incorporate directly into recommender systems, given that feature representations are high-dimensional, and dense.

Personalized recommendation problems involving visual content (e.g.~clothing) have been studied for several years, with
initial attempts ignoring visual data altogether.
%, and long predate solutions that model images explicitly. 
For example, an early system for clothing recommendation \citep{hu2014style} learns a user's `style' in order to recommend clothing, but does so using `likes' rather than any analysis of visual features. Likewise, \emph{YouTube's} early recommendation approaches \citep{davidson2010youtube} are based on heuristic `relatedness scores' based on co-visitation (essentially a form of neighborhood-based approach, as in \Cref{sec:neighborhood}), though some features based on video metadata are included in the model; newer solutions (based on deep learning) adopt more complicated candidate generation and ranking strategies, though again make little if any use of explicit visual signals \citep{covington2016deep}.

%\done{cut up text above, looks okay}

Below we focus on a
%discuss a 
few of the main approaches that explicitly incorporate visual data into recommendation and personalized ranking models.



\subsection{Visual Bayesian Personalized Ranking}
\label{subsec:VBPR}

Initial attempts to incorporate visual signals into ranking models follow the Bayesian Personalized Ranking framework from \Cref{subsec:BPR} to incorporate observed image features $f_i$ associated with each item, such as a product image.
%from a website.
%
That is, we want to define a compatibility function $x_{u,i,j} = x_{u,i} - x_{u,j}$ (as in \cref{eq:bprOptDiff}) that estimates which of two items $i$ and $j$ are more compatible with the user.

Starting with a simple latent factor-based compatibility model, 
%and given our goal of building a system that works in (item) cold-start scenarios, 
we might first consider simply replacing our (latent) item representations $\gamma_i$ with our observed image features, i.e.,
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + \gamma_u \cdot f_i.
    \label{eq:vbprAttempt1}
\end{equation}
In this way, $\gamma_u$ would now determine which features are most compatible with each user (in fact, this is a linear model as in \Cref{chap:basicsReg}).
Although conceptually reasonable, the issue in doing so becomes quickly apparent once we consider that image features are typically very high dimensional. 
For instance, the visual features used in the study below (extracted from ImageNet)
%(as used in \cite{he2015vbpr})
%\done{have mentioned yet?}
are 4,096 dimensional. Incorporating them directly into $x_{u,i}$ as in \Cref{eq:vbprAttempt1} would thus require fitting thousands of parameters \emph{per user}, which is not feasible in datasets that typically consist of only (e.g.)~tens of interactions per user.

\emph{Visual Bayesian Personalized Ranking} \citep{he2015vbpr} attempts to address this by projecting images into a low-dimensional embedding space via a matrix ${E}$. Here ${E}$ is $|f_i| \times K$ matrix (e.g.~$4096 \times K$), 
%\done{Don't think 4,096 renders right in math mode}
which projects the image into a $K$ dimensional space. Following this, the projected image dimensions can be matched to user preference dimensions:
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + \gamma_u \cdot ({E}f_i).
    \label{eq:vbprAttempt2}
\end{equation}
Note that the projected features ${E}f_i$ fulfil much the same role as $\gamma_i$ in a typical latent factor model, except that they are learned based on 
%observed 
features rather than historical interactions (and as such can be used in cool- or cold-start settings). 

Note that ${E}f_i$ is a \emph{learned} embedding, and is fit so as to maximize the probability of observed interactions, as with all other terms in \Cref{eq:vbprAttempt2}. Note also that while ${E}$ is high dimensional (e.g.~around 40,000 parameters if $|f_i|=4096$ and $K=10$), it is a \emph{global} term that is shared among all items; thus for a large enough dataset it accounts for only a small fraction of the model's parameters.

Because the embedding is low-rank, we are assuming that users' preferences toward these visual dimensions can be explained via a small number of factors. While this is similar to the assumption made by a `standard' latent factor model (e.g.~as in \Cref{eq:MFscore}), we are further assuming that these factors can be explained by visual dimensions. However in practice there could be several latent factors \emph{not} explainable by visual features (e.g.~factors due to price, material, brand, etc.) To address this the original paper includes \emph{both} latent item factors $\gamma_i$ as well as visual item factors ${E}f_i$:
\begin{equation}
    x_{u,i} = \alpha + \beta_u + \beta_i + 
    \overbrace{\vphantom{\beta^{(f)}}\gamma_u({E}f_i)}^{\mathclap{\text{\vphantom{lj}visual preference dimensions}}} +
    \underbrace{\gamma'_u \cdot \gamma_i}_{\mathclap{\text{latent preference dimensions}}} + \overbrace{\beta^{(f)} \cdot f_i}^{\mathclap{\text{\vphantom{lj}visual bias}}}.
    \label{eq:vbpr}
\end{equation}
%\done{equation formatting, and vertical spacing of underbrace}
Correspondingly, there are two sets of user terms: $\gamma_u$, which explains preferences toward visual factors, and $\gamma'_u$, which explains preferences toward non-visual factors. Intuitively, the two terms will play different roles depending on how `cold' an item is: for a cold (or `cool') item, visual features will be much more reliable than latent factors; whereas for 
`hot' items (i.e.,~those with many associated interactions)
%\done{Have I already defined these terms? Yes but fine to repeat}
$\gamma_i$ will be able to capture additional non-visual dimensions. \Cref{eq:vbpr} also includes a `visual bias' term $\beta^{(f)}\cdot f_i$ ($\beta^{(f)}$ is an $|f_i|$-dimensional vector) that is able to estimate item biases in cold scenarios.

Several other considerations must be made to implement such an algorithm efficiently. For instance, accessing (high-dimensional) image features at random (e.g.~within a stochastic gradient ascent algorithm), leads to poor caching performance; likewise computing the projection ${E}f_i$ is expensive. In practice these issues are dealt with by pre-computing all projections ${E}f_i$ (which can be performed as a single matrix-matrix product), and updating ${E}$ only periodically during gradient ascent.

Visual Bayesian Personalized Ranking is effective in settings where items have few associated interactions (which \citet{he2015vbpr} note is common in fashion recommendation scenarios). In the original paper the model is demonstrated on a clothing dataset from \emph{Amazon}, as well as a clothing trading dataset (\emph{Tradesy}).
%\done{Check spelling of tradesy, consistent italics. Looks right}
The latter is particularly challenging because traded items are not associated with long transaction histories, meaning that model predictions must largely rely on visual signals.

%\subsection{Case Study: Modeling the Visual Evolution of Fashion Trends}
% \subsection{Modeling the Visual Evolution of Fashion Trends}
\subsubsection{Modeling the visual evolution of fashion trends} 
%\label{subsec:fashionTrends}
%\done{Probably demote this to a paragraph. Is fine}

\cite{he2016ups} extended the above ideas from Visual Bayesian Personalized Ranking to incorporate temporal dynamics. Modeling temporal dynamics in this setting is interesting partly because the patterns of temporal variation in (e.g.)~clothing purchases are different from those that were successful in other settings, such as on \emph{Netflix} (\cref{sec:caseStudyNetflixTemporal}).
%Potentially, models of temporal dynamics in fashion are also 
Such models are
interesting 
%because they might reveal 
as a means of analyzing
historical trends in fashion over time.

The main idea in \cite{he2016ups} is simply to break the training dataset into a sequence of \emph{epochs}, each of which have their own parameters. These epochs are somewhat akin to the `bins' used to model long-term temporal dynamics on Netflix, though a key difference is that the bin sizes are variable and placed at learned intervals (using a dynamic programming procedure); given that the model has a large number of parameters, this helps to ensure that fewer bins are used during time periods with little temporal variation, whereas more (and smaller) bins are used during periods which are more dynamic.



% Mention point about projecting images into low dimensional space only once per iteration to save on computational costs

% \section{Visual Compatibility of Items}
% \label{sec:visualCompat}
\section{Case Studies: Visual and Fashion Compatibility}
\label{sec:fashionCaseStudies}

%Look for refs in WC Pinterest paper

In \Cref{sec:visualRec}
%\done{Is it the previous section?}
we saw how visually-aware recommender systems can be used to match items (or images of items) to users' preferences. 
%In domains like fashion, there is also interest in establishing compatibility between items (or recommending one item on the basis of another), for example to generate outfits, or even wardrobes of mutually compatible items. 
Earlier, in \Cref{sec:neighborhood} we discussed several types of recommendation approaches that considered similarity between items; such measures guide `item-to-item' recommendation approaches (e.g.~`people who bought X also bought Y').
%, as in \Cref{fig:alsoviewed}.
%\done{could make this figure and put these references back. No}
%
Here, we would like to develop similar approaches that establish visual similarity (or compatibility) between items. Rather than basing similarity on interaction histories as we did in \Cref{sec:neighborhood}, here we can base similarity directly on the visual appearance of items.

% \section{Case Studies: Fashion Compatibility}
% \label{sec:fashionCaseStudies}
%\done{Decide whether there should be sections with multiple case-studies like this. Yes}

Many studies on visual compatibility are specifically concerned with \emph{fashion images}. Estimating compatibility in such a domain has obvious applications to specific tasks like outfit generation and recommendation,
%(or recommending one item on the basis of another), for example to generate outfits, 
or even to generate `wardrobes' of mutually compatible items. 
More simply, in settings like fashion, 
%interactions tend to be guided by compatibility relationships.
visual
compatibility with past interactions or purchases is a strong predictor of future interactions.

Some of the specific characteristics that make this problem difficult (and different from other forms of 
%(e.g.)~
item-to-item recommendation), are as follows:
\begin{itemize}
    \item It is challenging to construct datasets that act as `groundtruth' for visual compatibility, i.e.,~pairs of items that are known to `go well' together.
    \item Further to the above, any groundtruth of compatible items is bound to be highly noisy, and highly subjective; successful methods need to account for these challenges, and possibly learn compatibility in a personalized way.
    \item The features that make items visually compatible in settings like fashion could be 
    %incredible 
    subtle, and could be quite different from the information available in co-purchase data, or even in most visual feature descriptors.
    \item Finally, the notion of `compatibility' is semantically quite different from `similarity.' E.g.~clothing items that go together should be similar in some ways but complementary in others.
\end{itemize}

Approaches to these problems mainly differ in their specific solutions to the problems above. We describe a few key 
%papers 
approaches
below.

\subsection{Estimating Compatibility from Co-purchases}
\label{sec:hengelPaper}

Early approaches to estimating visual compatibility built datasets from co-purchases, for example using publicly-available datasets of reviews from \emph{Amazon}.
%\done{how to italicize company names etc. Like this}
%\done{Is amazon data already mentioned above? Only mention it once. Is mentioned lots}

\cite{mcauley2015image} crawled data from Amazon's surfaced recommendations (`people who bought X also bought Y' etc.)~and,
%~as in \Cref{fig:alsoviewed}), and, 
in the case of clothing, treated these as `groundtruth' examples of items that are visually compatible.

Having defined such a compatibility function, the goal is to learn an appropriate distance function, such that frequently co-purchased items tend to be closer together than others. The distance function is then used in a simple binary classification framework (similar to logistic regression) to predict:
\begin{equation}
    p(i \text{\ co-purchased with\ } j) = \sigma(c - d(i, j)).
\end{equation}
In \Cref{chap:temporalSequential}, we considered how to learn distance functions for problems such as next Point-of-Interest recommendation (\cref{sec:PRME}). When doing so,
%\done{Check whether above is true of all cases}
items (and users) were projected into a latent space via parameters $\gamma$. To recommend compatible clothing, we might instead use features extracted directly from (the product images of) $i$ and $j$: first, general-purpose visual features are readily available, and are likely to be informative in fashion compatibility scenarios; second, reliance on features is desirable in cold-start settings, which might be common in settings (like fashion) where item vocabularies are large and changing; third, a model based only on visual features can be more straightforwardly \emph{transferred} to settings where user data is not available.

Given image features associated with the items $f_i$ and $f_j$, \cite{mcauley2015image} discuss several strategies for establishing visual similarity. Trivially, one could directly consider the (squared) distance between $f_i$ and $f_j$ (i.e.,~$\| f_i - f_j \|_2^2$), however general-purpose image features may not focus on attributes that are relevant to fashion.

A second solution is to learn a weighted distance function that discovers which features are relevant and discards those that are not, i.e.,~$\sum_k w_k (f_{i,k} - f_{j,k})^2$. However it is argued that in fashion scenarios `compatibility' cannot be captured by modeling similarity between features---for example, a user generally would not select a shirt because it looks `similar' to a pair of pants. To address this, a similarity function is proposed which projects the images into a low-dimensional `style space':
\begin{equation}
    d(i,j) = \| s_i - s_j \|_2^2; \quad \text{where} \quad s_i = E \times f_i.
    \label{eq:hengelPaper1}
\end{equation}
In the case of \cite{mcauley2015image} $f_i$ is a 4096-dimensional image descriptor, extracted from a model trained on ImageNet \citep{jia2014caffe}; $E$ is then a $4096 \times K$ vector, where $K$ is some small embedding dimension (on the order of $K=10$). Ultimately the embedded vector $s_i = E\times f_i$ is analogous to the latent vectors $\gamma_i$ from previous models (and very closely matches the embedding approach of \Cref{subsec:VBPR}), in the sense that it captures the underlying dimensions that explain variation in co-purchases.

The method is then trained using a dataset $\mathcal C$ of complementary pairs, along with a set of \emph{non-complementary} pairs $\mathcal C^-$ (which in practice are sampled randomly). The model is then trained using a logistic regression-like setup to distinguish complementary and non-complementary pairs:
%\done{Above seemed missing, so added, but check to see if it fits in the organization, and that it's not the same as the text used in Sceptre description. Fine}
\begin{equation}
\underbrace{\sum_{(i,j) \in \mathcal C} \log \sigma(c - d(i, j))}_{\text{complementary pairs}} + \overbrace{\sum_{(i,j) \in \mathcal C^-} \log (1 - \sigma(c - d(i, j)))}^{\text{non-complementary pairs}}.
\label{eq:hengelPaper2}
%\done{equation numbering? Fine}
\end{equation}

Finally, although the approach is mainly designed for item-to-item recommendation (and as such is not \emph{personalized}), a personalized version can be developed by adding a user latent vector $\gamma_u$ that encodes which dimensions of this `style space' are important to each user:
\begin{equation}
    d_u(i,j) = \sum_k (\gamma_{u,k} s_{ik} - \gamma_{u,k} s_{jk})^2
    %\done{fix (u,k) vs (uk) everywhere. I think is okay now}
    \label{eq:hengelPaper3}
\end{equation}
(in this case, the model is trained on triples $(u,i,j)$ of co-purchases of items by each user $u$).

%\done{Note similarity compared to cold-start paper from previous section (both bilinear models). Doesn't matter}
% Pairwise Preference Regression for Cold-start Recommendation

%Ultimately, the paper 
\citet{mcauley2015image}
show that this type of model can be used in several ways. First, it can predict co-purchases accurately, especially when predictions are personalized. Second, the use of image data is effective at \emph{visualizing} the parameters of the model, i.e.,~determining what are the primary dimensions that explain variance in users' `styles.' Finally, since the model (as in \cref{eq:hengelPaper1}) takes only images as input, it can be transferred to assess compatibility (and arguably, `fashionability') of outfits outside of the original training data.

%Various other papers have made use of the same co-purchase data, or followed up on the basic idea above. For example, 
\cite{veit2015learning} made use of the same co-purchase data to solve the same task, but did so directly from the `pixel level,' i.e.,~by training a Convolutional Neural Network, rather than using a pre-trained image representation. This type of architecture is depicted in \Cref{fig:Siamese}: two input images (items), labeled as `compatible' or `incompatible,' are passed through two CNNs, both of which share the same parameters. The CNNs learn low-dimensional representations $\phi(x)$ and $\phi(y)$ for the two items; these are essentially equivalent to the `style space' embeddings of \Cref{eq:hengelPaper1}, except that they are learned from the pixel level, and therefore could potentially capture subtle characteristics not available in pre-trained representations. Like \Cref{eq:hengelPaper1}, the model is trained to learn a metric, so that compatible items have nearby embeddings and incompatible items do not. 

\begin{figure}
%tikz code from https://github.com/PetarV-/TikZ/blob/master/Generative%20adversarial%20network/README.md
\centering
\begin{tikzpicture}
[
% ArrowC2/.style={% Arrows with big rounded corners
%         rounded corners=.5cm,
%         thick,
%         },
]

	\node (z) {item $y$};
	\node[circle, draw, thick, right=5em of z] (x) {$\phi(y)$};
	\draw[->] (z) -- node[above] {CNN} (x);
	%\node[left=of z] (i) {};
	%\draw[->, thick] (i) -- node[above] {$p_\theta(\vec{z})$} (z);
	\node[above=of x, circle, draw, thick] (xt) {$\phi(x)$};
	\node[left=5em of xt] (it) {item $x$};
	\draw[->] (it) -- node[above] {CNN} (xt);
	\node[thick, right=4em of x, yshift=3.1em, xshift=-0.9em] (D) {$\| \phi(x) - \phi(y) \|$};
	\node[right=2em of D] (out) {compatible?};
	\draw[->] (D) -- (out);
			
% 	\node[right=2.5em of x, circle, fill, inner sep=0.15em] (pt1) {};
% 	\node[right=2.5em of xt, circle, fill, inner sep=0.15em] (pt2) {};

			
	\draw[dashed, thick] (x) edge[bend right] (xt);
			
	\node[circle, draw, thick, fill=white, inner sep=0.15em, left=of D, xshift=1pt] (pt3) {};
			
% 	\draw[->, thick] (x) -- (pt1);
% 	\draw[->, thick] (xt) -- (pt2);
	\draw[->] (pt3) -- (D);

\end{tikzpicture}
\caption{Basic Siamese setup for item-to-item compatibility. \label{fig:Siamese}}
%\done{edit this for this specific case, improve caption. Is fine}
%\done{also add \gamma_u to the input to the generator. Not meant for here?}
%\done{round dot isn't in exactly the right place (this and other image). Fixed}
\end{figure}

\cite{he2016learning} showed that prediction performance can be improved in fashion settings by using separate embeddings for the `query' and `target' items $i$ and $j$, i.e.,
\begin{equation}
    d(i,j) = \| \gamma_i - \gamma'_j \|_2^2.
    \label{eq:nonMetricEq}
\end{equation}
Critically, by using two different latent spaces, `compatibility' need not follow the assumptions of a metric space, e.g.~an item need not be compatible with itself. In this way the model can learn which aspects should be systematically different when matching items---such as blue pants going with brown shoes.
%\done{A little too much about my own papers, could shrink. Fine}

\subsection{Learning Compatibility from Images in the Wild}
\label{subsec:compatWild}
%\done{I don't think there's anything else to add?}

The above papers showed that visual data can be effective when predicting co-purchases in domains like fashion. However, it is arguable whether such models actually learn a useful notion of `fashionability.' For one, a co-purchase is a very noisy indicator of whether two items are compatible: in practice, two purchases on one account may not even be for the same person. Secondly, the images in such datasets (e.g.~product images from Amazon) are not `wild' images (they are usually scaled, centered objects on a white background), so 
%could not be used 
might struggle
to capture the fashionability of an outfit in a photo (for example).

\cite{kang2019complete} tried to address this issue by developing models of fashionability that operated directly on images in the wild. At training time, the approach consists of an image (or `scene') that is known to contain a particular item; each item also has a `clean' product image much like those in \Cref{sec:hengelPaper}. The basic idea is then that the item must be compatible with other object in the scene.

%\done{mention street2shop dataset (from WC paper), maybe in next para. Covered paper}

To build a training set, the known items are cropped from each training image, each 
resulting in a scene image that does \emph{not} contain the known item, but ought to contain compatible objects. Ultimately, 
this means that we have a set of training pairs consisting of an item
%separately have an item, 
plus a (cropped) scene that is known to be compatible with the object (but does not contain it). From here, fashion compatibility can be estimated by learning a relationship between the scene ($s$) and product $p$) images:
\begin{equation}
    d(s,p) = \| f(s) - f(p) \|_2^2.
\end{equation}
Much like \Cref{eq:hengelPaper1}, this distance function is based on learned embeddings of the scene and product images.
%($s$ and $p$). 
Several differences exist between these embeddings and those used in \Cref{sec:hengelPaper}, mostly to account for the fact that the scene image likely contains a large number of \emph{irrelevant} objects. Critically, the method uses an \emph{attention mechanism} (as in \cref{subsec:attention}, see also e.g.~\cite{xu2015show}), which is used to identify regions of the scene image that are relevant for compatibility detection. For example, in an outdoor image, the attention mechanism may learn to focus on the person in the image and their outfit, while ignoring `background' objects in the surrounding scene.

Note that the above is ultimately not a \emph{personalized} model, i.e.,~there are no parameters associated with the individual users. However the system is arguably more useful than that of \Cref{sec:hengelPaper} from a personalization perspective, in the sense that it would allow a user to upload an image of themselves and receive recommendations that complement their personal style. As such, it is an example of 
%the type of 
\emph{contextual} or non-parametric personalization.
%that we discussed in \Cref{sec:nonparametric}.

In addition to experiments on personalized fashion, \cite{kang2019complete} show that the same technique can be used for complementary item recommendation in other settings, such as recommending compatible furniture based on items in a living room.

%\done{also mention the use for furniture}

%\done{Mention that other ruining paper that learned non-metric embeddings? Nah}

%\done{Throughout book: see if I should use \bm (boldmath) for math in headings. Done}

\subsection{Generating Fashionable Wardrobes}

The papers above consider \emph{pairwise} compatibility among items as a proxy for selecting sets of items that will form compatible outfits.

\cite{hsiao2018creating} consider the more challenging problem of finding \emph{collections} of items that can be used to generate compatible outfits (called `capsule wardrobes'). 
%They note a general limitation of much of the above work in terms of the difficulty and noise involved in collecting pairwise training data
%
Their specific notion of a `wardrobe' is a set of items belonging to fixed sets of categories (or `layers'), e.g.~tops, bottoms, outerwear. The suggested quality of good a wardrobe is that it should be capable of generating many outfits, i.e.,~a large number of sets of items should be mutually compatible. At the same time, a wardrobe should have a wide \emph{variety} of items (e.g.~many nearly-identical pairs of jeans and t-shirts would generate many outfits but would not be a good wardrobe).

%The approach proposed by \cite{hsiao2018creating} to defining the quality of a single outfi
First \cite{hsiao2018creating} define a measure to determine the compatibility of items in a single outfit. Their specific approach is based on \emph{topic modeling} which we studied a little in \Cref{sec:reviewAware}. Essentially, an outfit is represented by a low-dimensional vector; each dimension of this vector in turn corresponds to a mixture over certain visual attributes (e.g.~an outfit dimension might correspond to `floral patterns,' and this dimension might be associated with visual attributes describing appearance, shape, cut, etc.).\footnote{`Outfits' and `attributes' are roughly analogous to `documents' and `words' in the original topic model formulation \protect\citep{blei2003latent}.}
%
A set of items is said to constitute a good outfit if their collective attributes resemble those of training outfits.

\cite{hsiao2018creating} explore the relationship between these interacting components (outfits, wardrobes, versatility, as well as personalization of these components), and develop optimization schemes to circumvent the combinatorial nature of the problem. 

Perhaps the most important contributions in \cite{hsiao2018creating} are simply the creative use of training data, and the formalization of what it means for a set of items to be `good' in terms of compatibility.
%
\cite{hsiao2018creating} make the argument that this type of model and training procedure is preferable to other techniques that rely on e.g.~pairwise compatibility relationships; a topic model-based approach results in a holistic notion about the overall qualities of an outfit, and allows for training on complete outfit images rather (e.g.)~collecting training data from co-purchases (as in \cref{sec:hengelPaper}), which may be subject to noise or otherwise not representative of real fashion compatibility.

\subsection{Domains other than Fashion}

The visual dynamics of fashion items are often the focus of studies on personalized visual models, though fashion is not the only domain in which visual features play a key role.

A few others have sought to study personalized visual dynamics in related domains.
\cite{bell2015learning} learned models of visual compatibility of home decor items collected from \emph{Houzz};
%\done{style}
the model is similar to the fashion compatibility models we studied in \Cref{sec:hengelPaper}, with the main goal being to learn a similarity function between images via a Siamese network. 
%Here, rather than estimating whether two items of clothing would appear in an outfit, the model is trained to estimate whether two home decor items would appear together in a scene; a main challenge in this work is to build ground-truth data of visually compatible items in a scene
At training time, the problem is essentially cast as a form of visual search: that is, given a scene containing an item, identify the item. This is achieved by training on a dataset of image pairs, where one image consists of an item in a scene while the other consists of a clean (or `iconic') product image. As such, the learned distance metric simply attempts to map both \emph{in-situ} images and clean images to the same point. Although the main application for this task is visual search (i.e.,~find the item in this image, or find images containing this item), other potential applications are discussed that make use of the learned metric, such as identifying stylistically compatible items across categories.

%For instance the model of 
\cite{kang2019complete} also adapted their model (which we studied in \Cref{subsec:compatWild}) to the problem of furniture / home decor recommendation.
%\done{make sure reference points to an earlier section}
%was also used to study \emph{furniture} recommendation (using a dateset of interior design and home decor items from \emph{Pinterest});
The technique is essentially the same as that described in \Cref{subsec:compatWild}, but is trained on a dataset of interior design and home decor items from \emph{Pinterest}.
%\done{style}
Here, rather than estimating a clothing item which completes an outfit (by withholding that item from the scene during training), the method is used to 
%estimate a piece of furniture 
identify home decor items which are
%which is 
visually compatible (or complementary to) others in the scene.

%\done{both paragraphs above are essentially walls of text. Fine}



%Learning visual similarity for product design with convolutional neural networks
%\cite{kang2019complete} (furniture)

%Kang paper (art) - also mention that there is a social aspect to art recommendation.
\cite{he2016vista} considered visual dynamics in the context of \emph{art} recommendation. Their setting is an online art community (\emph{Behance}), where personalized preferences can potentially be guided by a variety of visual, temporal, and social factors. Spiritually, the modeling approach is similar to that used to model the temporal dynamics of fashion (as in \cref{subsec:VBPR}), i.e.,~by combining pre-trained image embeddings with a variety of application-specific temporal dynamics. The main component of their temporal model is a Markov chain-based approach (as in \cref{sec:markovCaseStudies}), whereby interactions are largely guided by recent context. \cite{he2016vista} also observe that art recommendation has a significant social component, whereby preferences can be estimated based on the identity of the artist as much as from the art itself (as in e.g.~\cref{subsec:bartering}).

\subsection{Other Techniques for Substitutable and Complementary Product Recommendation}
\label{subsec:otherTech}

While much of the work on substitutable and complementary item recommendation is motivated by applications in fashion (where such recommendations can naturally be used to generate outfits, etc.),
%\done{not good sentence. Fine}
a few others have considered the problem more broadly, either to consider different modalities of data (besides visual features), or to consider complementarity in settings other than fashion.

% \paragraph{Substitutes and Complements from Text}

% \cite{mcauley2015inferring} attempted to estimate substitutable and complementary products using the same dataset of \emph{Amazon} co-purchase and co-browsing data as in \Cref{sec:visualCompat}.

%\done{dropped}

% \begin{equation}
% \underbrace{\sum_{(i,j) \in \mathcal C} \log \sigma(\gamma_i \cdot \gamma_j)}_{\text{complementary pairs}} + \overbrace{\sum_{(i,j) \in \mathcal C^-} \log \sigma(\gamma_i \cdot \gamma_j)}^{\text{non-complementary pairs}} +
% \lambda \underbrace{\sum_{(u,i) \in \mathcal T} \sum_{w \in d_{u,i}} \log p(w | \gamma_u, \psi)}_{\text{topic likelihood}},
% \label{eq:sceptre}
% \end{equation}

%Inferring Networks of Substitutable
%and Complementary Products
%https://cseweb.ucsd.edu/~jmcauley/pdfs/kdd15.pdf

% \paragraph{Learning non-metric product relationships}
\subsubsection{Learning non-metric item relationships}
%\done{Is "asymmetric" the right word? Really it's about identity, not symmetry. Think about it. Here and elsewhere. Not perfect but okay}

\cite{wang2018path} also studied the problem of recommending substitutable and complementary products, proposing several modifications to the approaches
%used in \cite{mcauley2015inferring} (\cref{sec:?}).
we studied 
%in \Cref{sec:?}.
above.

First, as we saw in \Cref{sec:fashionCaseStudies}, a good model for complementary product recommendation ought to recognize that `complementarity' should ideally not be captured by a similarity function. That is, complementary items have systematically \emph{different} characteristics, and in particular an item is not complementary with itself.

Following this logic, 
%a simple modification of the complementarity function in \Cref{eq:sceptre} %consists of using 
we might train a complementarity model consisting of
two sets of factors $\gamma_i$ and $\gamma'_j$ (much like that of \Cref{eq:nonMetricEq}) for complementary pairs $i$ and $j$:\footnote{Note that in practice, complementary pairs $i \rightarrow j$ are \emph{directed}, e.g.~a large fraction of camera purchases are paired with a memory card, whereas only a small fraction of memory card purchases are paired with a camera.}
\begin{equation}
\underbrace{\sum_{(i,j) \in \mathcal C} \log \sigma(\gamma_i \cdot \gamma'_j)}_{\text{complementary pairs}} + \overbrace{\sum_{(i,j) \in \mathcal C^-} \log \sigma(\gamma_i \cdot \gamma'_j)}^{\text{non-complementary pairs}}
\label{eq:nonMetricComp}
\end{equation}

%Following this modification, 
Several other extensions are proposed, mostly to handle data sparsity issues and cold-start scenarios. 
%Whereas 
%\cite{mcauley2015inferring} did so using signals from text; 
\cite{wang2018path} leveraged knowledge about the specific semantics of substitutable items; for example, if complementary pairs tend to belong to specific sub-categories (e.g.~shirts are often compatible with jeans), this acts as a weak signal that other products within these categories are also complementary. Likewise, substitutable relationships might be \emph{transitive} (for example), i.e.,~if $i$ is substitutable for $j$, and $j$ is substitutable for $k$, then $i$ is likely substitutable for $k$; various `soft' constraints of these types ultimately help to improve performance.
%of the model.

% A Path-constrained Framework for Discriminating
% Substitutable and Complementary Products in E-commerce
%https://dl.acm.org/doi/pdf/10.1145/3159652.3159710

% \paragraph{Diversifying complementary product recommendation}
\subsubsection{Diversifying complementary item recommendation}

Although complementary recommendations are intended to be distinct from the query item, this does not necessarily mean that recommended complements will be distinct from \emph{each other}.
For example, it would presumably not be useful to recommend only t-shirts as a complement for a pair of jeans; instead, one might wish to recommend a combination of shirts, belts, shoes, etc.

Although we'll revisit the notion of diversity in depth in \Cref{chap:consequence}, here we discuss some approaches that consider the problem within the specific context of complementary item recommendation.

\cite{he2016learning} considered that a good set of complementary items might be represented as a \emph{mixture} over different notions or `modes' of compatibility $c$. They start with a simple pairwise compatibility model between items $i$ and $j$ of the form
\begin{equation}
    d_c(i,j) = \| \gamma_i - \gamma^{(c)}_j \|_2^2,
    \label{eq:compEq}
    %\done{reference from exercise. No}
\end{equation}
where $\gamma_i$ and $\gamma^{(c)}_j$ are based on image embeddings.
%The superscript $c$ denotes a specific `mode' of compatibility 
This is similar to the model of \Cref{eq:hengelPaper1}, though includes \emph{separate} embeddings $\gamma$ and $\gamma^{(c)}$ for the `query' item $i$ and complementary item $j$; using separate embeddings breaks the symmetry between $i$ and $j$, which is desirable for complementary items (e.g.~an item should not be complementary with itself).\footnote{As such, \protect\Cref{eq:compEq} is no longer a distance function.}

Next, \cite{he2016learning} note that \Cref{eq:compEq} captures only a single notion of compatibility; if a model is trained using such a function, this will presumably correspond to the predominant mode of compatibility in the data, but will not be diverse.  To address this, \cite{he2016learning} proposed treating the compatibility relationships in the data as a probabilistic mixture over several competing notions.
This idea borrows from a mathematical framework known as a \emph{mixture of experts} \citep{jacobs1991adaptive}.
Specifically:
\begin{equation}
  d(i,j) = \sum_{c = 1}^C \underbrace{p(c|i)}_{\mathclap{\text{\vphantom{\Large{L}}relevance of compatibility function $c$ to query $i$}}} \cdot d_c(i,j).
\end{equation}
Here, $p(c|i)$ measures which types of compatibility relationships $d_c(i,j)$ are most likely to be relevant for a query item $i$; while written as a probability, this can more simply be thought of as a function that is used to combine compatibility relationships with different weights. Here
\begin{equation}
    p(c|i) = \frac{\exp(\theta_c f_i)}{\sum_{c'} \exp(\theta_c f_{i'})},
\end{equation}
where $f_i$ is a feature vector describing the image $i$, and $\theta_c$ is a parameter vector associated with the $c^\text{th}$ compatibility function.

Ultimately this model learns $C$ separate embeddings $\gamma^{(c)}_i$ for each item (along with the query embedding $\gamma_i$), each corresponding to a different notion of compatibility or `complementarity.' In principle, this means the model is able to capture several different modes of compatibility that interact simultaneously. At test time, diverse lists of compatible
%\done{I use the terms compatible and complementary inconsistently. Is fine}
items can generated by sampling from different compatibility functions $d_c(i,j)$ according to their relevance $p(c|i)$.

% \paragraph{Incorporating product types}
\subsubsection{Incorporating item types}
%\done{This paper isn't very interesting, could drop. Is fine}
\cite{hao2020p} noted that both accuracy and diversity of complementary product recommendation can be achieved my making explicit use of available category data. Instead of directly predicting which items $j$ are compatible with a query item $i$, the approach first attempts to estimate which of several \emph{categories} are relevant to a given query; following this, the method generates complements via several category-specific compatibility functions, similar to $d_c(i,j)$ form \Cref{eq:compEq}.
%\done{Figure out venue and replace by actual citation}

%P-Companion: A Principled Framework for Diversified
%Complementary Product Recommendation
%https://assets.amazon.science/d5/16/3f7809974a899a11bacdadefdf24/p-companion-a-principled-framework-for-diversified-complementary-product-recommendation.pdf

\subsection{Implementing a Compatibility Model in \emph{Tensorflow}}
\label{subsec:imageComCode}
%\done{Check italics of tensorflow}

Most of the compatibility models we saw in the previous section are relatively straightforward to implement on top of pre-trained image features.

Below we assume a feature matrix $X$ such that $x_i$ is an image feature describing item $i$ (e.g.~a feature from ImageNet as in \cite{he2015vbpr}), and that each pair $(i,j)$ is associated with a label $y$ determining whether the pair is compatible ($y=1$) or not ($y=0$).
%\done{x_i or x_i? Lowercase}
The model then projects the images into `style space' via $s_i = E_i x_i$ and $s_j = E_j x_j$ (similar to \Cref{eq:vbprAttempt2}); here we use two separate embeddings so that the model can learn asymmetric relationships.
%\done{Doesn't match any of the notation from when I intorduced the model}
%\done{wrong notation, should be s_i, s_j not gamma}
% Finally compatibility is evaluated via $\sigma(c - d(\gamma_i, \gamma_j))$ (similar to \Cref{eq:hengelPaper2}):
Finally compatibility is evaluated via $\sigma(c - d(s_i - s_j))$ (similar to \Cref{eq:hengelPaper2}):

% \noindent
% \begin{minipage}{\linewidth}
% \begin{lstlisting}[escapechar=\%]
% class CompatibilityModel(tf.keras.Model):
%     def __init__(self):
%         super(CompatibilityModel, self).__init__()
%         # Embeddings for the query item (Ei) and target item (Ej)
%         self.Ei = tf.Variable(tf.random.normal([featDim,styleDim],stddev=0.001))
%         self.Ej = tf.Variable(tf.random.normal([featDim,styleDim],stddev=0.001))
%         # Offset term as in %\Cref{eq:antonModel?}%
%         self.c = tf.Variable(0.0)

%     # Given image features x1 and x2, and label y (0/1)
%     def call(self, xi, xj, y):
%         # Style-space embeddings %$\gamma_i$ and $\gamma_j$%
%         gammai = tf.matmul(xi, self.Ei)
%         gammaj = tf.matmul(xj, self.Ej)
%         # Shorthand for %\Cref{eq:?}%
%         return -tf.math.log(tf.math.sigmoid(self.c - tf.math.squared_difference(gammai,gammaj))*(2*y - 1) - y + 1)

% model = CompatibilityModel()
% \end{lstlisting}
% \end{minipage}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
class CompatibilityModel(tf.keras.Model):
    def __init__(self, featDim, styleDim):
        super(CompatibilityModel, self).__init__()
        # Embeddings for the query item (Ei) and target item (Ej)
        self.E1 = tf.Variable(tf.random.normal([featDim,styleDim],stddev=0.001))
        self.E2 = tf.Variable(tf.random.normal([featDim,styleDim],stddev=0.001))
        # Offset term as in %\Cref{eq:hengelPaper2}%
        self.c = tf.Variable(0.0)

    # Given image features x1 and x2, and label y (0/1)
    def call(self, x1, x2, y):
        # Style-space embeddings %$\gamma_i$ and $\gamma_j$%
        s1 = tf.matmul(x1, self.E1)
        s2 = tf.matmul(x2, self.E2)
        # Shorthand for %\Cref{eq:hengelPaper2}%
        return -tf.math.log(tf.math.sigmoid(self.c - tf.reduce_sum(tf.math.squared_difference(s1,s2)))*(2*y - 1) - y + 1)

model = CompatibilityModel(4096, 5)
\end{lstlisting}
\end{minipage}
%\done{fill in equation}
%\done{Change so that the number of style dimensions is a parameter to initialize the model}
%\done{Squared difference looks wrong, since it's an elementwise operation. Fixed}

Similarly, we could modify the code to use the compatibility 
%function of \Cref{eq:wangChengPaperEq}:
based on the inner product:
%\done{reference from exercise?}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
def call(self, x1, x2, y):
    s1 = tf.matmul(x1, self.E1)
    s2 = tf.matmul(x2, self.E2)
    return -tf.math.log(tf.math.sigmoid(self.c - tf.matmul(s1,tf.transpose(s2)))*(2*y - 1) - y + 1)
\end{lstlisting}
\end{minipage}

Finally, we train the model by sampling compatible and incompatible pairs from our training set and computing gradients:\footnote{Although this example is simple enough to allow reasonably fast training, there are several ways that this code could be made more efficient, e.g.~the embeddings of all images in $X$ could be computed simultaneously.}
%\done{example in footnote is vague. Fine}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
def trainingStep(compat):
    with tf.GradientTape() as tape:
        (i1,i2,y) = random.choice(compatiblePairs)
        x1,x2 = X[i1],X[i2]
        objective = model(x1,x2,y)
    gradients = tape.gradient(objective, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
\end{lstlisting}
\end{minipage}


\section{Personalized Generative Models of Images}
\label{sec:personalGans}

In \Cref{chap:text}, we examined personalized text models from two directions: first, we used text within \emph{predictive} tasks, e.g.~we saw how text can be used for regression problems (\cref{sec:textBasics}), and to improve the performance of recommender systems (\cref{sec:personalText}). Second, we saw how to personalize \emph{generative} models of text (\cref{sec:personalTextGeneration}), i.e.,~to generate text that matches a user's writing style or preferences.

Likewise, our discussion of visual data has so far considered using images to improve predictive performance; it is worth spending a little time exploring how to personalize \emph{generative} models of images.

Briefly, the basic framework we will consider extending is that of the \emph{Generative Adversarial Network}, or GAN.

%\done{taken from my own paper, maybe change text a bit more. Looks fine}
Generative Adversarial Networks are an unsupervised learning framework in which two components `compete' to generate realistic looking outputs (in particular, images) \citep{goodfellow2014generative}. One component (a generator) is trained to generate data, while another (a discriminator) is trained to
distinguish real versus generated data. Thus the generated
data are trained to look `realistic' in the sense that they are
indistinguishable from those in the dataset.
%
%\done{same with this paragraph. Looks fine}
Such systems can also be conditioned on additional inputs, in order to sample outputs with certain characteristics \citep{mirza2014conditional}. 


The basic setup of a Generative Adversarial Network 
%(GAN) 
is depicted in \Cref{fig:GANdiagram}. Here $x_{\mathit{real}}$ is an image sampled from a dataset, whereas $x_{\mathit{gen}}$ is a synthetically generated image. The discriminator 
%\done{Clean up all notation. Meaningless}
$D(x)$ is then given an image $x$, and is responsible for predicting whether it is a sample from the dataset or is a synthetic image. 
%\done{See if this is too similar to the above text or not. Looks okay}
For image generation, the discriminator is typically a form of convolutional neural network (CNN), whereas the generator is a series of \emph{deconvolution} operators, essentially operating via a similar principle as the CNN, but in reverse. The generator takes as input a latent code $z$, a random input which allows the generator to produce distinct images ($z$ is essentially a \emph{manifold} which describes patterns of variation in image data so as to capture the variability in the training dataset). The discriminator $D$ and generator $G$ are trained simultaneously, such that the generator gradually becomes better at generating images that are capable of `fooling' a better and better discriminator.


\begin{figure}
%tikz code from https://github.com/PetarV-/TikZ/blob/master/Generative%20adversarial%20network/README.md
\centering
\begin{tikzpicture}
[
ArrowC2/.style={% Arrows with big rounded corners
        rounded corners=.5cm,
        % thick,
        },
]

	\node[circle, draw, thick] (z) {${z}$};
	\node[circle, draw, thick, right=5em of z] (x) {${x}_{\mathit{gen}}$};
	\draw[->] (z) -- node[above] {$G({z})$} node[below] {generator} (x);
	\node[left=of z] (i) {};
	\draw[->] (i) -- node[above] {$p_\theta({z})$} (z);
	\node[above=of x, circle, draw, thick] (xt) {${x}_{\mathit{real}}$};
	\node[left=5em of xt] (it) {};
	\draw[->] (it) -- node[above] {$p_{\mathit{data}}({x})$} (xt);
	\node[circle, draw, thick, right=5em of x, yshift=2.5em] (D) {${x}$};
	\node[right=7em of D] (out) {real?};
	\draw[->] (D) -- node[above] {$D({x})$} node[below] {discriminator} (out);
	
	\node[below=of i, yshift=1em] (dotted1) {};
	\node[below=of out, right=of dotted1, xshift=25em] (dotted2) {};
			
	\node[right=2.5em of x, circle, fill, inner sep=0.15em] (pt1) {};
	\node[right=2.5em of xt, circle, fill, inner sep=0.15em] (pt2) {};
	
	\draw[dashed] (dotted1) -- (dotted2);

	\draw[dashed, thick] (pt1) edge[bend left] (pt2);
			
	\node[circle, draw, thick, fill=white, inner sep=0.15em] at ([xshift=-0.85em, yshift=3.9em]pt1.north) (pt3) {};
			
	\draw[->] (x) -- (pt1);
	\draw[->] (xt) -- (pt2);
	\draw[->] (pt3) -- (D);
	
	%\node[below=of pt1, circle, fill, inner sep=0.15em] (pt3) {};
	\node[below=6em of D, circle, draw, thick] (pt3) {};
	\node[right=0.5em of pt3, circle, fill, inner sep=0.15em] (pt4) {};
	\node[right=0.5em of pt4, circle, draw, thick] (pt5) {};
	
	\draw[-, ArrowC2] (x) |- node[above, xshift=3em] {$\Phi(x_{gen})$} node[below, xshift=3em] {(i.e.,~$\gamma_i$)} (pt3);
	\node[right=0.1em of pt5] (gamma) {$\gamma_u$};
	\node[below=of pt4] (out2) {compatibility};
	\draw[->] (pt4) -- (out2);
	
	%\draw[-stealth, thick] (pt4) -- (out2);
%	\draw[-stealth, thick] (z) -- node[above] {$G(\vec{z})$} node[below] {generator} (x);

%\done{Arrows don't look the same in this (and some) figures as elsewhere}
%\done{consistent style for all figure arrows}

\end{tikzpicture}
\caption[Basic setup of a personalized generative adversarial network.]{Basic setup of a Generative Adversarial Network (GAN), and a personalized GAN. The components above the dotted line depict the `standard' GAN setup, in which a generator competes with a discriminator to generate images that are indistinguishable from real data. Components below the dotted line are used to develop a personalized GAN that encourages the generated image to be compatible with a particular user. \label{fig:GANdiagram}}
%\done{edit this for this specific case, improve caption. Okay}
%\done{also add \gamma_u to the input to the generator. it's in the compat function}
\end{figure}

%We follow a similar approach based on activation maximization [11] to generate images that best match a selected user's personal style. GANs have been used for broad applications, though to our knowledge we are the first to apply this architecture to fashion image generation



The above type of architecture has been used to generate various types of realistic images, including artwork, clothing, and human faces.
%\done{could use citations but can't find useful ones. Useless sentence anyway. Fine}
\cite{kang2017visually} sought to develop \emph{personalized} GANs, that would generate images that capture the preferences of individual users.

The approach from \cite{kang2017visually} essentially combines the GAN framework with a personalized image preference model, similar to that of Visual Bayesian Personalized Ranking (\cref{subsec:VBPR}). The basic idea is depicted in \Cref{fig:GANdiagram} (bottom). Once a GAN is trained (as above), the image $G(z)$ generated by a given latent code is passed to both the discriminator ($D$) as well as a personalized preference model. For the preference model, the image is represented via $\Phi(G(z))$; this is analogous to $\gamma_i$, or ${E}f_i$ as in \Cref{eq:vbprAttempt2}, though unlike VBPR the synthetic image is not associated with any particular item $i$. A user's preference toward this synthetic item is then estimated via $\gamma_u \cdot \Phi(G(z))$. Ultimately, the objective is that a generated image should be simultanously plausible (according to $D(G(z))$), but also desirable to the user (according to $\gamma_u \cdot \Phi(G(z))$):
\begin{equation}
    \argmax_z \underbrace{\gamma_u \cdot \Phi(G(z))}_{\mathrlap{\hspace{-3mm}\text{user preference toward generated image}}} - \overbrace{\eta(D(G(z)) - 1)^2}^{\mathllap{\text{`plausability' of generated image}\hspace{-3mm}}}.
    \label{eq:WCGAN}
\end{equation}
%In practice, since the image which optimizes \Cref{eq:WCGAN} is no longer simply an output 

\cite{kang2017visually} argue that a model such as that above can be used in several ways. Most straightforwardly, it can be used to generate designs (or images) that match the preferences of individual users; \Cref{eq:WCGAN} can be straightforwardly modified to find optimal designs for a \emph{population} of users (e.g.~by taking an average over users $\sum_u \gamma_u \cdot \Phi(G(z))$). Alternately, given an existing image (rather than a generated image $G(z)$), \Cref{eq:WCGAN} (or rather, its gradient) can be used to suggest \emph{local modifications} to the image that will make it preferable to a user or population.




% \section{History and Related Work}

%\done{Moved}

% Personalized recommendation problems involving visual data have been studied for several years. Initial attempts ignore visual data altogether, and long predate solutions that model images explicitly. For example, an early system for clothing recommendation \cite{hu2014style} learns a user's `style' in order to recommend clothing, but does so using `likes' rather than any analysis of visual features. Likewise, \emph{YouTube's} early recommendation approaches \cite{davidson2010youtube} are based on heuristic `relatedness scores' based on co-visitation (essentially a form of neighborhood-based approach, as in \Cref{sec:neighborhood}), though some features based on video metadata are included in the model; newer solutions (based on deep learning) adopt more complicated candidate generation and ranking strategies, though again make little if any use of explicit visual signals \cite{covington2016deep}.

% Recommenders

% Generation

% \section{Exercises}
%\done{Sometimes called problems, sometimes called exercises? Exercises}

\begin{exercises}
\item Starting from the code from \Cref{subsec:imageComCode}, and using a small dataset of compatible (and non-compatible) pairs (e.g.~from \emph{Amazon} clothing products, as in \cite{mcauley2015image}),
%\done{have mentioned this data?}
set up a pipeline for estimating compatibility relationships (e.g.~`also bought' or `also viewed' products). 
%\done{style. Fine}
Tune the model (e.g.~in terms of the number of embedding dimensions, regularization, etc.)~and measure its accuracy (in terms of its ability to successfully distinguish compatible from non-compatible items).
%\done{any other tuning?}
\label{ex:vis1}
\item The model from Exercise~\ref{ex:vis1} is based on a distance (or similarity) function of the form
$\| s_i - s'_j \|_2^2$. Note that there is nothing particularly special about this specific choice of similarity function (much as we discussed in \Cref{sec:whyInner}). Consider whether variants of this model might lead to better performance, for example:
\begin{itemize}
    \item Is squared distance preferable to an inner product, or other choices of distance function?
    \item The model from \Cref{subsec:imageComCode} embeds image features $f_i$ into a latent space via $s_i = E f_i$. Is this superior to a purely latent embedding ($\gamma_i$)? How do they compare for cold versus warm items?
    \item Different categories could have different compatibility semantics. Is it useful to learn different embeddings $E_c$ per category?
\end{itemize}
%\done{Check whether that's correct. What?}
\item 
%Harder ways of sampling negative items
A challenging aspect of learning a compatibility model between images is that of generating negative samples (i.e.,~pairs of items believed to be incompatible). If we na\"ively generate samples 
%for a positive pair $(i,j)$ 
by randomly choosing 
%a random incompatible item $k$, 
incompatible items,
we may learn a trivial solution which merely predicts that (e.g.)~men's shoes don't tend to be compatible with women's dresses. In other words, the model may have learned to do little more than to indirectly categorize items. Such a model may be accurate, but would hardly have learned the semantics of what combinations are \emph{fashionable}. Alternately, try training a compatibility model that for each positive (compatible) pair $(i,j)$ selects a more challenging negative pair $(i,k)$ where $k$ has \emph{the same category} as $j$; this will force the model to rely on aspects like color, texture, patterns (etc.)~rather than simply learning to categorize items. Evaluate your solution by training both models (randomly sampled, and within-category), and comparing their performance on a within-category test set.
% \item Although models based on visual features are useful in cold-start settings, there is in principle no reason that 
%\done{don't remember what the above was?}
\item 
\label{ex:TSNE}
When studying notions like fashion compatibility (or indeed, any latent item representation in a recommender system), it is worth exploring whether the learned representations semantically correspond to our intuitive notion of similarity. To assess this, it helps to visualize latent item representations $\gamma_i$ in two dimensions (for the sake of plotting them). In \Cref{fig:item2vec} we did this simply by learning two-dimensional item representations, though by doing so we are likely visualizing a sub-optimal model.
%\doe{Could move visualization outside of this exercise? Nah}
Various techniques exist that learn distance-preserving embeddings for the sake of visualizing data.\footnote{That is, so that we can embed $K$-dimensional data into two dimensions such that `similar' items in the original space are still nearby once they are embedded.}
%https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
Below we show code for embedding a matrix of representations via \emph{t-SNE} \citep{mcinnes2018umap}:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
import numpy as np
from sklearn.manifold import TSNE

X_embedded = TSNE(n_components=2).fit_transform(X) # X is a matrix of all item representations %$\gamma_i$%
\end{lstlisting}
\end{minipage}

Using this (or an alternative embedding) strategy, visualize the embeddings you fit in the above exercises. To understand the semantics of the embeddings, it can be useful to visualize them by category (as in \cref{fig:item2vecTSNE}) or some other feature (price, brand, etc.).

\begin{SCfigure}
\includegraphics[scale=0.75]{fig/item2vecTSNE.pdf}
\caption[Item representations embedded via \emph{t-SNE}.]{Ten-dimensional item representations ($\gamma_i$) embedded into two dimensions via \emph{t-SNE}; this is a ten-dimensional version of the model from \protect\Cref{fig:item2vec}. %Representations for items from three distinct categories are shown. 
\label{fig:item2vecTSNE}}
%\done{not referenced}
\end{SCfigure}
%\done{reference figure, write TSNE exercise}

%\done{Transformers from Scratch -- look into. No}
%\done{Auralist - introducing serendipity into music recommendation -- look into. Covered in serendipity section}

\end{exercises}
%\done{These questions are more of a project, could rewrite as a project rather than exercises. Is okay}

\refstepcounter{projectCounter}
\label{proj:visualProj}
\section*{Project \theprojectCounter: Generating Compatible Outfits}
\addcontentsline{toc}{section}{Project \theprojectCounter: Generating Compatible Outfits}
%\done{Rename project. Seems okay}

%\done{fix all \cite vs \citep}

%\done{should I ever use double `` ''? Removed}

%\done{intro?}

In this project we'll explore various ways to build outfit recommenders, following similar strategies to those we developed in \Cref{sec:fashionCaseStudies}.

First, consider how you would generate a training dataset of compatible items. It is probably most realistic to start by considering \emph{pairwise} compatibility, i.e.,~to generate a training dataset of pairs of items $(i,j)$ that are mutually compatible (as in \cref{sec:fashionCaseStudies} or \cref{subsec:imageComCode}). Even then, several options are available for mining pairwise compatibility data. For example:
\begin{itemize}
    \item Co-purchase relationships (e.g.~`people who bought $i$ also bought $j$'), as in \cite{mcauley2015image}.
    %\done{how do I normally write this?}
    \item Directly mining co-purchased items from user interaction histories (e.g.~if a user $u$ purchased both items $i$ and $j$, this is an indication that they might be compatible). This strategy was also explored in \cite{mcauley2015image}.
    \item Both of the above approaches are highly noisy, as items are not necessarily co-purchased with the intention of being worn together. A third approach consists of mining explicit relationships from actual outfit data. e.g.~as in \Cref{subsec:compatWild}.
    %\done{select appropriate paper. pointed to section}
\end{itemize}

Consider the advantages and disadvantages of each of the above approaches. For example, which will allow you to collect the most data, and which will be the least noisy? Further consider how you should select samples, e.g.~you probably want to avoid pairs $(i,j)$ where both items belong to the same category, or might further restrict your dataset to certain categories of interest.

Similarly, you should choose an appropriate strategy to generate \emph{negative} samples for training, i.e.,~pairs $(i,j)$ that do \emph{not} go together. Trivially, such samples could be generated from pairs of random items, though more `difficult' negatives could be generated by selecting pairs with specific categories.

Having built your dataset, there are several potentially interesting directions for study. For example:
\begin{enumerate}
    \item What is an appropriate model to use to estimate compatibility relationships? A good starting point may a model such as that from \Cref{eq:nonMetricComp}, since compatibility relationships are likely to be asymmetrical in this context.
    %\done{Make sure this is sufficiently distinct between compatibility vs. outfits. Dunno}
    \item Consider whether it is worthwhile to incorporate visual features to estimate compatibility via an embedding strategy (e.g.~following the code from \Cref{subsec:imageComCode}) or whether it is sufficient to model compatibility in a latent space (e.g.~as in \Cref{eq:nonMetricEq}).
    \item Consider whether it is useful to incorporate other features, such as brands, prices, features from text, etc.
    \item 
    %(harder)
    %\done{Do I have these `hard' / `harder' tags anywhere? Either use them regularly or not at all. Removed for now}
    Is there value to training a \emph{personalized} model for this task, i.e.,~rather than predicting whether a pair of items $(i,j)$ are compatible, can you predict whether $i$ and $j$ are compatible \emph{for a particular user $u$}.
    %\done{I think this text may be similar to text elsewhere. Fine}
    Think carefully whether the identity of the user explains a significant amount of variation in compatibility relations, and whether enough data can be mined to fit a personalized model (similar to the approach in \Cref{eq:hengelPaper3}).
    %\done{did I already talk about such a model in the chapter? Yes}
\end{enumerate}

Finally, consider ways to visualize the model (or its predictions), either by representing items in a low-dimensional space (as in Exercise~\ref{ex:TSNE}), or by building a simple interface to explore compatible items. 

Note that other than the use of visual features, the above steps could be used to build item-to-item compatibility models for \emph{any} types of data (e.g.~dishes in a menu, songs in a playlist), and are not limited to outfit generation.



%\chapter{The Ethics of Personalized Machine Learning}
% \chapter{Bias and Fairness in Personalized Machine Learning}
% \chapter{Fairness, Bias, and the Consequences of Personalized Machine Learning}
\chapter{The Consequences of Personalized Machine Learning}
\label{chap:consequence}
%\done{Change chapter name}

% \section{How Can Personalized Machine Learning be Improved?}

So far, we have largely viewed personalized machine learning as a `black-box' 
%machine learning 
task. That is, given a user, their context, and some potential stimulus, can we estimate how the user will react to that stimulus?

This black-box view of machine learning, while effective for building accurate models, ignores the potential real-world consequences of how such models are applied.

Broadly, the dangers of blindly applying machine learning models are well-studied: ML algorithms can perpetuate, mask, or amplify biases in training data, or have low accuracy for underrepresented groups. Detecting and mitigating these types of biases largely describes the study of `fairness' in machine learning (see e.g.~\cite{dwork2012fairness}).

Within the context of \emph{personalized} machine learning, black-box models, if applied carelessly, can also hide or amplify biases or other issues. Below we highlight a few examples, to be studied throughout this chapter:
\begin{itemize}
    \item A recommender system, although ostensibly designed to aid \emph{discovery}, may actually have a `concentration' effect, where users are gradually locked into a `filter bubble' containing only a narrow set of items (\cref{sec:filterBubble}).
    \item Alternately, by recommending content that maximally aligns with a user's interests, a system may gradually push them toward more and more `extreme' content (\cref{sec:filterBubble}).
    \item Recommender systems may have reduced utility for users (or groups of users) who are underrepresented in the training data, for instance `popular' items that are widely recommended may merely reflect what is popular among the majority group (\cref{sec:fairness}).
    \item Recommendations may focus only a user's predominant interest, while failing to capture the diversity of breadth of their interactions (\cref{subsec:calibration}).
    \item Likewise, systems could disadvantage vendors (or content creators, etc.)~by failing to recommend products in the long tail (\cref{sec:multisided}).
\end{itemize}

%As an example 
To conceptually demonstrate some of the above issues, \Cref{fig:InnerNN} 
highlights 
the ways in which a recommender system, if applied na\"ively, may lead to a `concentration' or `extremification' effect. At left we show personalized recommendations generated by maximizing an inner product ($\gamma_u \cdot \gamma_i$), as in \Cref{subsec:matrixFactor}; at right we show item-to-item recommendations generated by finding similar items (i.e.,~nearest neighbors of $\gamma_i$).
%\done{description doesn't sound right. Done}
%
When maximizing an inner product (\cref{fig:InnerNN}, left), the recommended items are on the `fringe' of the item space; roughly speaking, if I know that a user likes action movies (for example), then I might recommend movies with \emph{the most} action. While this might make sense in the context of movie recommendation, when recommending (e.g.)~political videos on \emph{YouTube}, such a strategy may drive users toward fringe or `extreme' content. Alternately, when choosing nearest neighbors (\cref{fig:InnerNN}, right), users' recommendations are concentrated around very similar content, which may lead to a `filter-bubble' effect.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/recPointsInner.pdf}
    \includegraphics[scale=0.75]{fig/recPointsNN.pdf}
    \caption[Comparison of inner product and nearest neighbor recommendations.]{Recommendations selected for a user by maximizing an inner product (left) or taking nearest neighbors (right). \label{fig:InnerNN}}
\end{figure}
%\done{nowhere to put this. Here is fine}

While the above is merely a conceptual demonstration, in the following sections
%\done{check organization. Looks okay}
we'll 
%start by 
examine empirical studies (e.g.~from \emph{YouTube} and \emph{Facebook})
%\done{style. Looks okay}
that analyze filter bubbles and extremification in the context of deployed recommendation settings. 
%
%Later we'll look at issues of 
%\done{Nothing more to add?}
We'll also look further into issues of diversity, bias and fairness, exploring a wide variety of potential consequences of personalized model training.

These ideas connect to the broader topic of fairness and bias in machine learning, though as we'll see the issues in personalized settings can be quite different. Much of our focus when introducing these issues is to present strategies to address them, in order to build personalized models that are more diverse, unbiased, and fair.
%\done{ugly paragraph, not well connected. Fine}

%\section{Diversity}
%\done{Probably drop section since other sections on diversity are not subsections of this one. Dropped}

\section{Measuring Diversity}
\label{sec:measuringGini}
%\done{Probably not a subsection. Upgraded to section}

% Before exploring case studies measuring the effect of recommendations on diversity, and techniques to improve diversity, we first briefly consider the kind of measurements we'll use to assess diversity. 
Before exploring 
case studies measuring the effect of recommendations on 
diversity (and filter bubbles, extremification, etc.),~it is useful to briefly consider how we might 
%practically measure some of these quantities on top of the types of 
assess such effects on top of of
recommenders like those we've developed in previous chapters.
%and techniques to improve diversity, we first briefly consider the kind of measurements we'll use to assess diversity. 
We'll look at two main concepts: first,
%\done{Capital after : ? No}
across \emph{all} users, do recommended items follow the same \emph{distribution} as the set of items that were consumed? Second, among individual users, are the recommended items more or less diverse than their historical consumption trends?

We first train a recommender (here using comic books from \emph{Goodreads}), following the code presented in \Cref{subsec:BPRTensorflow} (i.e.,~using a model based on Bayesian Personalized Ranking).

Next we generate a set of example recommendations from the model and compare those with the original interaction data. For each user, we generate as many recommendations as they have interactions in the original data, so that each user is represented the same number of times in both our interaction data and our empirical recommendation data:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
countsPerItem = defaultdict(int)

for u in range(nUsers):
    # Given a matrix of interactions X, as in %\Cref{sec:implicit}%
    recs = model.recommend(u, X, N = len(itemsPerUser[u]))
    for i, score in recs:
        countsPerItem[i] += 1
\end{lstlisting}
\end{minipage}
%\done{Fix cref above}

Next we compare the measurements above to the same measurements derived from the interaction data. In particular, do popular items (based on the number of historical interactions) frequently appear among recommendations; and conversely, are frequently recommended items popular? We plot these comparisons in \Cref{fig:recDiversityComics}.

\begin{figure}
    \centering
    \includegraphics[scale=0.75]{fig/diversityDist.pdf}
    \includegraphics[scale=0.75]{fig/diversityDist2.pdf}
    \caption[Distribution of interactions compared to recommendations.]{Distribution of interactions compared to recommendations (based on an implicit-feedback model trained on comic books from \emph{Goodreads}). The left plot measures the \emph{recommendation} frequency of the 200 most popular items (as measured by the number of interactions in the training set); the right plot measures the \emph{interaction} frequency of the 200 most recommended items.}
    \label{fig:recDiversityComics}
    %\done{Plots could both have different titles. Looks fine}
    %\done{These plots don't look right. Shouldn't the mass of the compared distributions be the same? No because it's only the head of the distributions}
    %\done{confusing caption, especially last sentence. Fine}
\end{figure}

Surprisingly, neither of the two appear to match very closely (i.e.,~popular items do not get recommended frequently, and \emph{vice versa}).
%\done{Do I use the term vice versa ever? Yes several times. Italicize consistently. Always italics for now}

There are various ways we could formally measure the discrepancy between these two distributions, or compute summary statistics that help us to compare them.
In the context of item recommendations, we might be interested in whether one of the two distributions is more \emph{concentrated} than the other, i.e.,~whether recommendation frequencies are highly `peaked' around a few popular items (versus a flatter, or longer-tailed distribution).
One measure (that we'll see used 
%in several of 
in some of 
the studies below) 
%\done{check that it's below}
is the \emph{Gini coefficient}, a measure of statistical dispersion. 
%of item interaction frequencies.
Given a set of measurements $y$ (in this case, frequencies associated with each item), the Gini coefficient measures the \emph{average (absolute) difference between frequencies}, i.e.,:
\begin{equation}
    G(y) = \frac{\sum_{i = 1}^N \sum_{j=1}^N | y_i - y_j |}{2 N^2 \bar{y}}.
    \label{eq:gini}
\end{equation}
%\done{equation is ugly. Looks okay}
Highly concentrated data will have a large coefficient, while flatter distributions will have $G(y)$ close to zero
(the presence of $2\bar{y}$ in the denominator scales the expression to be in the range $\lbrack 0, 1 \rbrack$).
%\done{handwavy explanation. Looks okay}

In practice the coefficient is usually computed approximately, i.e.,~by sampling rather than enumerating all possible pairs of items:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
def gini(y, samples=1000000):
    m = sum(y) / len(y) # average
    denom = 2 * samples * m
    numer = 0
    for _ in range(samples):
        i = random.choice(y)
        j = random.choice(y)
        numer += math.fabs(i - j)
    return numer / denom
\end{lstlisting}
\end{minipage}

In the case of this particular experiment, the interaction data yields a Gini coefficient of $G \simeq 0.72$ while the recommendations yield $G \simeq 0.80$.
%0.7201648536364775
%0.80445217092293
In other words, this particular recommendation algorithm has resulted in recommendations that are somewhat more `concentrated' compared to historical interaction data.

\section{Filter Bubbles, Diversity, and Extremification}
\label{sec:filterBubble}

% \cite{pariser2011filter} (coins the term filter bubble)
%see https://www2.kbs.uni-hannover.de/fileadmin/institut/pdf/webscience/2016-17/papers/nej1.pdf
% for reading list on filter bubble background

The idea that 
%personalized recommendation traps 
recommender systems trap
users in `filter bubbles' \citep{pariser2011filter} amplify existing biases toward popular items, or guide users toward extreme content, are often reported in popular media and discussed anecdotally, 
though such concepts are often not precisely defined. 
%though it is hard to precie
It is also difficult to measure these types of dynamics empirically, as one rarely has the ability to analyze the counterfactual scenario in which the recommender wasn't in place.
%\done{Ugly opening paragraph, could connect. Added below}

Below we explore several attempts to measure diversity (and related notions such as filter bubbles and extremification) more precisely, either through simulation, or by empirically measuring the interaction patterns of real users.

%\done{use \paragraph for single paragraphs, subsubsection otherwise}

%Paper: Recommender Systems and their Impact on Sales Diversity

\subsection{Exploring Diversity Through Simulation}
\label{subsec:simulation}
%\done{not capitalized? Check all subsections}

An early paper that attempted to define and analyze the impact that recommender systems have on interaction diversity did so via simulation \citep{fleder2009blockbuster}. They noted the existence of two competing hypotheses as to why recommender systems might encourage or discourage diversity: on the one hand, recommender systems can guide content \emph{discovery}, which can increase the diversity of item interactions \citep{brynjolfsson2006niches}; on the other hand, recommender systems might reinforce the popularity of already-popular products, thus reducing diversity \citep{mooney2000content}. Their attempt to resolve this question built a simple simulation which generates recommendations, 
in which the probability of an item being recommended, or the probability of a user accepting it, can be controlled.
%which simulated users have a controlled probability of accepting. 
%By controlling the acceptance probability (or other properties of the simulation), they can simulate recommendations with varying levels of influence.
By varying the controllable parameters, they show that under almost all conditions (i.e.,~except in edge cases), the recommender system leads to a concentration effect (i.e.,~results in a reduction in interaction diversity), as measured by the Gini coefficient.
%(a measure of the statistical disbursion of item interaction frequencies).
%MOVED UP

Of course, the above is not necessarily true of \emph{every} recommender system; indeed as we'll see in \Cref{sec:diversification}, one can design a recommender system so as to explicitly target the diversity of recommended items.
Rather, the above result is simply a demonstration that under fairly minimal conditions, recommender systems \emph{can} lead to a concentration effect.
%(just as they did in \Cref{sec:?}).

%\paragraph{Directly Measuring Algorithmic Recommendation and Diversity}
\subsection{Empirically Measuring Recommendation Diversity}
\label{subsec:empirical}
%\done{Crappy section title. Better}

Following \cite{fleder2009blockbuster}, which studied the possibility of filter bubbles via simulation, \cite{nguyen2014exploring}
%was the first to 
present an initial attempt to
empirically measure the effect of recommender systems on content diversity in a real setting.

The research questions in \cite{fleder2009blockbuster} are similar to the ones studied via the simulations above, namely: do recommender systems gradually expose users to narrower content over time; and, how does this effect vary as a function of how receptive users are to recommendations.

Diversity in \cite{nguyen2014exploring} is defined in terms of a `tag genome,' which is a collection of tags assigned to movies. %Similarity is then defined in terms of standard measures (e.g.~cosine similarity) between movie features. 
Standard similarity measures (e.g.~cosine similarity) are then used to measure the similarity (or spread) between recommended and consumed movies.

Empirically, \cite{nguyen2014exploring} found reduced diversity over time, both for recommendations and users' actual interactions. Interestingly though, the effect is mitigated for users who tend to interact with (i.e.,~rate) the system's recommendations.
%Arguably this suggests that there is an 
As such, while there appears to be an
overall concentration effect in users' interaction patterns, 
%which is not 
%but that it may not be entirely
%necessarily 
%guided by the recommender system. 
it is not entirely clear what role the recommender system plays.
%This latter argument is somewhat comparable to 
We'll investigate this question a little further via
a more recent empirical study on \emph{Facebook} in \Cref{subsec:filter}.
%data that we discuss later (\cref{subsec:filter}).
%\done{could say more. no}

% \subsubsection{Exploring diversity through \emph{YouTube} recommendations}
%\done{Dunno about these paragraph or overall organization. Upgraded to subsubsection}

%
%Paper: The impact of YouTube recommendation system on video views .
%
%Following this work, 
\cite{zhou2010impact} conducted a similar empirical study of recommendations on \emph{YouTube},
%\done{Rewrite the above sentence, since I've reorganized}
%
%For example, an early paper studying \emph{YouTube} recommendations argued that \emph{YouTube's}
%\done{italics?}
and argued that YouTube's recommendations (specifically the `related videos' feature) have a positive impact on content diversity.
They showed that recommendations drive a large fraction of views on YouTube,
%\done{captilaitaion and italicize}
and that views driven by recommendations have higher diversity than views from a popularity-driven system.\footnote{Note however that it is hard to argue that recommendation-driven views are therefore `diverse,' given that popularity-driven views would presumably lead to high concentration, and as such are arguably 
%a weak 
not a particularly diverse
baseline.}
%\done{not beautifully worded}




% \subsection{Diversity and Extremification}
\subsection{Auditing Pathways to Extreme Content}

%However, 
% Even if consumption is `diverse,' as defined by the overall distribution of interactions across an item vocabulary, this does not guarantee that users are necessarily exposed to diverse viewpoints, or are not pushed toward extreme content.
%Let's consider the trivial motivating example in \Cref{fig:InnerNN}
%\done{Could introduce it here}
% For example, returning
%\done{is it cited before? Yes}
% to the motivating example from \Cref{fig:InnerNN}, recommendations based on maximizing an inner product (left) would expose users to more `diverse' content than the nearest neighbor model (right), but may gradually drive users toward `fringe' items that are aligned with, but are more extreme versions of, the content they initially consumed.

\cite{ribeiro2020auditing} attempted to empirically analyze the pathways via which users arrive at extreme content on \emph{YouTube}. 
The authors used curated lists of channels (for their study, of `alt-right' political channels) in order to establish a ground-truth of what is meant by `extreme' content; they also collected less extreme content (`alt-lite,' general media, etc.), in order to determine whether there are systematic pathways from less- to more-extreme content (by tracking users' commenting histories) over time.

Their main finding is that there appears to be a trajectory where users migrate from less (e.g.~`alt-lite') to more extreme content, and that users who interact with extreme content can often be traced to an earlier point in time where they primarily interacted with more moderate channels. They also consider the role that recommendations play in this radicalization process, noting that there tend to be pathways from more moderate communities to more extreme content (though this pathway tends to be through channel rather than video recommendations). Although they cannot assess the role of \emph{personalization} in this process (as they do not have access to the actual recommendations surfaced to users), this suggests that even simpler item-to-item recommenders can still guide users to extreme content.




%Auditing Radicalization Pathways on YouTube (https://arxiv.org/pdf/1908.08313.pdf); separate section on extremification?

\section{Diversification Techniques} % Filter bubble
\label{sec:diversification}

% https://www.sciencedirect.com/science/article/abs/pii/S0950705117300680

%\done{Shouldn't this go next to "measuring diversity" section. No it's fine here.}

Having argued in theory how recommenders might guide users to niche, highly similar, or extreme content (\cref{fig:InnerNN}), and having 
assessed the same problems
%shown 
empirically 
%the role that recommenders play 
through the case studies above, we now turn to strategies that can be used to mitigate these consequences. Here we study techniques that attempt to balance relevance with \emph{diversity}.  Diversification is a simple strategy that seeks to optimize the aggregate quality of a set of results by ensuring that none are excessively self-similar.
%before considering more elaborate  
Diversity is just the one of several 
%more `qualitative' metrics we'll consider: 
`beyond accuracy' metrics we'll consider (we'll explore several others in \Cref{sec:otherMetrics}). Such metrics are in some sense largely qualitative: to the extent that we optimize and evaluate models based on relevance (ratings, likelihood of purchasing, etc.), here we are generally seeking to improve some subjective notion of aggregate usefulness (in this case, that a set of results should not be too self-similar). 
One theme we'll see 
%achieving such goals is to show 
when designing diversification techniques
is that we can significantly increase diversity (and other metrics) with only a minimal reduction in relevance. 
We'll see more detailed strategies for evaluating these types of methods via case studies in \Cref{sec:caseStudiesDiverse}.

% one can imagine how recommendations based on nearest neighbors or maximum inner-products, as in \Cref{fig:InnerNN}, might focus exclusively on a user's predominant interest alone.

The methods discussed in this section (and another we'll discuss later) are summarized in \Cref{tab:divTech}.

%\done{table needs to move to earlier section, section refs adjusted}

%\done{think whether refs can be added to other table entries. Added}

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Summary of diversification techniques.]{Summary of diversification techniques. References: \protect\cite{carbonell1998use,adomavicius2011improving,wilhelm2018practical,zhang2012auralist,steck2018calibrated}.}
\label{tab:divTech}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{|XXX|}
% \textbf{Technique} & \textbf{Reference} & \textbf{Description}\\
% Maximal Marginal Relevance & \cite{carbonell1998use}, \Cref{sec:MMR} & Recommended items should balance utility against diversity compared to already-recommended items.\\
% Aggregate Diversity & \cite{adomavicius2011improving} & Recommended items should be those that have high compatibility for a particular user, but low aggregate compatibility (e.g.~popularity); this will lead to \emph{aggregate} diversity of recommendations across the entire population.\\
% Determinantal Point Processes & \cite{wilhelm2018practical} \Cref{?} & Balance utility and diversity (as with MMR above), but using a set-based objective.\\
% Calibration & \cite{steck2018calibrated} & Recommendations should exhibit the same distribution of attributes (e.g.~in terms of recommended categories) as users' historical interactions.\\
% \end{tabularx}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.22\textwidth}X}
\toprule
\textbf{Ref.} & \textbf{Technique} & \textbf{Description}\\
\midrule
% \cite{carbonell1998use}
CG98 &
Maximal Marginal Relevance  & Recommended items should balance utility against diversity compared to already-recommended items (\cref{sec:MMR}).\\
% \cite{adomavicius2011improving}
AK11 &
Aggregate Diversity  & Recommended items should be those that have high compatibility for a particular user, but low aggregate compatibility (e.g.~popularity); this will lead to \emph{aggregate} diversity of recommendations across the entire population (\cref{susbsec:rerank}).\\
% \cite{wilhelm2018practical} 
W18 &
Determinantal Point Processes & Balances utility and diversity (as with MMR above), but using a set-based objective (\cref{subsec:dpps}).\\
% \cite{steck2018calibrated}
Z12 & Serendipity & Recommendations should be relevant, but unexpected compared to those in the user's history (\cref{subsec:serendipity}).\\
S18 &
Calibration & Recommendations should exhibit the same distribution of attributes (e.g.~in terms of recommended categories) as users' historical interactions (\cref{subsec:calibration}).\\
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{Fix, figure out how best to reference sections in table}
%\done{not referenced}

% Diversifying Search Results (Agarwal)


%\done{Move diversity to recommender systems chapter? No}

\subsection{Maximal Marginal Relevance}
\label{sec:MMR}
%\done{I think I found this from a DPP paper, probably the youtube one. Maybe check the argument they make relating this to other ideas. Don't remember what this is}

%\done{Maximum or maximal?}

A simple notion of diversity that is commonly used in document retrieval scenarios is that among a (ranked) list of retrieved documents, each retrieved item should simultaneously be relevant, but at the same time not too similar to the already-returned items.

This notion is captured by the \emph{Maximal Marginal Relevance} \citep{carbonell1998use} procedure (MMR). The approach was originally designed for retrieving sets of text passages that best summarize a document (with respect to some query): each document should be similar to the query, but also dissimilar from the documents already retrieved.

The same concept can straightforwardly be applied in recommendation scenarios, given that we have notions of both \emph{relevance} and \emph{similarity} available (e.g.~relevance might be the output of a latent factor model, while similarity could be defined in terms of cosine similarity, or as an inner product between item representations $\gamma_i$ and $\gamma_j$.
%for two products).

To apply the concept to recommendation, we would define the Maximal Marginal Relevance as follows:
\begin{equation}
    \mathit{MMR} = \argmax_{i \in R \setminus S} \Bigl[ \lambda\underbrace{\mathit{Sim}^{\text{user}}(i,u)}_{\mathclap{\text{relevance to the user}}} - (1 - \lambda)\overbrace{\max_{j \in S}\mathit{Sim}^{\text{item}}(i,j)}^{\mathclap{\text{similarity to already-recommended items}}}\Bigr],
    \label{eq:MMR}
\end{equation}
where $R$ is an initial candidate set of recommendations (most trivially e.g.~a list of items the user hasn't already interacted with), and $S$ is a set of items retrieved so far. $\mathrm{Sim}^{\text{user}}$ and $\mathrm{Sim}^{\text{item}}$ are item-to-user and item-to-item similarity functions (respectively); the former is presumably the compatibility function returned by a recommender system;
%\done{repetitive writing compared to above paragraph. Fine}
the latter is any item-to-item similarity measure (e.g.~a cosine similarity or an inner product between item representations).

Note that the above is computed iteratively, that is we add one result at a time by maximizing the MMR until the list $S$ has the desired size. Finally, $\lambda$ trades off the extent to which we care about compatibility versus diversity.

\subsection{Other Re-ranking Approaches to Diverse Recommendation}
\label{susbsec:rerank}
%\done{Bad section name, since MMR is also reranking. (added "other")}

%Improving aggregate recommendation diversity using ranking-based techniques \cite{adomavicius2011improving}

Similar to Maximal Marginal Relevance, several approaches have been specifically designed for re-ranking in recommendation scenarios. \emph{Re-ranking} approaches, including MMR, assume that we are given an initial ranking function which we trust to find items of high relevance, but which lack diversity; 
%thus one seeks a re-ranking procedure that balances the two concerns.
thus we wish to re-rank
%procedure that balances the two concerns.
these initial results to balance the two concerns.

One such re-ranking approach for recommendation was proposed in \cite{adomavicius2011improving}. The approach assumes the presence of three components: first, 
%they assume the availability of 
a compatibility score, e.g.~a rating prediction $r(u,i)$. Second, a \emph{relevance-oriented} ranking technique, $\mathit{rank}_u(i)$; this could be any ranking function (though most trivially one might simply order predictions by 
%the predicted rating score 
$r(u,i)$). And third, another
%second 
`diversity oriented' ranking function; conceptually, this should focus on recommending items to users that they would \emph{not} normally consider.

An example of a diversity-oriented loss from \cite{adomavicius2011improving} is to sort items by popularity, with the \emph{least} popular items being ranked first:
\begin{equation}
    \mathit{rank}^{(\text{pop})}(i) = | U_i |.
    %\done{needs an argument}
\end{equation}
Recommending \emph{unpopular} items does not at first appear to be a particularly effective recommendation strategy; however this ranking is used in conjunction with the prediction score $r(u,i)$. Specifically, to encourage diversity we want to find \emph{unpopular} items that \emph{this} user is likely to enjoy. Spiritually, this is somewhat reminiscent of our \emph{tf-idf} approach to finding important words in documents (\cref{sec:tfidf}).
%; here `words' correspond to items, and documents (roughly) correspond to the set of items this user 

The specific (re-)ranking objective from \cite{adomavicius2011improving} then looks like
\begin{equation}
    \mathit{rank}'_u(i, t) = \left\lbrace
    \begin{tabular}{ll}
    $\mathit{rank}^{(\text{pop})}(i)$ & if $r(u,i) \geq t$\\
    $\alpha_u + \mathit{rank}_u(i)$ & otherwise
    \end{tabular}
    \right..
    \label{eq:adomDiv}
\end{equation}
Here $t$ is a threshold term, essentially determining whether one of the low-popularity recommendations has a high enough score to recommend; $\alpha_u$ is an offset term ensuring that the popularity-based recommendations appear first in the ranking before those of $\mathit{rank}_u(i)$.

\cite{adomavicius2011improving} show that as the threshold $t$ is changed, the system gradually trades-off between recommendation precision and diversity. 
Several different ranking functions are considered, for example to replace popularity-based ranking by alternatives based on the average rating, rating variance, etc.

They also note that the type of diversity achieved by this ranking mechanism is quite different from that in \Cref{sec:MMR}, as it does nothing to encourage diversity (or dissimilarity) among an individual user's item list. Instead, they discuss the related notion of \emph{aggregate diversity}, which defines diversity across the item vocabulary itself, i.e.,~recommendations across \emph{all} users should have reasonable coverage of the complete item vocabulary. This is related to the notion of $P$-fairness
%\done{Is this C or P? I think P is right}
that we'll discuss in \Cref{sec:multisided}.



\subsection{Determinantal Point Processes}
\label{subsec:dpps}
%\done{Needs to move to diversity section}
%Youtube papers

%In \Cref{sec:MMR} 
%and \Cref{sec:ImprovingRecommendationListsPaper}
%\done{Others. Is fine}
So far we've
we discussed various approaches that attempt to balance accuracy and diversity via what are essentially `heuristic' strategies that greedily select items that maximize utility while being sufficiently novel compared to the rest.
%\done{Is the above a fair description? Yes}
%\done{Needs gluing sentence. I think it's fine}

Determinantal Point Processes (or DPPs) \citep{kulesza2012determinantal} are a \emph{set-based} optimization technique that can be used to identify subsets of items that simultaneously maximize item quality and diversity among items. Specifically, given a set of items $I$, a DPP assigns a probability $p(S)$ to every subset 
%$S \in \mathbb{P}(I)$, where $\mathcal{P}(I)$ is the \emph{powerset} (i.e.,~the set of all subsets) of $I$. 
$S \subseteq I$.
The goal is then to model (i.e.,~parameterize) this probability, either globally or for individual users, such that finding the subset of items which maximizes $p(S)$ has an optimal tradeoff between utility and diversity.

\cite{wilhelm2018practical} studied the application of DPPs to diversify recommendations on \emph{YouTube}.
%\done{italics, caps}

The method assumes a few inputs. First, as with previous diversification techniques, we assume a utility or `quality' estimate $f(u,i)$ is given (e.g.~from a pre-trained recommender system), which encodes the probability that user $u$ will interact with item $i$ given $i$'s features; we also assume a predefined distance function $d(i,j)$ between two items.

Next, we have historical sets of items that have been surfaced to the user (i.e.,~the outputs of an existing system), along with subsets of items that the user selected 
%(i.e.,~binary labels 
(indicated by binary labels $y_{u,i}$). The goal is to select subsets of items that will maximize the total number of interactions, which in practice is trained by maximizing the Cumulative Cain:
\begin{equation}
    \sum_{u}\sum_{i} \frac{y_{u,i}}{\mathit{rank}_u(i)},
\end{equation}
where $\mathit{rank}_u(i)$ is the new rank assigned by the proposed algorithm. That is, items the user interacted with ($y_{u,i} = 1$) should have high rank (see~\cref{subsec:cumGain}).
%\done{Cover Cumulative Gain in recommender systems evaluation or something}

Note that the above seems reminiscent of `traditional' approaches to recommendation, i.e.,~we are ranking items such that positive interactions should have high rank (which seems similar to what we saw in \Cref{subsec:cumGain}). The main difference here is simply the observation that the total number of interactions will be maximized when utility and diversity are balanced (e.g.~a user will quickly become bored if recommendations cover only one of their interests).

Next, given a candidate set of $N$ items, we next define a matrix $L^{(u)}$ such that diaginal entries $L^{(u)}_{i,i}$ encode the utility of an item $i$ and off-diagonal entries $L^{(u)}_{i,j}$ encode the similarity of two items $i$ and $j$. The specific parameterization used in \cite{wilhelm2018practical} is:
\begin{align}
  L^{(u)}_{i,i} & = f(u,i)^2\\
  L^{(u)}_{i,j} & = \alpha f(u,i) f(u,j) \exp(- \frac{d(i,j)}{2\sigma^2}) \quad \text{for $i \neq j$}.
  \label{eq:dppParams}
\end{align}

Now, the quality of a subset $S$ is proportional to the determinant of the submatrix of $L$ induced by $S$, $\det(L_S)$. Specifically:
\begin{equation}
    % p(S) = \frac{\det(L_S)}{\sum_{S' \in \mathbb{P}(I)} \det(L_{S'})}.
    p(S) = \frac{\det(L_S)}{\sum_{S' \subseteq I} \det(L_{S'})}.
    \label{eq:dpp1}
\end{equation}
%\done{Could rewrite everything with subset notation rather than element of powerset}
Critically, the denominator of the above equation can be computed efficiently as
\begin{equation}
    % \sum_{S' \in \mathbb{P}(I)} \det(L_{S'}) = \det(L + I),
    \sum_{S' \subseteq I} \det(L_{S'}) = \det(L + I),
\end{equation}
where $I$ is the identity matrix.

To understand (roughly) why the determinant is diversifying, it helps to consider the trivial example where $S$ consists of only two items $i$ and $j$; then the determinant is given by
\begin{equation}
    \det \left( \left[ \begin{tabular}{cc}
      $L_{i,i}$ & $L_{i,j}$\\
      $L_{j,i}$ & $L_{j,j}$
    \end{tabular} \right] \right) = L_{i,i}L_{j,j} - L_{i,j}L_{j,i};
\end{equation}
this value will be maximized when the utility is high ($L_{i,i}L_{j,j}$) and the similarity is low ($L_{i,j}L_{j,i}$).

In spite of the relatively simple form of \Cref{eq:dpp1}, it is still not practical to solve the (NP-hard) problem of finding the optimal subset. In \cite{wilhelm2018practical} this is addressed by a simple greedy algorithm (similar to that of \Cref{sec:MMR}, among others), in which one starts with the empty set of videos $S = \emptyset$, 
%\done{Consistent emptyset symbol. Think it's always \emptyset, and this is the only one}
and iteratively adds the item $i$ which maximizes the determinant $\det(L_{S \cup \lbrace i \rbrace})$

Note that the parameterization in \Cref{eq:dppParams} includes two tunable parameters, $\alpha$ and $\sigma$.
%, which in practice are tuned using grid search. 
Intuitively these parameters control the relative weight of utility versus diversity ($\alpha$), and the `tightness' of the similarity function ($\sigma$). These parameters are selected globally, but in principle could be learned per-user.
%\done{Have I defined grid search anywhere? Removed}

Ultimately, the experiments find that implementing the above DPP on a user's video feed increases user satisfaction (as measured by session duration), compared to various other diversification strategies.

%Practical Diversified Recommendations on YouTube with Determinantal Point Processes



% \subsection{Topic Diversification}
% \label{sec:topicDiverse}

%\done{finish or drop. Dropped}

% Improving recommendation lists through topic diversification \cite{ziegler2005improving}

% Avoiding Monotony: Improving the Diversity of Recommendation Lists \cite{zhang2008avoiding}
%can't find pdf

% Category-Driven Approach for Local Related Business
% Recommendations
%https://dl.acm.org/doi/pdf/10.1145/2806416.2806495



\section{Implementing a Diverse Recommender}
\label{sec:implementDiverse}
%\done{Should this be subsection here or elsewhere? Moved}

Here we'll briefly describe an implementation of a diversified recommender. Our implementation is based on the \emph{maximal marginal relevance} method of \Cref{sec:MMR},
%though other reranking-based methods are similar).
%This code 
though
could be straightforwardly adapted to implement other re-ranking strategies from \Cref{susbsec:rerank}.
%\done{generic opening. Fixed}

We start by building a few utility data structures. First we collect the list of candidate recommendations, excluding any items the user has already consumed. Next we compute compatibility scores between a user $u$ and all items. Here our compatibility scores (i.e.,~$\mathrm{Sim}^{\text{user}}(i,u)$ in \Cref{eq:MMR})
are simply the output of a latent factor recommender
(here we use a batch-based prediction function as in \Cref{subsec:batchBased}). We sort these in order from the highest- to the lowest-rated items. In practice we might want to re-rank only the top few hundred items rather than computing diversity scores for extremely low-compatibility items.

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
candidates = list(itemSet.difference(itemsPerUser[u]))
compatScores = list(zip([float(f) for f in model.predictSample([userIDs[u]]*len(candidates), [itemIDs[i] for i in candidates])], candidates))

compatScores.sort(reverse=True)
\end{lstlisting}
\end{minipage}

Next
we implement a function to determine the similarity between a candidate recommendation and others already in the list (i.e.,~$\max_{j \in S}\mathrm{Sim}^{\text{item}}(i,j)$ from \Cref{eq:MMR}). 
\texttt{itemEmbeddings} is a lookup table containing embeddings $\gamma_i$ for each item. 
The similarity function (\texttt{sim}) is the cosine similarity (not shown), though other similarity functions could be substituted (including simple alternatives such as checking whether items belong to the same category):

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
itemEmbeddings = dict(zip(candidates, tf.nn.embedding_lookup(model.gammaI, [itemIDs[i] for i in candidates])))

def maxSim(itemEmbeddings, i, seq):
    if len(seq) == 0: return 0
    return max([sim(itemEmbeddings,i,j) for j in seq])
\end{lstlisting}
\end{minipage}

To implement the iterative re-ranker
%\done{rerank or re-rank. Re-rank}
we define a method that takes the list of recommendations generated so far (\texttt{seq}), and generates the next item to be added to the list, based on the weighted combination from \Cref{eq:MMR}. $\lambda$ is passed as an argument to the function to trade-off the importance of compatibility and diversity:

%\done{Correct all code to have even spacing between arguments "item, user" vs "item,user"}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%]
def getNextRec(model, compatScores, itemEmbeddings, seq, lamb):
    scores = [(lamb * s - (1 - lamb) * maxSim(itemEmbeddings,i,seq), i) for (s,i) in compatScores if not i in seq]
    (maxScore,maxItem) = max(scores)
    print(maxScore)
    return maxItem
\end{lstlisting}
\end{minipage}

Note that the above implementation is inefficient and even on a modestly sized dataset (in terms of the size of the item vocabulary) requires several seconds to generate recommendations. Several strategies might be used to improve its performance, including efficient retrieval techniques (as in \cref{sec:retrieval}), or by exploiting certain structure in our compatibility or diversity functions that would obviate the need to exhaustively compute all scores.

\paragraph{Examples of diversified recommendations}
%
\Cref{tab:diverseRec} shows examples of diversified recommendations on beer review data.
%from beer reviews.
%\emph{BeerAdvocate}.
%\done{Remove explicit references to beeradvocate?}
%\done{Do I call it beeradvocate or just the beer dataset?}
Different values of $\lambda$ are chosen to control the compatibility/diversity trade-off (for a randomly chosen user). The first set of recommendations ($\lambda = 1$) optimizes only for compatibility: the user is recommended a selection of rich stouts and IPAs. Decreasing $\lambda$ by a little (middle column) introduces a few `lighter' yet similar beers; decreasing $\lambda$ further results in beers from a wide variety of categories (wheat beers, lambics, scotch ales, etc.).


\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption{Diversified recommendations (maximal marginal relevance).
%\done{maximum or max? Here and elsewhere. Maximum}
}
\label{tab:diverseRec}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
\begin{tabularx}{\linewidth}{lXXX}
% \hrule\\
\toprule
\textbf{rank} & \textbf{Low diversity} & \textbf{Medium diversity} & \textbf{High diversity}\\
\midrule
1  & Founders KBS (Kentucky Breakfast Stout) & Founders KBS (Kentucky Breakfast Stout) & Founders KBS (Kentucky Breakfast Stout)\\
2  & Two Hearted Ale                         & Samuel Smith's Nut Brown Ale            & Samuel Smith's Nut Brown Ale\\
3  & Bell's Hopslam Ale                      & Two Hearted Ale                         & Salvator Doppel Bock\\
4  & Pliny The Elder                         & Bell's Hopslam Ale                      & Oil Of Aphrodite - Rum Barrel Aged\\
5  & Samuel Smith's Oatmeal Stout            & Kolsch                                  & Great Lakes Grassroots Ale\\
6  & Blind Pig IPA                           & Drax Beer                               & Blue Dot Double India Pale Ale\\
7  & Stone Ruination IPA                     & A Little Sumpin' Extra!~Ale             & Calistoga Wheat\\
8  & Schneider Aventinus                     & Odell Cutthroat Porter                  & Dogwood Decadent Ale\\
9  & The Abyss                               & Miner's Daughter Oatmeal Stout          & Traquair Jacobite\\
10 & Northern Hemisphere Harvest Wet Hop Ale & Rare Bourbon County Stout               & Cantillon Gueuze 100\% Lambic\\
\bottomrule
%
% Founders KBS (Kentucky Breakfast Stout)
% Samuel Smith's Nut Brown Ale
% Two Hearted Ale
% Bell's Hopslam Ale
% Kolsch
% Drax Beer
% A Little Sumpin' Extra!~Ale
% Odell Cutthroat Porter
% Miner's Daughter Oatmeal Stout
% Rare Bourbon County Stout
%
% Founders KBS (Kentucky Breakfast Stout)
% Samuel Smith's Nut Brown Ale
% Salvator Doppel Bock
% Oil Of Aphrodite - Rum Barrel Aged
% Great Lakes Grassroots Ale
% Blue Dot Double India Pale Ale
% Calistoga Wheat
% Dogwood Decadent Ale
% Traquair Jacobite
% Cantillon Gueuze 100\% Lambic
%
%
\end{tabularx}
\end{minipage}
\end{table}
%\done{Table caption above or below? I think usually above but check. Always above}

%\done{scan lstlistnigs formatting}

Note that the ideal value of $\lambda$ depends on a variety of factors, including our specific choices of compatibility and diversity functions.\footnote{Which may not even be on the same scale: in our case one is a rating (in the range $\lbrack 1, 5 \rbrack$) and the other is a cosine similarity (in the range $\lbrack -1, 1 \rbrack$).} The solution can also be sensitive to hyperparameters (e.g.~the number of factors and how strongly we regularize). In practice the optimal amount of diversity may simply be guided by what `looks right.'
%\done{style. Seems fine?}



%\section{Case Studies on Recommendation Diversity}
\section{Case Studies on Recommendation and Consumption Diversity}
\label{sec:caseStudiesDiverse}
%\done{This isn't really what the section is about, but maybe okay. fine}

In \Cref{subsec:empirical}, we saw an empirical study of diversity on \emph{YouTube}, which argued that recommender systems led to diverse views, though this analysis was limited in that the point of comparison was a popularity-based alternative (which 
%is almost certainly 
is likely
not diversity-inducing). Below we explore a few additional case-studies that study diversity within the context of music (\cref{sec:spotify}) and news (\cref{subsec:filter}) recommendation, attempting to characterize users in terms of their consumption patterns, and potentially how to guide users to more diverse content.

\subsection{Diversity on Spotify}
\label{sec:spotify}

%In \Cref{sec_?}, we saw an empirical study of diversity on \emph{YouTube}, which argued that recommender systems led to diverse views, though this analysis was limited in that the point of comparison was a popularity-based alternative (which is almost certainly not diversity-inducing).

\cite{anderson2020algorithmic} 
%tries to 
sought to empirically
study the effect that recommendation algorithms have on diversity,
%more directly.
and more critically 
%tries 
to understand how different types of users respond to diverse recommendations.
%\done{bad intro, too much "tries to" etc.}

%\done{tense of words (plurals) after \citet}

The paper considers the listening patterns of around 100 million users on \emph{Spotify}. Unlike (e.g.)~\cite{fleder2009blockbuster}, which defined `diversity' in terms of the Gini coefficient (i.e.,~the statistical dispersion of \emph{which} items get consumed), \cite{anderson2020algorithmic} define diversity in terms of song representations, i.e.,~essentially the $\gamma_i$ values in a recommender system.\footnote{Of course, this version of `diversity' has its own limitations, as it assumes that the learned latent space accurately captures semantic diversity among items.
%\done{Is this limitation addressed in the paper? Fine}
}

The specific embeddings $\gamma_i$ on \emph{Spotify} are estimated using an \emph{item2vec}-like method (as we saw in \Cref{sec:item2Vec}). The \emph{musical diversity} of a user $u$'s listening activity is defined in terms of a score which they term \emph{generalist-specialist} (or $\mathit{GS}$), following previous work \citep{waller2019generalists}. Specifically, we start by defining the centroid of a user's listening history (which we'll term $\gamma_u$) as
\begin{equation}
    \gamma_u = \frac{1}{|H|} \sum_{j = 1}^{|H|} \gamma_{H_j},
\end{equation}
where $H$ is a list of songs in the user's \emph{listening history}, with repetition, such that repeated listens will count more to the average. Then the $\mathit{GS}$-score is defined as the average cosine similarity between the user representation $\gamma_u$ and the items they listen to:
\begin{equation}
    \mathit{GS}(u) = \frac{1}{|H|} \sum_{j = 1}^{|H|} \frac{\gamma_{H_j}\cdot \gamma_u}{\| \gamma_{H_j} \| \| \gamma_u \|}.
\end{equation}
Intuitively, \emph{specialists} (high $\mathit{GS}(u)$) tend to have songs $\gamma_i$ in their listening history oriented primarily in a certain direction;  \emph{generalists} (low $\mathit{GS}(u)$) do not, ostensibly corresponding to a broader range of prefrences.
%has a broad range of preferences, such that songs $\gamma_i$ in their history do not tend to `point' in any specific direction; 

Part of the analysis in \cite{anderson2020algorithmic} is a study of the relationship between diversity (as measured by $\mathit{GS}(u)$) and various other attributes. For example, less active users tend to be specialists (high $\mathit{GS}(u)$), generalist users are less likely to abandon the system (`churn'), and more likely to subscribe to the `premium' version of the product.

However the main feature in the analysis is 
%the effect that \emph{recommendations} have on diversity. 
to study the relationship between recommendations and diversity, and in particular how generalists and specialists respond differently to algorithmic recommendations.
This is measured experimentally by exposing real users of \emph{Spotify} to different recommendation conditions. Three types of recommender system are used: one which merely ranks songs (within a specific predefined subgenre) by popularity; one which is a simple relevance ranker based on user-to-item similarity (essentially a form of heuristic recommendation); and one which is a learned recommender specifically trained to maximize the probability that a user will listen to a song to completion.

First, compared to popularity, recommendation approaches lead to a substantial increase in the number of songs streamed for both groups (they also lead to an increase in the number of songs \emph{skipped}, but this is more than made up for in additional streams). That is, users could be said to be more \emph{engaged} when interacting with recommendations compared to a popularity baseline.
Second, the benefit of recommendations appears across both groups (generalists and specialists), though is significantly more pronounced for specialists: this aligns with the paper's hypothesis in the sense that specialists are more sensitive to songs matching their personal relevance criteria.
%
Finally, the learned ranker confers a slight additional benefit over the relevance ranker, though the benefit is surprisingly modest, indicating that simple relevance ranking is sufficient in this context.

% \paragraph{Guiding users to more diverse content}
\subsubsection{Guiding users to more diverse content}
%
In a follow-up paper, \cite{hansen2021shifting}
%https://dl.acm.org/doi/pdf/10.1145/3437963.3441775
also consider consumption patterns on \emph{Spotify}, and broadly explore the trade-offs involved in terms of algorithmic choices, diversity methods, and user satisfaction. They note (as we've seen throughout this chapter) that several ranking approaches bias recommendations toward highly-popular content that closely resemble interactions from users' histories; like \cite{anderson2020algorithmic} they also find evidence that users can in many cases be satisfied by recommendations that are more diverse and less popular. 

Several diversification techniques are explored, each of which essentially attempts to trade-off a relevance versus a diversity term. \cite{hansen2021shifting} explore the merits of each; they broadly favor reinforcement learning-based approaches as a means of swaying users toward diverse content, but note the difficulties involved in productionalizing such systems.

%\done{could say more but is okay}

\subsection{Filter Bubbles and Online News Consumption}
\label{subsec:filter}

Much of the discussion of `filter bubbles' has been in the context of online news, where concerns generally center around whether recommender systems (or more simply, algorithmic ranking techniques) will limit the ideological diversity of content users consume.

\cite{bakshy2015exposure} study the extent to which users on \emph{Facebook} tend to consume news that conforms to their political ideology. The analysis begins by training a supervised learning system to label news articles as `liberal,' `conservative,' or `neutral,' based on shares by users who volunteer their political affiliation as part of their profile.
%Exposure to ideologically diverse news and opinion on Facebook (or somewhere else)

The main questions of interest center around the extent to which users are exposed to (or choose to interact with) content that is aligned with their own ideology versus content which is `cross-cutting.' `Exposure' refers to algorithmic feed ranking 
%choosing to surface 
surfacing
the content, whereas `interaction' refers to a user's choice to click on exposed content.

There are many confounding factors in such an analysis, which the study attempts to control for. For example, users' social networks are primarily composed of friends who share a common ideology, so naturally the content users could potentially be exposed to via their social network is predominantly not cross-cutting. Likewise, users' tendency to interact with (i.e.,~click on) content is confounded by the fact that the feed ranker already factors in click probability when determining which (and how prominently) content is surfaced to the user in the first place.

After attempting to control for these effects, the study's main findings are that algorithmic ranking indeed exposes users to less ideologically diverse news than would be expected by the ideological makeup of their social group, however users interact with ideologically diverse content even less then they are exposed to it; from this, the authors conclude that individual choice plays the largest role in users' exposure to content that is ideologically homogeneous.

However, the argument above does not refute the possibility of a `filter bubble' of online news consumption, it merely argues that its primary cause (in the case of \emph{Facebook}) is not necessarily algorithmic.
%(similar to the argument made by \cite{nguyen2014exploring} above).

%\done{check subsubsections for capitals}

% \paragraph{Filter bubbles in online news consumption}
%\subsubsection{Filter Bubbles in Online News Consumption}
\subsubsection{Diversity across consumption channels}
\cite{flaxman2016filter}
%Filter bubbles, echo chambers, and online news consumption
%\done{Write this section, see if this fits in between}
sought to measure the impact that new forms of consumption (news aggregators, social recommendation, etc.)~have on the diversity and extremity of news consumption.
Their analysis is based on 50,000 users who have the \emph{Bing Toolbar} plugin installed, which allows for their interaction patterns to be tracked.
%bing, 50,000

The main goal of the paper is to measure how diversity differs across users who interact with news via different consumption channels. \emph{Direct} consumption (directly visiting a URL or accessing a bookmark); \emph{aggregator}-based consumption (specifically, visiting links from \emph{Google News} in the case of their study); \emph{social} consumption (consumption from \emph{Facebook}, \emph{Twitter}, or e-mail); and \emph{search} (consumption via queries on \emph{Google}, \emph{Bing}, and \emph{Yahoo} search).
%
%consumption channels: direct, aggregator, social, and search.
%
Various mundane aspects must also be dealt with to determine which links correspond to news articles, versus opinion pieces, etc. Much of the data collection effort centers around determining the ideological stance of articles and publishers (for which there is no ground truth); the ideological stance of individuals is then measured in terms of the articles they consume.

Consumption from these four sources is measured in various ways. First, \emph{segregation} measures the average distance in polarity scores between two randomly chosen users who consume news via the same channel. These scores reveal that consumers of opinion pieces are more segregated than consumers of news across all four channels, with social media and search traffic being the most segregated. This arguably aligns with the concept of a filter bubble, to the extent that these media lead to ideologically more segregated groups.

Counter to this result, they also find that users who consume media from search engines and social media \emph{also} experience higher exposure to ideologically diverse news (as opposed to users who consume news from aggregators or via direct consumption). \cite{flaxman2016filter} argue that most online news consumption mimics patterns of traditional media consumption, with users predominately visiting homepages of their preferred mainstream outlets; ultimately, to the extent that `filter bubbles' exist in online news, their dynamics are not as straightforward as they might first appear.

%measure political stance of publishers



% \paragraph{Filter bubbles on Google News}
\subsubsection{Filter bubbles on Google News}

%\cite{haim2018burst}
%Burst of the Filter Bubble?
%Effects of personalization on the diversity of Google News
\cite{haim2018burst} conducted an exploratory study of recommendation on \emph{Google News}, in order to determine the effects of personalization on content diversity. Like \cite{bakshy2015exposure}, they broadly argue that the effects of filter bubbles are somewhat overstated, or otherwise that the patterns of bias in recommendations are not the same as what is anecdotally understood to be a `filter bubble.'

They conduct two studies, to look at `explicit' and `implicit' personalization. Both are based on empirical observation of the actual news recommendations provided by \emph{Google News}, sampled from several synthetic user accounts. Recommendations are then compared against `traditional' (i.e.,~non-personalized, curated) news sources in terms of topic and content diversity.

In the `explicit' setting, they make use of a \emph{Google News} feature that allows users to specify the types of news they are interested in, among a set of broad categories (e.g.~sports, entertainment, politics). Annotators then labeled recommended articles according to these categories in order to quantify the alignment between the explicit preferences and the recommended articles.

The first finding is simply that \emph{Google News} does indeed respect users' explicit preferences, in the sense that the proportion of recommended articles matching the desired topic far exceeds their proportion in a non-personalized setting.

\cite{haim2018burst} also evaluate recommendations in terms of source diversity (i.e.,~in terms of the original news sources that \emph{Google News} aggregates). Here, they find surprisingly that a few somewhat niche news sources dominate recommendations, whereas more mainstream sources are underrepresented; this result is relatively consistent across each of the personalized accounts.

In the `implicit' setting, \cite{haim2018burst} made use of several social media accounts, corresponding to users with specified (but synthetic) demographics and preferences (such as a marketing manager, an elderly conservative, etc.). Each of these simulated agents then interacts with social media (liking articles on \emph{Facebook}, \emph{Google+}, etc.), after which their \emph{Google News} recommendations are compared.

The main conclusion of this second study is simply that implicit personalization has little effect on the recommended results (though there is evidence that some results are indeed personalized).

Ultimately while both \cite{bakshy2015exposure} and \cite{haim2018burst} argue against a `filter bubble' as such, both point to potential issues of bias in recommendations; \cite{bakshy2015exposure} suggest that recommendations do indeed present an overall more biased perspective compared to users' broader social groups, while \cite{haim2018burst} show that certain niche sources tend to be over-represented in news recommendations.



% Blockbuster culture’s next rise or fall:
% The impact of recommender systems on sales diversity (already covered)


% Breaking the filter bubble: democracy and design
% (maybe)

% Cover:
% Exploring the filter bubble: the effect of using recommender systems on content diversity \cite{nguyen2014exploring}




%\section{Other Metrics Beyond Accuracy: Novelty, Discovery, and Serendipity}
\section{Other Metrics Beyond Accuracy}
\label{sec:otherMetrics}
%\done{Don't like section title, as diversity is covered in other subsections above and below. CHANGED}
%\done{probably could go after the DPP section}
%\done{But could still go after code and case studies on diversity. Moved}

So far, we have considered diversity in terms of the trade-off between recommending the highest relevance items (e.g.~highest click probability) while ensuring that recommended items are not too similar to each other.
Other than relevance, diversity among items is only one desirable characteristic to trade-off.

Besides relevance and diversity, other desirable features of a recommendation list might include:
\begin{itemize}
    \item Items should be \emph{novel} to the user, i.e.,~the recommender system should balance \emph{discovery} of new items against recommending items with high interaction probability, but which are already known to the user.
    \item Rather than being internally diverse, we might have goals such as mutual compatibility among items (e.g.~\cite{hao2020p}).
    %\done{citation? tong paper maybe?}
    \item Recommended items should have good \emph{coverage}, i.e.,~they should represent a broad range of categories or features; or they should be \emph{balanced}, in terms of matching the category distribution from the user's history.
    %(as in \cref{subsec:calibration}).
    %\done{check that this is after this. Removed ref}
    %Coverage and balance
    \item Other goals could be more nebulous, such as perceived unexpectedness, serendipity, or overall user satisfaction.
\end{itemize}

\cite{kaminskas2016diversity} broadly survey these alternate optimization criteria for recommender systems, focusing in particular on diversity, serendipity, novelty, and coverage. We briefly survey some of their main findings (as well as more recent work) below.
%\done{What do they say about diversity? Adding a bit below}

%\done{items in exercises sometimes hyphens. Why? Don't see any?}

% \subsection{Further Perspectives on Diversity}
%\done{move out of this section into diversity section? Or drop?}
%\done{removed section heading, make sure text looks okay. Looks okay}
Many of the approaches to diverse recommendation discussed in \cite{kaminskas2016diversity} 
%are focused on 
are
\emph{re-ranking} strategies, similar to maximal marginal relevance (\cref{sec:MMR}) and other techniques we've discussed so far.
%\done{what else do they have to say? There's enough here}
They also discuss other settings where diversity might be desirable, such as conversational recommendation 
%(\cite{kelly2006enhancing}, see \Cref{sec_?}),
(\cref{subsec:convRec})
%\done{keep ref only if it's covered}
and the relationship to more traditional work in `portfolio optimization' from information retrieval \citep{markowitz1968portfolio}.
%\done{Fine I think}


%  \cite{vargas2011rank} and \cite{vargas2011new} studied the relationship between novelty and diversity when making recommendations, and presented several metrics to evaluate existing systems in terms of such characteristics.
%\done{drop?}

We discuss a selection of such approaches below.
%in this section, which we summarize
%We summarize a selection of beyond-accuracy metrics
%covered in this section are summarized 
%in \Cref{tab:divTech}.
%(which also includes a few techniques from \Cref{sec:diversification}).
%\done{add others (e.g. new ones). Is fine}




\subsection{Serendipity}
\label{subsec:serendipity}
%\done{Don't like the subsections/section, keep working on. Okay for now}

%\done{fix that one box that doesn't look right}

Various attempts have been made to define `serendipity' in the context of recommendations. \cite{kaminskas2016diversity} start with the core property of `surprise' (i.e.,~recommendations should be different from one's expectations); \cite{kotkov2018investigating} state that serendipity should be a combination of relevance, novelty and unexpectedness.

Each of these competing elements is difficult to define precisely, and some (like a recommendation being `surprising') are likely subjective. Below we discuss a few specific attempts to incorporate serendipity into recommendations, and to understand what it means to users in practice.

% \cite{ge2010beyond}




%\done{Would be nice to make these paragraphs and upper paragraph subsection. This looks okay}
\subsubsection{Serendipity in music recommendation}

\cite{zhang2012auralist} consider how music recommendations can be improved by balancing goals of accuracy, diversity, novelty, and serendipity. Their specific approach combines many of the ideas we've seen already:
%\done{check ordering. Fine}
\emph{diversity} is measured in terms of the cosine similarity between items in a recommendation list (as in \cref{sec:cosine}); \emph{novelty} or `unexpectedness' is defined in terms of overall item popularity (as in \cref{susbsec:rerank}); serendipity (or `unserendipity,' since low values mean high serendipity) is defined using a novel function, which essentially measures how similar recommended items are to those in the user's interaction history:
\begin{equation}
  \text{Unserendipity} = \frac{1}{|U|}\sum_{u \in U}
  \frac{1}{|I_u|}\sum_{i \in I_u} \sum_{j \in R_u} \frac{\mathrm{Cos}(i,j)}{|R_u|},
  \label{eq:auralistSerendipity}
\end{equation}
%\done{Formatting of Cos}
%\done{check notation. Looks fine}
where $R_u$ is a set of items recommended to the user (and $I_u$ is the item history for user $u$). This measure takes a low value if recommended items are on average different from those that appeared in users' histories.
%\done{last sentence somewhat redundant. Fine}

%\done{add to method table}

Given these three metrics (diversity, novelty, and serendipity), \cite{zhang2012auralist} seek recommendation techniques that can optimize them without overly compromising accuracy. While metrics such as that of \Cref{eq:auralistSerendipity} cannot straightforwardly be incorporated into the optimization scheme directly, various models are designed 
to ensure that recommendations are topically diverse or belong to distinct clusters. Quantitatively, \cite{zhang2012auralist} study the trade-off between accuracy, diversity, novelty, and serendipity under different configurations of this model. They also conduct a user study to evaluate the qualitative aspects of the model, revealing that subjective notions of `subjectivity' and `usefulness' can be improved without overly harming user enjoyment.


\subsubsection{Investigating serendipity via user studies}

Given the ambiguous nature of the precise definition of serendipity, \cite{kotkov2018investigating} attempted to assess what it means to users via a survey. 
%\cite{kotkov2018investigating} 
They
survey commonly proposed notions for diversity, ranging from 
%items that are simply unfamiliar to a user \cite{kapoor2015like}, to 
items the user simply hasn't heard of, didn't expect to be recommended, or are highly dissimilar to what they usually consume.
%
%
They found that serendipitous recommendations are effective in broadening user preferences, though do not have a significant impact in terms of satisfaction. They 
%particularly 
investigate the key feature of \emph{unexpectedness} and its different definitions in the literature (a few of which we study in \Cref{subsec:unexpected}). 
%In particular they find that some notions (e.g.~didn't expect to be relevant) 
In particular they find that items a user didn't expect to be relevant (or didn't expect to like) tend to have a negative effect in terms of user satisfaction, and are not as effective at broadening preferences compared to other notions of unexpectedness.
%They find that several definitions of unexpectedness can broaden preferences, though 
%Definitions range from a user expecting an item to be relevant
%An item can be `unexpected' in terms of a user's expectation that it will be relevant, recommended, or different from what they would normally consume or find on their own. 

%\done{Another paragraph about this paper}

%\done{Maybe. Somewhat too complex. Probably couple of pages to write. No}
% Rank and Relevance
% in Novelty and Diversity Metrics for Recommender Systems \cite{vargas2011rank}

% New Approaches to Diversity and Novelty in
% Recommender Systems \cite{vargas2011new}

%The Impacts of Item Features and User Characteristics on Users' Perceived Serendipity of Recommendations
\cite{wang2020impacts} also studied serendipity via a large-scale user study, asking users directly what type of item features contributed to the perceived serendipity of a recommendation. They found that while perceived serendipity is positively influenced by lower popularity (similar to the principle from our simple diversification technique in \Cref{eq:adomDiv}), characteristics such as being from a distant category, or separated temporally from similar recommendations, do not contribute to perceived serendipity. 
%The latter finding 
That 
`serendipitous'
results can be close in time and category compared to previous interactions is somewhat surprising, 
%as a serendipity goal, 
given our efforts to define serendipity above.
%They 
\cite{wang2020impacts}
hypothesize that this is due to the rapidly evolving nature of user preferences, where distant interactions rapidly lose meaning. 
They 
%\cite{wang2020impacts}
also find that (perceived) serendipity is not static across user demographics (older and/or male users tend to perceive recommendations as more serendipitous, younger users are more sensitive to item popularity, etc.); one hypothesis is that this relates to overall familiarity with the particular shopping platform.




%Investigating serendipity in recommender systems based on real user feedback
%https://www.researchgate.net/publication/326164444_Investigating_serendipity_in_recommender_systems_based_on_real_user_feedback









\subsection{Unexpectedness}
\label{subsec:unexpected}
%On Unexpectedness in Recommender Systems: Or How to
%Expect the Unexpected 
%
\cite{adamopoulos2014unexpectedness}
%http://ceur-ws.org/Vol-816/paper2.pdf
attempt to define the notion of `unexpectedness' as it relates to (movie) recommendations. They note that one cannot target unexpectedness in isolation, or one could trivially generate poor-quality but unexpected recommendations. As such they seek a notion of \emph{utility} that balances unexpectedness 
%must be balanced against 
against traditional metrics of recommendation
quality.
%
They define unexpectedness (for a user $u$ and item $i$) as a distance between $i$ and the set of items the user $u$ `expects' to receive. They further assume that there is some optimal value for this distance (which could be different for each user): recommendations that are too expected are uninteresting, while recommendations that are too unexpected will be regarded as irrelevant; `quality' is defined more straightforwardly in terms of ratings. 
%
%The goal is then to determine the 

Several quantities must then be determined: each user's personal tolerance for unexpectedness,
the ideal trade-off between unexpectedness and utility, and finally the definition of what is `expected.' 
For the latter \cite{adamopoulos2014unexpectedness} use a definitions based on content similarity in terms of movie attributes (movies with similar attributes are `expected').
%\done{mention where that it's movies? Mentioned}
The goal of \cite{adamopoulos2014unexpectedness} is not to fit these values (which are largely subjective quantities) but to evaluate the performance of different recommendation approaches under various hypothetical scenarios. The most promising finding is that optimizing this type of joint utility need not harm performance compared to methods that target quality exclusively.
%\done{Not a great summary. Fine}

%\done{Add above to table (and change name of table). No}

%PURS: Personalized Unexpected Recommender System for Improving User Satisfaction 
\cite{li2020purs} define unexpectedness in terms of clustering: users' 
%previously
%-consumed items 
consumption histories
%consumption
($\gamma_i$) are clustered in latent space. An `unexpected' item $j$ is one that is not close to any cluster. To prevent the model from simply recommending 
%extremely 
outlying or
`fringe' items (which might trivially maximize unexpectedness), a unimodal distribution over desired utility values is introduced. Unexpectedness is then balanced against utility, weighted according to a personalized factor measuring the extent to which each
%particular 
user tends to 
%respond positively to observations with high unexpectendes
favor unexpectedness over relevance.
%Maybe, similar to the tf-idf-like paper, maybe talk about in same place


% \subsection{Discovery}

% Discovery-oriented Collaborative Filtering for Improving User Satisfaction (not sure)


\subsection{Calibration}
\label{subsec:calibration}

A related notion to diversity is that of \emph{calibration} of predictions or recommendations. Whereas a diversity metric might suggest (for example) that we should expose users to a wide distribution of recommendations, which potentially span beyond their explicit preferences, \emph{calibration} refers to the idea that recommendations should be made \emph{in proportion} to expressed preferences. For instance, if a user watches 40\% sci-fi movies and 60\% romantic comedies, they should not exclusively be recommended romantic comedies, as might happen when na\"ively recommending by maximizing compatibility.
%nearest neighbors or maximizing inner products.

\cite{steck2018calibrated} introduce such a notion of \emph{calibrated recommendations}, in the context of movie recommendations on \emph{Netflix}. Their work discusses metrics to assess calibration, as well as methods to calibrate the outputs of an existing recommender system.

Their notion of calibration operates over a pre-defined set of item \emph{genres}, described using a stochastic genre vector $p(g|i)$ (e.g.~a movie might be categorized as 80\% `action' and 20\% `sci-fi'); this could potentially be adapted to other attributes toward which one desired calibration. The basic idea behind a calibration metric is then that the distribution of genres $g$ among a 
user's history $i \in I_u$
%\done{What is H? Just I_u? Think so. Changed}
should match the distribution of \emph{recommended} items $i \in R_u$. 
%$i \in I$
The two terms are defined (respectively) as:
\begin{align}
\text{historical:}\quad p(g|u) & = \frac{\sum_{i \in  I_u} w_{u,i} \cdot p(g|i)}{\sum_{i \in  I_u} w_{u,i}}\\
\text{recommended:}\quad q(g|u) & = \frac{\sum_{i \in R_u} w_{r(i)} \cdot p(g|i)}{\sum_{i \in R_u} w_{r(i)}}.
%\text{recommended:}\quad q(g|u) & = \frac{\sum_{i \in r(u)} w_{r(i)} \cdot p(g|i)}{\sum_{i \in r(u)} w_{r(i)}}.
\end{align}
%\done{not clear enough what these functions are. I is bad notation}
Both expressions include a `weighting' term $w$. In the case of the historical distribution $w_{u,i}$ might weight items according to recency (for example), or for the case of recommendations $w_{r(i)}$ might weight recommendations according to their position in a list (i.e.,~their ranking); either term could also be ignored.

Now, the goal is to generate a set of recommended items 
%$I$ 
$R_u$
such that the two distributions should match closely. The difference between the two distributions can be measured by (e.g.)~the Kullback Liebler divergence:
\begin{equation}
    \text{KL}(p,q) = \sum_{g} p(g|u)\log \frac{p(g|u)}{q(g|u)}.
\end{equation}
Of course, 
in addition to being well-calibrated,
recommendations should also be highly compatible according to the recommender system itself. This is achieved in \cite{steck2018calibrated} with a simple expression that trades off recommendation utility and calibration (via a trade-off hyperparameter $\lambda$):
\begin{equation}
  R_u = \argmax_{R} \underbrace{(1 - \lambda) \cdot \sum_{i \in R} f(u,i)}_{\text{compatibility}} - \underbrace{\lambda \cdot \text{KL}(p, q(R))}_{\text{calibration}}.
  \label{eq:calibrated}
\end{equation}
%\done{equation formatting. Is fine}
\cite{steck2018calibrated} note that the above is a hard combinatorial optimization problem, but can be approximated greedily (with a certain optimality guarantee), by iteratively adding one item at a time to $R$ so as to optimize the above criterion until the desired number of items is reached.

An appealing property of this approach is that it can be applied in a purely post-hoc fashion to the outputs of any recommender system that associates scores between users and items. The experiments in \cite{steck2018calibrated} show (by varying $\lambda$ in \Cref{eq:calibrated}) that a reasonable degree of calibration can be achieved with minimal loss in recommendation utility.

%Calibrated recommendations: https://dl.acm.org/doi/10.1145/3240323.3240372



%\done{Figure needs to go elsewhere. Already moved I guess}

% \section{Clickbait, Fake News, and Review Fraud}

%  Review fraud:
% BIRDNEST: Bayesian Inference for Ratings-Fraud Detection

% \section{Optimizing Short-Term versus Long-Trem Goals}









\section{Fairness}
\label{sec:fairness}


% Bias Disparity in Recommendation Systems
%\done{add this somewhere? No}

%Paper: Beyond Parity: Fairness Objectives for Collaborative Filtering (http://papers.nips.cc/paper/6885-beyond-parity-fairness-objectives-for-collaborative-filtering.pdf)

\emph{Fairness} in machine learning is often defined in terms of predictions and protected characteristics. For example, when building a classifier to aid in hiring decisions, we might be interested in ensuring that men and women are ranked as `qualified' at approximately the same rate. Or, a system for predicting recidivism should not be biased against individuals of a certain race \citep{chouldechova2017fair}. 

Typically, we might 
%want the predictions of 
desire that the outputs of
our classifier $f(x_i)$ 
%to be independent of a 
do not depend on some
protected feature $x_{i,f}$ (indicating race, gender, etc.).
%
Some common definitions include, for example, \emph{demographic parity}, which states that the probability of a positive prediction (e.g.~being ranked as `qualified') should be the same whether one has the protected feature or not:
\begin{equation}
    p(f(x_i) = 1 | x_{i,f} = 1) = p(f(x_i) = 1 | x_{i,f} = 0).
    %\done{Check definition against some paper. Looks fine}
    \label{eq:demographic}
\end{equation}
The related notion of \emph{equal opportunity} allows for the possibility that the target variable depends on the protected feature,
%\emph{among qualified candidates} (i.e.,~$y_i = 1$)
and states that \emph{among the qualified} ($y_i = 1$) \emph{or unqualified} ($y_i = 0$)
%\done{y_i = 0 or y_i = 1. Looks okay}
\emph{individuals}, the probability of a positive prediction should be the same whether or not one has the protected features:
%Equal opportunity:
%\done{descriptions of both. Seems adequate}
\begin{equation}
    p(f(x_i) = 1 | x_{i,f} = 1, y_i = y) = p(f(x_i) = 1 | x_{i,f} = 0, y_i = y).
\end{equation}
These are just two examples out of dozens of possible notions of fairness that may be of interest, see e.g.~\cite{mehrabi2019survey} for a comprehensive survey.

A classifier may violate the above rules for a variety of reasons. For example, the training data may exhibit historical bias against a certain group; or, a classifier trained on highly imbalanced data may simply make inaccurate (or imbalanced) predictions for groups that are poorly represented in the data (we already saw a simple example of this for an imbalanced dataset in \Cref{sec:type1type2}).  Several machine learning techniques have been proposed to mitigate unfairness in such scenarios, for example by pre-processing the biased data \citep{kamiran2009classifying}, or by altering the classifier itself \citep{zafar2017fairness}.
%, or post-process the outputs \cite{me}.

%\cite{pedreshi2008discrimination}

In the contexts of recommendations and personalized predictions, one may have slightly different definitions and goals in terms of building a `fair' model. \cite{yao2017beyond} attempt to adapt notions of fairness to personalized recommendation contexts. %The paper considers 
They consider
a running example of recommendation in the context of education, where course evaluations in Computer Science (for example) may primarily represent the preferences of the predominantly male population; models trained on such data 
(or even simple statistics or heuristics based on popularity, etc.)~%
%(or even simple statistics compiled from the data)
may merely reflect the preferences or activities of the majority group.
%\done{incomplete sentence}

\cite{yao2017beyond} introduce several metrics of fairness with respect to the outputs of a recommender system, and show that these metrics can be straightforwardly incorporated into the training objective (meaning that the model can be discouraged from making unfair predictions). Their metrics are defined by dividing users into groups $g_u$ and items into `genres' or attributes $h_i$; user groups are assumed to be binary, though in the studied case of gender $g_u$ can simply be divided into the over-represented group (male) and the under-represented group (non-male).
%\done{should specify that g is male, not g is not male. Clear enough I think}

% Briefly, the fairness metrics they define include:
% \begin{description}
  \emph{Value Unfairness} measures the extent to which one group tends to have their ratings over- or under-predicted compared to the other:
  \begin{equation}
      U_{\text{val}} = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \Bigl( \underbrace{\mathbb{E}_g[y]_i}_{\hspace{-10mm}\mathrlap{\text{expected \emph{prediction} for group $g$ on item $i$}}} - \overbrace{\mathbb{E}_g[r]_i}^{\hspace{-10mm}\mathrlap{\text{average \emph{rating} for group $g$ on item $i$}}} \Bigr) - \Bigl( \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i\Bigr) \Bigr|.
      \label{eq:valueUnfairness}
  \end{equation}
  Note that since both sides take expectations (or averages), the measure is invariant to size differences between the two groups.
  
  Value unfairness could occur in a latent factor model (like that of \Cref{eq:MFscore}) if, for example, predictions are dominated via the bias terms $\beta_i$; in a model in which one group is over-represented, the bias terms may essentially reflect the preferences only of the over-represented group.
  %\done{Tenses in above sound funny. Seems okay}
% \end{description}

\emph{Absolute Unfairness} replaces 
%the 
differences in expectation (from \cref{eq:valueUnfairness}) with absolute values:
\begin{equation}
      U_{\text{abs}} = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \Bigl| \mathbb{E}_g[y]_i - \mathbb{E}_g[r]_i \Bigr| - \Bigl| \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i\Bigr| \Bigr|.
      \label{eq:absoluteUnfairness}
  \end{equation}
  Following this change, absolute unfairness now captures the extent to which one group has their ratings \emph{mispredicted} (in an absolute sense) more than the other. This essentially measures a difference in the system's \emph{utility} between the two groups, in the sense that if one group routinely receives recommendations with high error then the system is unlikely to be useful to them.
  %\done{poorly phrased. Okay}
  
  Next, \cite{yao2017beyond} define \emph{under-} and \emph{over-estimation} unfairness to assess the model's tendency 
  %of the model 
  to either under- or over-predict the true ratings:%
  \begin{align}
      U_{\text{under}} & = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \max\lbrace 0,  \mathbb{E}_g[r]_i - \mathbb{E}_g[y]_i \rbrace - \max\lbrace 0, \mathbb{E}_{\neg g}[r]_i - \mathbb{E}_{\neg g}[y]_i \rbrace \Bigr|,\\
      U_{\text{over}} & = \frac{1}{|I|} \sum_{i = 1}^{|I|} \Bigl| \max\lbrace 0,  \mathbb{E}_g[y]_i - \mathbb{E}_g[r]_i \rbrace - \max\lbrace 0, \mathbb{E}_{\neg g}[y]_i - \mathbb{E}_{\neg g}[r]_i \rbrace \Bigr|.
      \label{eq:overunder}
\end{align}
%\done{Check 0s in equations involving max (Rebecca Hu sent me note about error). All look fine}
%\done{doesn't render right, comma is blocked. Looks fine}
These definitions are somewhat analogous to related concepts we saw when evaluating ranking models in \Cref{sec:evaluatingClassifiers}; consistently underpredicting is analogous to having low recall (failing to retrieve relevant items), whereas overpredicting is analogous to having low precision (retrieving items that are not relevant). Both can potentially reduce the utility of the recommender system for one of the groups.
%\done{Precision and recall don't feel quite right... Seem okay}

Ultimately, \cite{yao2017beyond} show that each of the above metrics can be incorporated into recommender systems of the form given in
\Cref{eq:MFObjective}.
That is, they can be combined via a trade-off term so that the model is accurate while minimizing unfairness, e.g.:
\begin{equation}
    \frac{1}{|\mathcal{T}|} \sum_{(u,i) \in \mathcal{T}} \underbrace{(\alpha + \beta_i + \beta_u + \gamma_i + \gamma_u - R_{u,i})^2}_{\vphantom{ly}\text{accuracy}} + \lambda \underbrace{\vphantom{\beta_u\gamma_uR_{u,i}}U_{\text{abs}}}_{\mathclap{\vphantom{ly}\text{(absolute) fairness}}}.
    \label{eq:fairnessTradeOff}
    %\done{Might fix alignment of underbraces}
\end{equation}
Optimization remains straightforward, as each fairness metric is differentiable with respect to the model parameters.
%\footnote{Or at least, has a subderivative in the case of absolute value functions.}
The main finding of the paper is then that fairness metrics can optimized while paying only a minimal price in terms of overall model accuracy.
%\done{should extend the above argument to demonstrate that these types of unfairness do actually exist in data!}

In addition to presenting the above objectives, \cite{yao2017beyond} also show that real data do in fact exhibit biases with respect to the above metrics. They do so using data from \emph{MovieLens} across different genres where women or men are over-represented; finally they show that such biases can be mitigated using the above techniques.

%http://www.ec.tuwien.ac.at/~dimitris/research/recsys-fairness.html



\subsection{Multisided Fairness}
\label{sec:multisided}

A separate attempt to introduce fairness metrics into recommendation problems is described in \cite{burke2017multisided}. In comparison to the fairness metrics defined above, the main difference is to consider fairness from both the perspective of users of the system (`consumers') as well as content providers (`producers'). As a motivating example, they consider a hypothetical recommendation scenario on the microfinancing website \emph{Kiva.org}, where 
%there may be an desire 
it may be desirable
%to ensure 
that proposals
%\done{is that the right word? Removed}
from different businesses receive somewhat balanced representation among recommendations. More broadly, this is an instance of recommendation in a `matchmaking' setting where both sides (in this case, users and businesses) are being matched to each other; in such cases, fairness should not be defined in terms of one `side' only, but should consider the needs of both types of stakeholders. Other examples are cited including online advertising, the sharing economy, or online dating \citep{pizzato2010recon}.

To achieve this notion of fairness, they consider fairness separately from the perspective of consumers and producers, which they term $C$- and $P$-fairness. Following these definitions, the fairness metrics we studied above
%\done{check}
would be examples of $C$-fairness. \cite{burke2017multisided} note that $C$- and $P$-fairness are not merely symmetric definitions, and that $P$-fairness may have requirements not encountered when studying $C$-fairness, such as in the examples above. For example, in a product recommendation setting, if we wanted to encourage sales diversity,
%(as we considered in \Cref{sec:salesDiversityAbove}, and will revisit in \Cref{sec:salesDiversityBelow}), 
the producers are \emph{passive} in the sense that they are not actively seeking out recommendations in the system.

Finally, \cite{burke2017multisided} consider settings of $\mathit{CP}$-fairness, where fairness must be considered from the perspectives of both sides simultaneously. We'll revisit examples of $P$-fairness and $\mathit{CP}$-fairness as we examine case studies of gender bias in \Cref{sec:genderCaseStudies}.

A selection of the fairness objectives from this section (as well as our case studies in \Cref{sec:genderCaseStudies}) is summarized in \Cref{tab:fairObj}.
%\done{move to case-studies section? Is fine}

%Citation: Sales diversity is useful to \cite{fleder2009blockbuster}
%Have this above somewhere too

% Multisided Fairness for Recommendation (https://arxiv.org/abs/1707.00093)

% Paper: Balanced Neighborhoods for Multi-sided Fairness in Recommendation (maybe)

% \subsection{Group-Aware Recommendation} 

% %\done{drop}

% Group-aware Recommendation?
% Paper: Top-N Group Recommendations with Fairness

% Fairness-Aware Group Recommendation with Pareto-Efficiency

\begin{table}
\begin{minipage}{\linewidth}
% note that the square brace option below is only required
% if you intend to produce a list of tables
\caption[Comparison of personalized fairness objectives.]{Comparison of personalized fairness objectives. References: \protect\cite{yao2017beyond,ekstrand2018exploring,wan2020addressing}.}
%\done{Could put the refs in the third column (all tables). Dunno. Obsolete}
\label{tab:fairObj}
%\addtolength\tabcolsep{2pt}% to stretch columns, if required
% \begin{tabularx}{\linewidth}{|XlX|}
% \textbf{Objective} & \textbf{Reference} & \textbf{Description}\\
% Value Unfairness & \cite{yao2017beyond} & Neither of two groups should have their compatibility over- or under-predicted more than the other.\\
% Absolute Unfairness & \cite{yao2017beyond} & Neither of two groups should have their compatibility mispredicted more than the other.\\
% Demographic parity among recommendations & \cite{ekstrand2018exploring} & Demographics (e.g.~author gender) should be reasonably balanced among the items being recommended.\\
% Marketing fairness & \cite{wan2020addressing} & Individuals underrepresented in marketing (e.g.~images) should not have reduced recommendation utility.\\
% \end{tabularx}
\begin{tabularx}{\linewidth}{l>{\raggedright}p{0.22\textwidth}X}
\toprule
% \textbf{Objective} & \textbf{Reference} & \textbf{Description}\\
\textbf{Ref.} & \textbf{Objective} & \textbf{Description}\\
\midrule
% \cite{yao2017beyond}
YH17 & 
Value Unfairness & Neither of two groups should have their compatibility over- or under-predicted more than the other (\cref{sec:fairness}).\\
% \cite{yao2017beyond}
YH17 & Absolute Unfairness & Neither of two groups should have their compatibility mispredicted more than the other (\cref{sec:fairness}).\\
% \cite{ekstrand2018exploring}
E18 &
Demographic parity among recommendations & Demographics (e.g.~author gender) should be reasonably balanced (or should match the training distribution) among the items being recommended (\cref{sec:genderBook}).\\
%  \cite{wan2020addressing}
W20 & 
Marketing fairness & Individuals underrepresented in marketing media (e.g.~images) should not have reduced recommendation utility (\cref{subsec:genderMarket}).\\
%\done{poorly worded. Fine}
\bottomrule
\end{tabularx}
\end{minipage}
\end{table}
%\done{add cP fairness, maybe others? No}
%\done{not referenced}
%\done{re-check formatting all tables}


\subsection{Implementing Fairness Objectives in \emph{Tensorflow}}
\label{subsec:implementFairness}
%\done{section should go here? Yes, could be section}

%As we discussed, 
Part of
the appeal of the fairness objectives 
%such as those we developed in \Cref{sec:fairness} 
we've developed in this section
is that they can straightforwardly be incorporated into the learning objectives of standard recommenders.\footnote{This is in contrast to fairness objectives in some classical settings. For example, when balancing hiring decisions with respect to gender it may not be permissible for the algorithm to base the decision on the protected attribute (see e.g.~\protect\cite{lipton2018does}).}
%\done{clunky sentence. Fine}
%
Below we'll implement `absolute unfairness' as in \Cref{sec:fairness}. We'll use data from beer reviews (this is the same data we used in \Cref{sec:binaryFeat})
%which we'll explore further in Project~\ref{proj:fairBeer}), 
which includes user gender information, and in which male users are substantially over-represented.
%\done{check reference. Looks right?}
First, we read the data, recording user gender along with each interaction:

%\done{Did we use this data in chap 2? reference}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
for d in parse('beer.json.gz'):
    g = 'user/gender' in d and d['user/gender'] == 'Male'
    u = d['user/profileName']
    i = d['beer/beerId']
    r = d['review/overall']
    if not u in userIDs: userIDs[u] = len(userIDs)
    if not i in itemIDs: itemIDs[i] = len(itemIDs)
    interactions.append((g,u,i,r))
\end{lstlisting}
\end{minipage}

Next we build some utility data structures to store interactions for each item according to group membership ($g$ and $\neg g$ for males and females):
%\done{notation. Looks right?}

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
interactionsPerItemG = defaultdict(list)
interactionsPerItemGneg = defaultdict(list)

for g,u,i,r in interactions:
    if g: interactionsPerItemG[i].append((u,r))
    else: interactionsPerItemGneg[i].append((u,r))
\end{lstlisting}
\end{minipage}

We also store item sets for each group for sampling:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
itemsG = set(interactionsPerItemG.keys())
itemsGneg = set(interactionsPerItemGneg.keys())
itemsBoth = itemsG.intersection(itemsGneg)
\end{lstlisting}
\end{minipage}

Finally, we implement the absolute (un)fairness objective. This implementation computes the fairness objective for a single item (i.e.,~one term in the summation in \Cref{eq:absoluteUnfairness}).
%\done{check}
During training, this objective can be called for a small sample of items,
%(just as we compute the accuracy over a small sample as in \Cref{sec_?}), 
and added to the accuracy term:

\noindent
\begin{minipage}{\linewidth}
\begin{lstlisting}[escapechar=\%,firstnumber=last]
def absoluteUnfairness(self, i):
    G = interactionsPerItemG[i]
    Gneg = interactionsPerItemGneg[i]
    # Compute the terms from %\Cref{eq:absoluteUnfairness}%
    rG = tf.reduce_mean(tf.convert_to_tensor([r for _,r in G])) # %$\mathbb{E}_{g}[r]_i$%
    rGneg = tf.reduce_mean(tf.convert_to_tensor([r for _,r in Gneg])) # %$\mathbb{E}_{\neg g}[r]_i$%
    pG = tf.reduce_mean(self.predictSample([userIDs[u] for u,_ in G], [itemIDs[i]]*len(G))) # %$\mathbb{E}_{g}[y]_i$%
    pGneg = tf.reduce_mean(self.predictSample([userIDs[u] for u,_ in Gneg], [itemIDs[i]]*len(Gneg))) # %$\mathbb{E}_{\neg g}[y]_i$%
    Uabs = tf.abs(tf.abs(pG - rG) - tf.abs(pGneg - rGneg))
    return self.lambFair * Uabs
\end{lstlisting}
\end{minipage}
%\done{add some comments}

\section{Case Studies on Gender Bias in Recommendation}
\label{sec:genderCaseStudies}
%\done{Could be a section with subsections, rather than a subsection with paragraphs}

Just as \cite{yao2017beyond} used gender imbalance as a motivating example to study fairness and bias with regard to under-represented groups in recommender systems, several studies have investigated specific scenarios where recommenders exhibit significant bias, or have reduced utility for a specific gender.

\subsection{Data Resampling and Popularity Bias}

\cite{ekstrand2018all} study a similar problem to \cite{yao2017beyond} (\cref{sec:fairness}),
also noting that there is a substantial \emph{utility gap} between the majority versus under-represented groups (i.e.,~closest to \emph{absolute unfairness} as in \Cref{eq:absoluteUnfairness}). Bias is reported with respect to both gender and age attributes, both of which are self-reported by uses in datasets of movies and songs (from \emph{MovieLens} \citep{harper2015movielens} and \emph{Last.FM} \citep{celma2008music}).

Unlike \cite{yao2017beyond}, 
%which corrects for this type of bias 
where this type of bias is corrected
using a joint objective that balances overall utility with unfairness (\cref{eq:fairnessTradeOff}), \cite{ekstrand2018all} use a \emph{data resampling} approach to correct for bias. This type of approach is borrowed from \cite{kamiran2009classifying} where it was used in the context of fair classification. The basic idea is to resample the data so as to achieve equal representation among groups; practically speaking this is fairly similar to the reweighting schemes we explored in \Cref{subsec:optimizeBalanced}.

\cite{ekstrand2018all} also raise the potential issue of \emph{popularity bias} in recommender systems (also discussed in \cite{bellogin2011precision}), in which algorithms that work well for popular items will generally be favored over algorithms which \emph{personalize} better (but whose performance is worse for popular items).
%\done{Clunky phrasing above}
To address this they introduce evaluation metrics that control for the effect of popularity, so that algorithms can be compared according to their degree of personalization rather than their tendency to select popular items.
%\done{Clunky phrasing here too}

\subsection{Bias and Author Gender in Book Recommendations}
\label{sec:genderBook}

%Paper: Exploring Author Gender in Book Rating and Recommendation 

\cite{ekstrand2018exploring} 
%\done{Is this the same as Exploring Author Gender in Book Rating and Recommendation, Michael D. Ekstrand and Daniel Kluver? Latter only has two authors. Doesn't matter}
explore bias from the perspective of book \emph{authors}. This is somewhat analogous to the idea of $P$-fairness from \Cref{sec:multisided}, given that we are interested in how recommendations could be biased against `producers' (in this case, authors of a certain gender).

%Data is 
Book reviews and metadata are
collected from
\emph{BookCrossing} \citep{ziegler2005improving},
\emph{Amazon} \citep{mcauley2015image},
and \emph{GoodReads} \citep{wan2018item}. An interesting component of the study is how these datasets can be augmented to incorporate the gender of each author, which is not a feature immediately available in any of the above datasets; author gender information is compiled from external sources, which is matched to records in each of the datasets.


%The main research thrust in
\cite{ekstrand2018exploring} begin by analyzing the overall gender distribution of authors in the datasets, as compared to the gender distribution among reading histories of individual users. Beyond this, they seek to study how gender bias is `propagated' by recommendation algorithms, i.e.,~the extent to which users who exhibit a moderate tendency toward authors of a certain gender will tend to have recommendations in which that gender is more extremely over-represented. Finally, they analyze the extent to which these issues can be mitigated algorithmically.

Ultimately, the study concludes that authors in all three datasets (at least those whose identities could be resolved) are predominantly male. In terms of rating histories by users, the distribution is less skewed. 
In terms of recommendation algorithms, results are quite mixed, with certain algorithms and datasets leading to more or less skewed recommendations, or otherwise recommendations that mimic users' own gender preferences.

Finally, the authors find that gender imbalance in recommendations can be mitigated easily via simple re-ranking strategies, with minimal impact on performance.
%(the strategy they use is similar to the one of \cite{ziegler2005improving}, as we studied in \Cref{sec:topicDiverse}).
%\done{section still exists? No}
This analysis bears some similarity to our study of filter bubbles (\cref{subsec:filter}), or the techniques used to calibrate recommendations (in this case to match a desired gender distribution rather than a genre distribution) from \Cref{subsec:calibration}.
%or extremification (\cref{sec_?})
%\done{referenc from calibartion section. No}


%\done{Make sure I use this citation the first time I introduce the GoodReads Data in chapter 1 \citep{wan2018item}}

\subsection{Gender Bias in Marketing}
\label{subsec:genderMarket}

\cite{wan2020addressing} investigate bias in terms of how products are \emph{marketed}. For example, a user may be more (or less) inclined to purchase a clothing item if it is modeled by somebody sharing their gender, weight, age, skin-tone, etc. In some instances, these features may be directly relevant to the suitability of the item, but in others they may not be. If users are disinclined to interact with items simply because their own identity is not represented, this reduces the utility of the system to the users, represents a missed opportunity in terms of sales, and raises broader issues of representation in marketing. `Fairness' from this perspective is 
%somewhat between $C$- and $P$-fairness
an instance of $\mathit{CP}$-fairness
from \Cref{sec:multisided}, as both producers and consumers 
%stand to be harmed.
face consequences from unfair treatment.

Like \cite{ekstrand2018exploring},
%the above
%\done{check}
%study, 
\cite{wan2020addressing} start by assessing the extent to which these types of bias can be found in historical interactions (in their case, purchases). They consider two settings: clothing, using a dataset from \emph{ModCloth},
%\done{italics. Like this}
and electronics, using data from \emph{Amazon}. On \emph{ModCloth}, they are interested in whether users are less inclined to buy items if the model has a different body type than the user (e.g.~the user is plus-size but the model is not, even though the item is available in plus-sizes). On \emph{Amazon}, they are interested in whether ostensibly `genderless' products have different sales patterns among male and female users, based on their marketing images.

Again, the study faces difficult issues of augmenting the data, since gender and size attributes of users are not readily available. \emph{ModCloth} specifies the size of the models in marketing images, and user sizes are inferred from their historical tendency to purchase only items of a certain size. Data augmentation is more difficult on \emph{Amazon}: gender attributes in marketing images must be inferred using computer vision techniques; gender attributes of users are inferred from their purchases in the \emph{clothing} category.\footnote{Of course, clothing purchases are a rough proxy for gender identity, and users whose purchases span both gender categories are not considered.}

Indeed, the study determines that there is significant correlation between users' attributes and their purchase patterns (e.g.~male users tend to purchase electronics items marketed by male models).
%
Of course, as with gender in book recommendations
%above
%\done{check. Removed}
(\cref{sec:genderBook}), it is hard to disentangle `bias' or `unfairness' from users' intrinsic preferences or legitimate marketing choices (e.g.~the reason that women tend to buy women's watches may be largely practical). However the goal is to determine whether bias is \emph{amplified} by recommender systems, and whether this effect can be mitigated.

The specific question that is asked is whether recommendation \emph{errors} are correlated with market segments and marketing images. This bears passing similarity to the notion of \emph{absolute unfairness} as in \Cref{eq:absoluteUnfairness}, though that measure considers unfairness only from the perspective of the \emph{user's} identity, whereas the question asked here concerns both user and item `identity' simultaneously. Specifically, four possible types of error are investigated:
\begin{equation}
\begin{tabular}{cc}
$\text{Product Image}\left\lbrace
\begin{tabular}{c}
Female\\
Male
\end{tabular}\right.$ \hspace{-6mm} &
$\left[ \begin{tabular}{cc}
$\bar{e}_{F,F}$ & $\bar{e}_{M,F}$\\
$\bar{e}_{F,M}$ & $\bar{e}_{M,M}$
\end{tabular} \right]$\\
 & $\underbrace{\begin{tabular}{cc}
 Female & Male
 \end{tabular}}_{\text{\normalsize User Identity}}$
 \end{tabular}.
\end{equation}
Under a null model, the errors should not be correlated with market segments (this can be measured via a specific statistical test).

Finding that these errors are indeed significantly correlated with market segments, \cite{wan2020addressing} seek to address this via a loss which balances model error and error correlation:
\begin{equation}
    \overbrace{\sum_{u,i} (f(u,i) - r_{u,i})^2}^{\text{prediction error}} + \alpha \underbrace{\mathcal{L}_{\mathit{corr}}}_{\mathclap{\text{error parity on market segments}}}.
    %\done{make not look ugly. Looks fine}
\end{equation}
Again, this joint loss can be optimized much like the one in \Cref{eq:fairnessTradeOff}, satisfying the fairness objective with minimal loss in prediction accuracy.

Ultimately, the above case-studies demonstrate that even with regard to a single characteristic (gender), the potential fairness consequences are surprisingly varied and require careful attention to resolve.

%\done{closing sentence}

% \section{Bias and the ‘Long-Tail’ of Personalization}

%Paper:All The Cool Kids, How Do They Fit In?
%Moved this to fairness section



% \section{Justification and Trust}

% %\done{drop}

% Building User Trust in Recommendations via Fairness and Explanations

% Designing and evaluating explanations for recommender systems


% Amazon papers:
% ?

% Netflix:
% Big \& Personal: data and models behind Netflix recommendations (talks about explainability, but not very good)


%\section{Case Studies from Industry}



%Paper: Algorithmic Effects on the Diversity of Consumption on Spotify 

%\done{Need to organize these papers. Should there be a separate `filter bubble' section, or a separate 'diversity' section? Current org is fine}

% \section{Exercises}
%\done{Sometimes called problems, sometimes called exercises?}
%\done{Remove section heading, use subsection* for project. "exercises" environment already shows up in ToC}

\begin{exercises}
\item 
\label{ex:diversity1}
% %Max Marginal Relevance on some recommendation problem
In this exercise we'll explore 
%extending a 
recommender systems that balance relevance with diversity. 
You may base your implementation on the code and data from \Cref{sec:implementDiverse}.
%\done{Wrong section reference?}
%Start by implementing a standard latent factor recommender, as in \Cref{chap:recsys} (or borrowing one from a previous exercise). A good dataset for this task might be one that includes category information (such as the beer dataset from \Cref{ref_?}), or some other feature that allows you to assess recommendation diversity. In this exercise we'll implement the \emph{Maximal Marginal Relevance} (MRR) criterion as in \Cref{sec:MMR} (though using a DPP (\cref{sec:DPP}) or another diversity criterion would be similar).
% %\done{reword a little. Dropped}
Start by experimenting with a variety of diversity objectives. For example:
\begin{itemize}
    \item Replacing the cosine similarity ($\texttt{sim}$) with other similarity functions based on item representations.
    \item Use similarity functions based on item \emph{features}. For example, a simple diversity function might simply measure whether two items belong to a different category, or have a different ABV (etc.).
    \item Replace the Maximal Marginal Relevance criterion with an alternative from \Cref{sec:diversification}, e.g.~as in \Cref{eq:adomDiv} or \Cref{eq:dpp1}.
    %\done{imprecise. Okay}
\end{itemize}
%Next, try to evaluate the trade-off between relevance and diversity quantitatively. 
Evaluating diversification techniques is difficult, since they make a qualitative improvement at the cost of a quantitative metric. Evaluate your diversification techniques by plotting a relevance metric (such as those in \Cref{sec:evaluateRec}) as the diversity parameter (e.g.~$\lambda$ in \Cref{eq:MMR}) changes. Does your plot contain an `elbow,' i.e.,~a region in which diversity is significantly increased without sacrificing relevance?
% We'll implement and evaluate the model in the following way:
% \begin{itemize}
%     \item \emph{Relevance} can be defined in the usual way, i.e.,~the inner product of user and item terms $\gamma_u \cdot \gamma_i$. \emph{Diversity} could be defined in various ways, e.g.~in terms of the difference between item features $\| \gamma_i - \gamma_j \|$,
%     %\done{How do I normally write distance functions? Dropped}
%     or even in terms of some simple attribute (e.g.~if you wanted to recommend beers with different alcohol levels, you could define diversity in terms of the ABV feature).
%     \item Use the MMR technique to generate recommendation lists for a sample of users. The 
% \end{itemize}
%\done{Drop the above one since there's starter code now}
\item 
\label{ex:concentration}
%Measure whether a recommender increases or decreases diversity / concentration compared to training data.
In addition to issues of diversity as we saw in Exercise~\ref{ex:diversity1}, we also studied \emph{concentration} effects in \Cref{subsec:simulation,subsec:empirical}, whereby a recommender system can skew the distribution of recommended items toward a smaller set of items than those represented in the training data. In \Cref{sec:measuringGini} we measured concentration in terms of the Gini coefficient (\cref{eq:gini}) of historical data versus recommendations. Consider some strategies to reduce the concentration among recommendations. For example:
\begin{itemize}
    \item Explicitly penalize highly popular (or highly recommended) items from being recommended too often (e.g.~by adding a small negative bias to popular items).
    \item Incorporate a diversification strategy such as one from Exercise~\ref{ex:diversity1}.
    \item Add some small amount of randomization into recommendations.
\end{itemize}
Note that one can trivially produce recommendations that are less concentrated simply by recommending uniformly at random. As in Exercise~\ref{ex:diversity1}, see if you can produce a strategy that improves concentration (in terms of the Gini coefficient) without significantly harming relevance metrics.
%\done{poorly worded. Fine}
\item In \Cref{sec:fairness} we developed various fairness objectives for personalized recommender systems; although we'll explore these objectives more in Project~\ref{proj:fairBeer}, for the moment let's consider the notion of \emph{demographic parity} as in \Cref{eq:demographic}. In \cite{ekstrand2018exploring} 
%the authors measured 
demographic parity 
is measured
with respect to gender (of book authors), though for the purpose of this exercise you could consider any attribute associated with the items (e.g.~whether a beer is low or high ABV). For a few such attributes, compare the training distribution (i.e.,~proportion of historical interactions having that attribute) to the recommendation distribution. Consider whether you can design simple strategies to correct any disparity, e.g.~by systematically assigning higher relevance scores to items from an under-represented class.
% \item Some other re-ranking task
%\item The dataset of beer reviews (from e.g.~\Cref{sec:someExample}) contains a considerable level of gender imbalance among users. After training a recommender system to predict ratings on this data (following e.g.~the code from \Cref{sec:surprise}), compute the four fairness metrics from \Cref{sec_?} (i.e.,~value unfairness, absolute unfairness, under- and over-estimation unfairness), comparing male ($g$) to non-male ($\neg g$) users.
%\done{Drop this question, since we study beer in the project}
%\item Study beer dataset (or some other dataset) with respect to gender bias. Compute the various unfairness metrics on the data.
\end{exercises}

\refstepcounter{projectCounter}
\label{proj:fairBeer}
\section*{Project \theprojectCounter: Diverse and Fair Recommendations}
\addcontentsline{toc}{section}{Project \theprojectCounter: Diverse and Fair Recommendations}
%\done{Some project labels are wrong since they apply to the subsection. Fix and check. Fixed I think, check}

In this project we'll consider how we can improve the outputs of the types of recommendation approach we originally developed in \Cref{chap:modelbasedRecsys}. 
%We'll consider the same beer datasets we've been using in various examples throughout the book.
%Several datasets could be used for this project, such as a dataset 
Select a dataset
that includes a gender attribute, such as the beer data we used in \Cref{sec:binaryFeat}, or others from \Cref{sec:genderCaseStudies}. A suitable dataset would be one that:
%We select this dataset for two reasons:
%Such a 
\begin{itemize}
    \item 
    Contains a gender attribute
    %(like the datasets used in \Cref{sec:genderCaseStudies}), 
    and 
    %in particular 
    is imbalanced with respect to this attribute (e.g.~the majority of users are male); %thus 
    in such a dataset
    we might be concerned that recommendations will have reduced utility for the underrepresented group.
    \item 
    %It contains item metadata, such as beer styles, which can be used to measure recommendation diversity, calibration, etc.
    Contains item metadata, such as categories, prices, or other item attributes, which can be used to measure recommendation diversity, calibration, etc.
\end{itemize}
In principle this project could be completed with any similar dataset that includes (a) an attribute of interest with respect to which we can measure bias, such gender, age, etc.;~and (b) item metadata with respect to which we can measure diversity.

Use this dataset to analyze diversity and fairness from the following perspectives:
\begin{enumerate}
    \item Implement a recommender system to predict ratings in the dataset, e.g.~a latent factor model as in \Cref{subsec:matrixFactor}.
    \item Using the above model, compute the four fairness metrics from \Cref{sec:fairness} (i.e.,~value unfairness, absolute unfairness, under- and over-estimation unfairness), comparing male ($g$) to non-male ($\neg g$) users.\footnote{Or whichever group $g$ is over-represented.}
    \item Next, assess recommended items in terms of diversity. Diversity could be measured in several ways, for example you 
    %\done{Do I ever refer to the reader as 'you'? Yes, sometimes}
    could measure diversity with respect to the distribution of recommend items, or with respect to some attribute (e.g.~the style or brand). This could be a formal measure of dispersion such as the Gini coefficient (as in \cref{eq:gini}), or a plot of recommendation versus interaction frequency as in \Cref{fig:recDiversityComics}.
\end{enumerate}


%\done{make copy of references withou page numbers}


% \chapter{Further Reading}
% \label{chap:further}



  \backmatter
%   \appendix
% if you only have one appendix, use \oneappendix instead of \appendix
%   \include{theorem}
%   \include{root}
%   \include{appnum}
%   \endappendix
%   \addtocontents{toc}{\vspace{\baselineskip}}

% the following lines will give you references at the end of the book
  \renewcommand{\refname}{Bibliography}% if you prefer this heading
%   \bookreferences % if you already have references at the end of chapters,
                  % you will need this command to start a new \chapter* heading
  \bibliography{refsnopage}\label{refs}
  \bibliographystyle{cambridgeauthordate}

%   \cleardoublepage

% end notes, if you have them
% \theendnotes


% glossary
%   \printglossary

% indexes

% for a single index
\printindex

% for multiple indexes using multind.sty
%   \printindex{authors}{Author index}
%   \printindex{subject}{Subject index}

\end{document}
